From fbf4266e7729a8f7e7899c56683d8ea4e82321f9 Mon Sep 17 00:00:00 2001 From: jp170na Date: Tue, 27 Oct 2026 03:42:44 +0100 Subject: [PATCH] =?UTF-8?q?Opravy=20RAG=20odpoved=C3=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 12 +++ scripts/verify_project.py | 97 +++++++++++++++---- test/test_verified_rag_extended.py | 75 ++++++++++++++ .../evaluate_verified_rag.py | 23 +++-- tools/answer_evaluation/rag_evidence.py | 3 +- tools/answer_evaluation/verified_rag.py | 55 +++++++++-- tools/rag/rag_query_evidence.py | 14 ++- tools/rag/tests/test_rag_query_evidence.py | 8 ++ 8 files changed, 251 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 63a63c7..8c2144e 100644 --- a/README.md +++ b/README.md @@ -1156,6 +1156,11 @@ Lokálne testy: .venv/bin/python scripts/verify_project.py --git-network ``` +Bez vytvárania reportov a logových súborov pridajte `--no-save`. +Výsledky kontrol sa vtedy zobrazia iba v termináli. Overenie zahŕňa aj +skutočné tokenové limity všetkých chunkov a embeddingových segmentov, +zachovanie obsahu pri segmentácii a úplnosť uložených vektorov. + Testy používajú skutočný dočasný HTTP server na loopback rozhraní a FastAPI s riadenými dátami. Overujú autorizáciu, zachovanie otázky, zdrojov a úplnosti, chyby a timeouty, zákaz presmerovaní aj import JSON/ZIP @@ -1436,6 +1441,13 @@ explicitný dôkaz roku práce. Táto neistota je uvedená v datasete. .venv/bin/python -m tools.answer_evaluation.evaluate_verified_rag --mode replay --input ``` +Aj evaluačný runner podporuje `--no-save`; v tomto režime otázky a odpovede +spracuje v pamäti a vypíše skóre bez zápisu reportu či deklarácie. +Kontrolovaná cesta používa pri všeobecných otázkach iba doslovné výňatky +bez nežiadaných akademických tvrdení. Referenčný odkaz v citovanom texte +sa prijme iba vtedy, keď je doložený tým istým dodaným zdrojom; položky +`source_urls` naďalej označujú pôvodné wiki zdroje. + Runner posiela na backend iba otázku a parametre; očakávané odpovede model nedostane. Vyhodnocuje finálny stav, prijatie odpovede, citácie, úplné väzby osoba–práca–typ–rok a pokrytie očakávaných údajov vo výňatkoch. diff --git a/scripts/verify_project.py b/scripts/verify_project.py index cd2105e..8e4df67 100644 --- a/scripts/verify_project.py +++ b/scripts/verify_project.py @@ -16,6 +16,7 @@ import sqlite3 import subprocess import sys import time +import tempfile import types import urllib.request import zipfile @@ -66,7 +67,7 @@ def local_test_worker() -> int: socket.socket.connect, socket.socket.connect_ex = guarded_connect, guarded_connect_ex socket.getaddrinfo = guarded_dns import pytest - return pytest.main(['-q', 'test', 'tools/rag/tests', 'tools/graphrag/tests']) + return pytest.main(['-q', '-p', 'no:cacheprovider', 'test', 'tools/rag/tests', 'tools/graphrag/tests']) def verify_code() -> dict: @@ -99,6 +100,44 @@ def verify_database() -> dict: return counts +def verify_chunk_limits() -> dict: + import numpy as np + from tools.rag.build_chunks import TokenCounter + from tools.rag.embedding_utils import build_embedding_text, split_passage, passage_token_count, get_embedding_model, embedding_model_name + chunks = json.loads((ROOT/'data/chunks.json').read_text()) + if isinstance(chunks, dict): + chunks = chunks['chunks'] + counter = TokenCounter(os.getenv('CHUNK_TOKEN_ENCODING', 'cl100k_base')) + limit = int(os.getenv('CHUNK_MAX_TOKENS', '450')) + model = get_embedding_model(embedding_model_name()) + maximum = 0 + expected_segments = set() + with sqlite3.connect((ROOT/'data/zp_index.sqlite').resolve().as_uri()+'?mode=ro',uri=True) as conn: + stored = dict(conn.execute('SELECT chunk_id,text FROM chunks')) + assert len(stored) == len(chunks) + for chunk in chunks: + assert stored[chunk['chunk_id']] == chunk['text'] + size = counter.count(chunk['text']) + maximum = max(maximum, size) + assert size <= limit, 'Oversized source chunk' + original = build_embedding_text(chunk) + parts = split_passage(original) + assert ''.join(parts) == original, 'Embedding segmentation loses content' + for index, part in enumerate(parts): + assert passage_token_count(part) <= model.max_seq_length, 'Oversized embedding input' + expected_segments.add((chunk['chunk_id'],index)) + actual_segments = set() + for cid,index,dimensions,blob in conn.execute('SELECT chunk_id,segment_index,dimensions,embedding FROM chunk_embeddings'): + vector = np.frombuffer(blob,dtype=np.float32) + assert len(vector)==dimensions and np.isfinite(vector).all() + assert np.isclose(np.linalg.norm(vector),1.0,atol=0.001), 'Invalid embedding vector' + actual_segments.add((cid,index)) + assert actual_segments == expected_segments, 'Embedding coverage mismatch' + return {'chunks':len(chunks),'max_chunk_tokens':maximum,'chunk_limit':limit, + 'embedding_segments':len(actual_segments),'embedding_limit':model.max_seq_length, + 'source_content_preserved':True} + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--docker-context', default='default') @@ -106,6 +145,7 @@ def main() -> int: parser.add_argument('--wait-ready-seconds', type=int, default=90, help='Bounded wait for API/model warm-up after Docker startup') parser.add_argument('--output', type=Path) + parser.add_argument('--no-save', action='store_true', help='Check in memory without writing reports or logs') parser.add_argument('--test-worker', action='store_true', help=argparse.SUPPRESS) args = parser.parse_args() os.chdir(ROOT) @@ -113,9 +153,12 @@ def main() -> int: if args.test_worker: return local_test_worker() from tools.runtime.report_files import report_output, report_log, retention_plan, apply_moves - args.output = report_output(args.output, 'verification') + if args.no_save and args.output: + parser.error('--no-save cannot be combined with --output') + args.output = None if args.no_save else report_output(args.output, 'verification') report = {'checks': {}, 'external_llm_calls': False} - args.output.parent.mkdir(parents=True, exist_ok=True) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) def check(name, action): started = time.monotonic() @@ -126,7 +169,8 @@ def main() -> int: record = {'passed': False, 'error_type': type(error).__name__} record['seconds'] = round(time.monotonic() - started, 2) report['checks'][name] = record - args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + if args.output: + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') print(f'{name}: {"OK" if record["passed"] else "FAILED"}', flush=True) def command(*argv): @@ -163,6 +207,7 @@ def main() -> int: check('api_and_both_indexes', health) check('sqlite_integrity', verify_database) + check('chunk_and_embedding_limits', verify_chunk_limits) check('container_readiness', lambda: command(*compose, 'exec', '-T', 'zp-agent-api', 'python', '-m', 'tools.runtime.runtime_health')) if args.git_network: @@ -183,11 +228,11 @@ def main() -> int: assert key, 'Search API key is missing' built = [] - async def run(): + async def run(package_dir): graph_tool = None versions = [] for engine in ('rag', 'graphrag'): - package = build_package(engine, ROOT / 'data/tool_packages') + package = build_package(engine, package_dir) with zipfile.ZipFile(package['archive']) as archive: form = json.loads(archive.read('openwebui-tool.json'))[0] module = types.ModuleType('verify_packaged_' + engine) @@ -213,32 +258,46 @@ def main() -> int: assert negative['status'] == 'no_answer' and not negative['facts'] graph_tool.valves.BACKEND_API_KEY = 'invalid-test-key' assert json.loads(await graph_tool.retrieve_zpwiki_graph('Práca?'))['reason'] == 'unauthorized' - asyncio.run(run()) + if args.no_save: + with tempfile.TemporaryDirectory(prefix='zp-verify-packages-') as folder: + asyncio.run(run(Path(folder))) + else: + asyncio.run(run(ROOT / 'data/tool_packages')) return {'packages': built, 'live_retrieval': True, 'same_data_version': True, 'negative_query': True, 'authorization': True, 'external_llm_calls': False} check('openwebui_tool_packages', tool_packages) def tests(): - log = report_log(args.output) environment = {**os.environ} environment.pop('OPENWEBUI_API_KEY', None) - with log.open('w') as output: - result = subprocess.run([sys.executable, str(Path(__file__).resolve()), '--test-worker'], - env=environment, stdout=output, stderr=subprocess.STDOUT, timeout=900) - assert result.returncode == 0, 'Regression failed; see local pytest log' - text = log.read_text() + environment['PYTHONDONTWRITEBYTECODE'] = '1' + command = [sys.executable, str(Path(__file__).resolve()), '--test-worker'] + if args.no_save: + result = subprocess.run(command, env=environment, capture_output=True, text=True, timeout=900) + text = result.stdout + result.stderr + if result.returncode: + print(text[-16000:], flush=True) + log = None + else: + log = report_log(args.output) + with log.open('w') as output: + result = subprocess.run(command, env=environment, stdout=output, stderr=subprocess.STDOUT, timeout=900) + text = log.read_text() + assert result.returncode == 0, 'Regression failed; see console or selected pytest log' counts = {name: int(value) for value, name in re.findall(r'(\d+) (passed|failed|skipped)', text)} assert counts.get('passed', 0) > 0 and counts.get('skipped', 0) == 0 - return {**counts, 'log': str(log), 'external_network_blocked': True} + print('Regression: ' + str(counts), flush=True) + return {**counts, 'log': str(log) if log else None, 'external_network_blocked': True} check('full_local_regression', tests) report['passed'] = all(item['passed'] for item in report['checks'].values()) - args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') - moves = retention_plan() - if moves: - apply_moves(moves) - print('Report: ' + str(args.output)) + if args.output: + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + moves = retention_plan() + if moves: + apply_moves(moves) + print('Report: ' + str(args.output)) return 0 if report['passed'] else 1 diff --git a/test/test_verified_rag_extended.py b/test/test_verified_rag_extended.py index 99e6aa0..98af2c5 100644 --- a/test/test_verified_rag_extended.py +++ b/test/test_verified_rag_extended.py @@ -130,3 +130,78 @@ def test_evaluator_accepts_extra_supported_project_but_rejects_wrong_rendered_ye assert evaluate_case(case,p,delivered)['passed'] delivered['answer'] += '\nPráca bola ukončená v roku 2099.' assert 'delivered_text_differs_from_verified_renderer' in evaluate_case(case,p,delivered)['issues'] + + +@pytest.mark.parametrize('reference', ['https://library.example/docs', '/topics/library', + '../manual', '#installation', '[guide]: /guide']) +def test_literal_source_references_are_allowed_without_replacing_wiki_citation(reference): + from tools.answer_evaluation.verified_rag import quoted_reference_urls, render_rag_answer + from tools.answer_evaluation.context_policy import citation_check, allowed_urls + quote = ('Dokumentácia: '+reference if not reference.startswith('[guide]') else reference) + if reference.startswith(('/', '.', '#')): + quote = '[Dokumentácia]('+reference+')' + c = rag_evidence_context('Aká dokumentácia je uvedená?',payload(quote)) + a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'], + 'evidence_quotes':[{'source_id':'S1','quote':quote}]} + assert rag_answer_issues(a,c) == [] + rendered = {**a,'answer':render_rag_answer(a,c)} + assert citation_check(rendered,allowed_urls(c)|quoted_reference_urls(a,c))['citations_ok'] + refs = quoted_reference_urls(a,c) + assert refs + a['source_urls'].extend(refs) + assert 'unsupported_citation' in rag_answer_issues(a,c) + + +@pytest.mark.parametrize('quote', ['Pozri https://unknown.example/doc', + '[manuál](/unknown)', 'manuál']) +def test_quote_cannot_borrow_an_unknown_or_other_source_reference(quote): + p = payload('Presný obsah bez odkazu.') + p['sources'].append({**p['sources'][0],'source_id':'S2','title':'Iný dokument', + 'source_url':'https://example.test/other','text':quote}) + p['context'] = build_context_text(p['sources']) + c = rag_evidence_context('Aký je obsah dokumentu?',p) + a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'], + 'evidence_quotes':[{'source_id':'S1','quote':quote}]} + assert 'unsupported_quote_citation' in rag_answer_issues(a,c) + + +def test_active_url_schemes_and_noncontiguous_quotes_are_rejected(): + c = rag_evidence_context('Aký je obsah dokumentu?',payload('[odkaz](javascript:alert(1))\nPrvý fakt.\nMedziľahlý text.\nDruhý fakt.')) + a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'], + 'evidence_quotes':[{'source_id':'S1','quote':'[odkaz](javascript:alert(1))'}]} + assert 'unsupported_quote_citation' in rag_answer_issues(a,c) + a['evidence_quotes']=[{'source_id':'S1','quote':'Prvý fakt.\nDruhý fakt.'}] + assert 'unsupported_evidence_quote' in rag_answer_issues(a,c) + + +def test_general_question_repairs_unrequested_claims_and_sends_only_quote_evidence(): + p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu\nPoužíva Python.') + bad = {'status':'ok','answer':'','claims':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025}], + 'evidence_quotes':[{'source_id':'S1','quote':'Používa Python.'}], + 'source_urls':['https://example.test/alena']} + good = {**bad,'claims':[]} + outputs = [bad, good]; calls=[] + def complete(messages): + calls.append(json.loads(messages[1]['content'])) + return {'choices':[{'message':{'content':json.dumps(outputs[len(calls)-1])}}]} + delivered = generate_verified_rag('Aký jazyk používa Alena Nová?',p,complete) + assert delivered['accepted_for_delivery'] and len(calls)==2 + assert 'unrequested_structured_claims' in delivered['attempts'][0]['issues'] + assert all('facts' not in call['context'] for call in calls) + assert delivered['claims']==[] and 'Používa Python.' in delivered['answer'] + + +def test_evaluator_accepts_exact_quoted_reference_but_rejects_invented_rendered_link(): + from tools.answer_evaluation.verified_rag import render_rag_answer + p = payload('Knižnica Library: https://library.example/docs') + question = 'Aká knižnica je uvedená?' + c = rag_evidence_context(question,p) + parsed = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'], + 'evidence_quotes':[{'source_id':'S1','quote':'Knižnica Library: https://library.example/docs'}]} + delivered = {**parsed,'accepted_for_delivery':True,'answer':render_rag_answer(parsed,c), + 'attempts':[{'parsed_answer':parsed,'issues':[]}]} + case = {'id':'reference','category':'specific_fact','question':question,'should_answer':True, + 'expected_bindings':[],'expected_source_urls':parsed['source_urls'],'expected_answer_contains':['Library']} + assert evaluate_case(case,p,delivered)['passed'] + delivered['answer']+=' https://invented.example/' + assert 'delivered_unsupported_citation' in evaluate_case(case,p,delivered)['issues'] diff --git a/tools/answer_evaluation/evaluate_verified_rag.py b/tools/answer_evaluation/evaluate_verified_rag.py index e8b8563..9d60e61 100644 --- a/tools/answer_evaluation/evaluate_verified_rag.py +++ b/tools/answer_evaluation/evaluate_verified_rag.py @@ -19,7 +19,7 @@ from dotenv import load_dotenv from tools.answer_evaluation.context_policy import normalized, citation_check, allowed_urls from tools.answer_evaluation.rag_evidence import rag_evidence_context -from tools.answer_evaluation.verified_rag import rag_answer_issues, render_rag_answer +from tools.answer_evaluation.verified_rag import rag_answer_issues, render_rag_answer, quoted_reference_urls from tools.graphrag.evaluation.source_oracle import DEFAULT_WIKI, load_source_oracle from tools.rag.evaluation.rag_metrics import expected_phrase_matches from tools.runtime.report_files import report_output, resolve_report_input @@ -86,7 +86,8 @@ def evaluate_case(case: dict, payload: dict, delivered: dict | None = None) -> d issues.append('delivered_text_differs_from_verified_renderer') if delivered.get('claims', []) != parsed.get('claims', []): issues.append('delivered_claims_differ_from_verified_selection') - if accepted and not citation_check(delivered, source_urls)['citations_ok']: + references = quoted_reference_urls(parsed or {}, context) + if accepted and not citation_check(delivered, source_urls | references)['citations_ok']: issues.append('delivered_unsupported_citation') claims = delivered.get('claims', []) if accepted: @@ -150,6 +151,7 @@ def main(): parser.add_argument('--dataset', type=Path, default=DATASET) parser.add_argument('--backend', default='http://127.0.0.1:8000') parser.add_argument('--output', type=Path) + parser.add_argument('--no-save', action='store_true', help='Print results only; create no reports or logs') args = parser.parse_args() load_dotenv(ROOT / '.env', override=False) dataset = json.loads(args.dataset.read_text()) @@ -157,7 +159,9 @@ def main(): oracle = load_source_oracle() audit = {c['id']: source_audit(c, oracle) for c in dataset['cases']} assert not any(audit.values()), 'Expected answer/source audit failed' - output = report_output(args.output, 'rag_verified') + if args.no_save and args.output: + parser.error('--no-save cannot be combined with --output') + output = None if args.no_save else report_output(args.output, 'rag_verified') files = [Path(__file__), ROOT/'tools/answers/answer_delivery.py', ROOT/'app/routes.py', *[(ROOT/'tools/answer_evaluation'/name) for name in ['verified_rag.py','rag_evidence.py','context_policy.py']], ROOT/'tools/rag/rag_utils.py', ROOT/'tools/rag/llm_context.py'] @@ -167,7 +171,8 @@ def main(): 'code_hashes': {str(p.relative_to(ROOT)): digest(p) for p in files}, 'model_calls': args.mode == 'deliver', 'backend': args.backend, 'model': os.getenv('ANSWER_MODEL', 'model120-fast') if args.mode == 'deliver' else None} - output.with_suffix('.declaration.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2)+'\n') + if output: + output.with_suffix('.declaration.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2)+'\n') key = os.getenv('SEARCH_API_KEY', '') saved = None if args.mode == 'replay': @@ -207,11 +212,17 @@ def main(): for future in as_completed(futures): rows.append(future.result()) result = {'configuration': metadata, 'summary': summarize(rows), 'results': sorted(rows, key=lambda r: r['id'])} - output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') + if output: + output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') print(f'{len(rows)}/{len(dataset["cases"])} {rows[-1]["id"]}: {rows[-1].get("issues", "retrieved")}', flush=True) assert metadata['code_hashes'] == {name: digest(ROOT/name) for name in metadata['code_hashes']}, 'Code changed during measurement' print(json.dumps(result['summary'], ensure_ascii=False)) - print('Report: '+str(output)) + if output: + print('Report: '+str(output)) + if args.no_save: + print(json.dumps({'failures':[{'id':r['id'],'issues':r['issues'], + 'attempt_issues':[a.get('issues') for a in r.get('delivered',{}).get('attempts',[])]} + for r in result['results'] if not r['passed']]}, ensure_ascii=False)) if __name__ == '__main__': diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index a02c4cc..c31ce0c 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -239,7 +239,8 @@ def rag_evidence_context(question: str, payload: dict) -> dict: requirements = {'people': requested, 'work_types': expected_types, 'academic': academic, 'title': 'nazov' in query or 'tema' in query, 'year': academic and ('rok' in query or 'porovnaj' in query), - 'start_year': start_year} + 'start_year': start_year, + 'response_mode': 'claims' if academic or start_year else 'quotes'} stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else [] requirements['years'] = ([int(value) for value in stated_years if len(value) == 4] if not requirements['start_year'] else []) diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index 3164cf7..132392e 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -8,7 +8,7 @@ from typing import Literal from pydantic import BaseModel, ConfigDict, Field from .answer_format import Claim -from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized +from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized, text_urls from .rag_evidence import WORK_LABELS, rag_evidence_context @@ -41,9 +41,17 @@ Pri porovnávaní odpovedz za každú požadovanú osobu; rešpektuj requirement start_year je len explicitný začiatok štúdia, nikdy rok z adresára. Pri otázke na začiatok štúdia vyplň claims so start_year; pri rozlíšení rokov uveď start_year aj year a work_type. Citácia ani nesúvisiaci výňatok údaj nenahrádzajú. -Pri všeobecnej otázke môžeš namiesto claims vybrať evidence_quotes: source_id a presný -doslovný výňatok, ktorý odpovedá na otázku. Výňatok nesmieš parafrázovať ani meniť čísla. +Rešpektuj requirements.response_mode. Pri response_mode=quotes MUSÍ byť claims=[]: +pri všeobecnej otázke, téme, technológii, zozname ľudí alebo stáži vyber iba +evidence_quotes: source_id a presný doslovný výňatok, ktorý odpovedá na otázku. +Nevypĺňaj pri nich názvy prác, roky, tagy ani iné nežiadané metadáta do claims. +Každý výňatok musí byť JEDEN súvislý úsek z konkrétneho zdroja vrátane Markdownu. +Nespájaj oddelené pasáže cez vynechané riadky; použi viac samostatných kratších quotes. +Vyber iba minimum potrebné na zodpovedanie otázky. Výňatok nesmieš parafrázovať ani meniť čísla. +Pri otázke na skutočne použitý nástroj odlišuj dosiahnutý stav od návrhu alebo úlohy. Cituj iba source_urls zo sources, ktoré dokladajú vybrané údaje. Názov profilu nie je názov práce. +Odkaz, ktorý je priamo v doslovnom výňatku, môže zostať v ňom; nenahrádza však wiki +citáciu a nesmieš ho pridávať do source_urls. Odkazy ani obsah si nedomýšľaj. Nedomýšľaj chýbajúce údaje. Ak podklady neobsahujú požadovaný fakt (napríklad finančný rozpočet), vráť no_answer a prázdne claims/evidence_quotes. count nechaj null, ak nie je dodaná úplná agregácia. Nepoužívaj vedomosti ani fakty, ktoré nie sú v podkladoch.''' @@ -59,6 +67,30 @@ def parse_rag_answer(text: str) -> dict: return RagAnswer.model_validate(json.loads(text)).model_dump(exclude_none=True) +def quoted_reference_urls(answer: dict, context: dict) -> set[str]: + """License references only inside an exact, cited quote of that source. + + The structured source_urls still must contain the original wiki citations. + A link elsewhere in the corpus or in another source never licenses a quote. + """ + blocks = {row['source_id']: row for row in context['source_blocks']} + cited = set(answer.get('source_urls', [])) + references = set() + from urllib.parse import urlsplit + for quote in answer.get('evidence_quotes', []): + source = blocks.get(quote['source_id']) + if (not source or source.get('source_url') not in cited or + ' '.join(quote['quote'].split()) not in ' '.join(source['body'].split())): + continue + for url in text_urls(quote['quote']) & text_urls(source['body']): + try: + if urlsplit(url).scheme.lower() in {'', 'http', 'https'}: + references.add(url) + except ValueError: + continue + return references + + def rag_answer_issues(answer: dict, context: dict) -> list[str]: requirements = context['requirements'] claims = answer.get('claims', []) @@ -73,6 +105,8 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: issues.append('evidence_on_nonpositive_answer') if answer['status'] == 'ok' and not claims and not quotes and answer.get('count') is None: issues.append('missing_required_evidence') + if requirements.get('response_mode') == 'quotes' and claims: + issues.append('unrequested_structured_claims') requested = requirements['people'] types = requirements['work_types'] for claim in claims: @@ -130,12 +164,14 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: not any(source.get('document_path', '').startswith(row['document_path'].removesuffix('README.md')) for row in context['source_blocks'] if row.get('title') in requested)): issues.append('unrequested_quote_person') - identity_question = bool(re.search(r'\b(?:url|odkaz\w*|dokument\w*)\b', normalized(context.get('question', '')))) + identity_question = bool(re.search(r'\b(?:url|odkaz\w*|dokument\w*|zaznam\w*|tag\w*|kategor\w*|autor\w*|klucov\w*)\b', normalized(context.get('question', '')))) if (not identity_question and ' '.join(quote['quote'].split()) not in ' '.join(source['body'].split())): issues.append('metadata_quote_does_not_answer_fact') - # A quoted embedded URL is not automatically an allowed citation. - if not citation_check({'answer': quote['quote'], 'source_urls': []}, allowed_urls(context))['citations_ok']: + # Source references may occur inside a verified literal quote, while + # evidence citations themselves remain restricted to the wiki sources. + references = quoted_reference_urls({**answer, 'evidence_quotes': [quote]}, context) + if not citation_check({'answer': quote['quote'], 'source_urls': []}, allowed_urls(context) | references)['citations_ok']: issues.append('unsupported_quote_citation') if not citation_check(answer, allowed_urls(context))['citations_ok']: issues.append('unsupported_citation') @@ -186,6 +222,8 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: 'truncation_reasons', 'index_version', 'sources', 'facts', 'requirements']} if not context['requirements']['academic']: sent['context'] = payload.get('context', '') + if context['requirements']['response_mode'] == 'quotes': + sent.pop('facts') messages = [{'role': 'system', 'content': PROMPT + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)}, {'role': 'user', 'content': json.dumps({'question': question, 'context': sent}, ensure_ascii=False)}] attempts = [] @@ -215,7 +253,10 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: if number == 0: messages += [{'role': 'assistant', 'content': record.get('raw_response', '')}, {'role': 'user', 'content': 'Oprav jediný JSON podľa pôvodných podkladov. Porušenia: ' + ', '.join(issues) - + '. Odpoveď answer nechaj prázdnu. Nevymýšľaj údaje; osoba, typ, názov a rok musia patriť jednému záznamu.'}] + + '. Odpoveď answer nechaj prázdnu. Nevymýšľaj údaje; osoba, typ, názov a rok musia patriť jednému záznamu.' + + (' Pri tejto všeobecnej otázke nastav claims=[] a vyber iba krátke doslovné evidence_quotes. ' + 'Každý quote musí byť samostatný súvislý úsek, nesmie preskočiť riadky ani meniť Markdown. ' + 'Nepotrebné citáty a metadáta vynechaj.' if context['requirements']['response_mode'] == 'quotes' else '')}] return {'status': 'error', 'reason': 'answer_rejected', 'accepted_for_delivery': False, 'answer': 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.', 'attempts': attempts, 'index_version': context['index_version']} diff --git a/tools/rag/rag_query_evidence.py b/tools/rag/rag_query_evidence.py index 523532f..84380e6 100644 --- a/tools/rag/rag_query_evidence.py +++ b/tools/rag/rag_query_evidence.py @@ -32,6 +32,8 @@ FOCUS_LINE_RADIUS = 7 MAX_FOCUS_CHARACTERS = 2200 CANONICAL_TOKEN_ALIASES = { + "aplikacia": "application", "aplikaciu": "application", "aplikacie": "application", + "aplikacii": "application", "application": "application", "applications": "application", "anotacia": "annotation", "anotacii": "annotation", "anotacny": "annotation", @@ -540,6 +542,8 @@ def evidence_score( def _line_score( query: str, line: str, + *, + ignored_tokens: frozenset[str] = frozenset(), ) -> float: normalized_line = ( normalize_for_compare( @@ -550,9 +554,7 @@ def _line_score( score = ( 5.0 * count_token_matches( - query_content_tokens( - query - ), + [token for token in query_content_tokens(query) if token not in ignored_tokens], line, ) ) @@ -579,12 +581,18 @@ def build_focus_excerpt( return "" lines = clean_text.splitlines() + # The generated document header is repeated in every chunk. Its owner's + # name identifies the document, but must not hide a later factual passage + # when choosing an excerpt within that already selected document. + owner = re.search(r'^Dokument:\s*(.+)$', clean_text, re.M) + ignored = frozenset(normalized_tokens(owner[1])) if owner else frozenset() scored = [ ( _line_score( query, line, + ignored_tokens=ignored, ), index, ) diff --git a/tools/rag/tests/test_rag_query_evidence.py b/tools/rag/tests/test_rag_query_evidence.py index cade144..ba62b19 100644 --- a/tools/rag/tests/test_rag_query_evidence.py +++ b/tools/rag/tests/test_rag_query_evidence.py @@ -10,6 +10,14 @@ from tools.rag.rag_query_evidence import ( ) +def test_profile_header_does_not_hide_later_application_evidence(): + text = ('Dokument: Anna Nová\n\nAnna Nová\n' + '\n'.join(['Všeobecná poznámka.']*25) + + '\nState:\n- Proposed own Flask application\n- Created a FastAPI application with a data model.\n') + excerpt = build_focus_excerpt('Aký webový framework použila Anna Nová pre aplikáciu?',text) + assert 'Created a FastAPI application' in excerpt + assert 'Proposed own Flask application' in excerpt + + def make_db( tmp_path: Path, ) -> Path: