diff --git a/README.md b/README.md index 940cd37..35c304a 100644 --- a/README.md +++ b/README.md @@ -1160,7 +1160,7 @@ upozornenie na neúplnosť, nezamieňať autora dokumentu s autorom práce a pri sa nevysvetľujú odhadom; verzia dát neoznačuje verziu programu. Tieto pokyny ovplyvňujú správanie Claude, jeho finálny text však náš backend nemôže programovo vynútiť. Po aktualizácii už nahratých Skills nahraď ich -novými ZIPmi z `packages/claude/dist/` (verzia 1.3.0) a over správanie v novom chate. +novými ZIPmi z `packages/claude/dist/` (verzia 1.4.0) a over správanie v novom chate. ### Nahratie Skills do webového Claude @@ -1503,6 +1503,16 @@ známe rozkazovacie formulácie zadania menia na pomenovania cieľov, napríklad pravidlá po kontrole doslovného dôkazu, nie o voľné dopĺňanie faktov modelom. Neznámy opis, podmienky, negácie a údaje o dosiahnutom stave zostávajú zachované. +Známe gramatické úpravy sa uplatňujú iba na text mimo URL, kódu a HTML značiek. +Citované výňatky v dôkazoch sa nemenia. Konektor posiela aktuálnu otázku bez +parafrázy; môže odstrániť iba úvodný príkaz `/rag` alebo `/graphrag`. Odpoveď +z `/answer` nesie aj `query` a `engine`; klient odmietne nezhodu s požiadavkou. +Skills a inicializačné pokyny MCP vyžadujú porovnať otázku aj s aktuálnou správou +používateľa. Výber otázky samotným Claude zostáva potrebné overiť v jeho rozhraní. +Pri názve a roku práce model dostáva typované fakty a stručné zdroje. RAG tvrdenia +nepoužívajú grafové `record_id` ani `work_label`; každé tvrdenie musí mať doloženú +wiki URL. Označenie zdroja ako `S1` nenahrádza citáciu. + Pri obsahovom porovnaní viacerých študentov sa pre každého načíta požadovaný typ práce z rovnakého SQLite snapshotu; všetky podklady naďalej podliehajú spoločnému tokenovému rozpočtu. Zadanie a zaznamenaný stav sa kontrolujú a zobrazujú oddelene. diff --git a/app/routes.py b/app/routes.py index 9d98c3c..87222e2 100644 --- a/app/routes.py +++ b/app/routes.py @@ -194,6 +194,7 @@ def verified_answer(request: AnswerRequest) -> dict: if (result.get('accepted_for_delivery') is not True or result.get('status') not in {'ok', 'no_answer', 'ambiguous'}): result = public_failure(result) + result={**result,'query':request.query.strip(),'engine':request.engine} if not request.include_diagnostics: result.pop('attempts', None) result.pop('retrieval_context', None) diff --git a/packages/claude/client.py b/packages/claude/client.py index 539b398..748af74 100644 --- a/packages/claude/client.py +++ b/packages/claude/client.py @@ -6,6 +6,7 @@ from dataclasses import dataclass import ipaddress import json import os +import re from pathlib import Path import stat import urllib.error @@ -85,6 +86,8 @@ def error_result(reason): 'unauthorized':'Prístup nebol autorizovaný. Skontroluj prihlásenie konektora a jeho konfiguráciu.', 'forbidden':'Pre tento nástroj nemáš potrebné oprávnenie.', 'invalid_query':'Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.', + 'question_mismatch':'Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.', + 'engine_mismatch':'Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag alebo /graphrag.', 'model_unavailable':'Jazykový model momentálne nie je dostupný. Skús otázku neskôr.', 'answer_rejected':'Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. ' 'Spresni osobu, typ práce alebo rok; neoverený text sa nezobrazí.', @@ -146,8 +149,14 @@ def query_backend(query: str, engine: str, *, mode='answer', limit=5, config=Non """Return only approved answer fields, or explicitly marked retrieval context.""" try: if (engine not in ENGINES or mode not in {'answer', 'retrieve'} or type(limit) is not int - or not 1 <= limit <= 20 or not isinstance(query, str) or not 1 <= len(query.strip()) <= 500): + or not 1 <= limit <= 20 or not isinstance(query, str)): raise ClientError('invalid_query') + query=query.strip() + command=re.match(r'^/(rag|graphrag)(?=\s|$)',query,re.I) + if command: + if command[1].lower()!=engine:raise ClientError('engine_mismatch') + query=query[command.end():].strip() + if not 1 <= len(query) <= 500:raise ClientError('invalid_query') config = config or Config.load() endpoint = '/answer' if mode == 'answer' else '/' + engine body = {'query': query.strip(), 'limit': limit} @@ -170,6 +179,10 @@ def query_backend(query: str, engine: str, *, mode='answer', limit=5, config=Non validate_metadata(value) fields = {'status', 'reason', 'index_version', 'complete', 'truncated', 'truncation_reasons'} if mode == 'answer': + if not isinstance(value.get('query'),str): + raise ClientError('invalid_backend_response') + if value['query']!=body['query'] or value.get('engine')!=engine: + raise ClientError('question_mismatch') if value.get('accepted_for_delivery') is not True: raise ClientError('answer_rejected') if (not isinstance(value.get('answer'), str) or not value['answer'].strip() @@ -184,7 +197,7 @@ def query_backend(query: str, engine: str, *, mode='answer', limit=5, config=Non or any(not isinstance(coverage.get(key), list) or any(not isinstance(item, str) for item in coverage[key]) for key in ['warnings','limitations'])): raise ClientError('invalid_backend_response') - fields |= {'answer', 'source_urls', 'claims', 'count', 'accepted_for_delivery', 'answer_coverage'} + fields |= {'query','answer', 'source_urls', 'claims', 'count', 'accepted_for_delivery', 'answer_coverage'} else: if not isinstance(value.get('sources'), list): raise ClientError('invalid_backend_response') diff --git a/packages/claude/dist/packages.json b/packages/claude/dist/packages.json index bcbfe9a..3090d27 100644 --- a/packages/claude/dist/packages.json +++ b/packages/claude/dist/packages.json @@ -3,12 +3,12 @@ { "archive": "zpwiki-rag.zip", "name": "rag", - "sha256": "09d3e6b196b61f6c025eb86ccd1773b913b37660163a995ad1a2acfa6d947a0c" + "sha256": "0c4b1a65d7ec97e89505000c7f60239ae220098ba11dc8a6ae9fca8be396f151" }, { "archive": "zpwiki-graphrag.zip", "name": "graphrag", - "sha256": "cfcb60213e103d40c2adbc6c7d4133751120148415836cfffa7935c020167e95" + "sha256": "cbfd609ec66c24a03e4c75b33e5a92438a1131c25019df8f2ea522bc917ba5e3" } ], "schema_version": 1 diff --git a/packages/claude/dist/zpwiki-graphrag.zip b/packages/claude/dist/zpwiki-graphrag.zip index 01336d6..e4b6b6c 100644 Binary files a/packages/claude/dist/zpwiki-graphrag.zip and b/packages/claude/dist/zpwiki-graphrag.zip differ diff --git a/packages/claude/dist/zpwiki-rag.zip b/packages/claude/dist/zpwiki-rag.zip index ec09e2d..c75e00c 100644 Binary files a/packages/claude/dist/zpwiki-rag.zip and b/packages/claude/dist/zpwiki-rag.zip differ diff --git a/packages/claude/graphrag/SKILL.md b/packages/claude/graphrag/SKILL.md index ebb4541..40f4b0d 100644 --- a/packages/claude/graphrag/SKILL.md +++ b/packages/claude/graphrag/SKILL.md @@ -10,6 +10,15 @@ For `/graphrag `, send the text after the leading `/graphrag` as the q to `zpwiki_graphrag`. Preserve every condition. If no question follows, ask for one. Use this graph tool rather than RAG when the user explicitly selects `/graphrag`. +For every call, copy the latest user question verbatim after removing only its leading +command. Do not reuse an earlier question or an example, paraphrase, silently correct a +name, add a year or drop a condition. Before calling, check names, work types, years, +negations and comparisons against the current message. Before presenting the result, +check that its `query` matches that question. A mismatch is not a usable answer. +If clarification is needed, ask specifically for the missing or ambiguous condition. +For multiple explicit commands in one message, handle each question separately in order; +keep all conditions belonging to each question together. + 1. With a ZP Wiki MCP connector, call `zpwiki_graphrag` with the complete question. 2. Otherwise, with securely configured backend access in a Python execution environment, run `python scripts/query.py "the complete question"` relative to this directory. diff --git a/packages/claude/mcp_server.py b/packages/claude/mcp_server.py index 6aec4d2..914c3b7 100644 --- a/packages/claude/mcp_server.py +++ b/packages/claude/mcp_server.py @@ -19,6 +19,15 @@ from starlette.responses import JSONResponse # Host-model rules belong to MCP initialization and Skills, not tool-result data. +QUESTION_POLICY = ( + 'For each tool call, copy the latest user question verbatim after removing only its leading ' + '/rag or /graphrag command. Keep names, work types, years, negations, comparisons and every condition. ' + 'Never reuse a previous question or an example, paraphrase it or split its conditions across searches. ' + 'If the user supplies multiple explicit commands, handle each exact question separately in order. ' + 'Before calling, compare the query with the current user message. Respect the explicitly selected engine. ' + 'The returned query is audit data: compare it with the current question before presenting its answer. ' + 'If they differ, do not present the mismatched answer. Ask for a concrete clarification when needed. ' +) ANSWER_PRESENTATION_POLICY = ( 'For this question, present the current accepted answer unchanged, including its sources and caveats. ' 'Do not add factual claims, guesses or commentary from earlier chat answers. ' @@ -71,7 +80,7 @@ async def zpwiki_health() -> CallToolResult: def create_server(**kwargs): server = FastMCP('ZP Wiki', instructions=( 'Use zpwiki_rag for text and individual facts; zpwiki_graphrag for relationships and counts. ' - 'Tools return backend-verified answers. ' + ANSWER_PRESENTATION_POLICY), **kwargs) + 'Tools return backend-verified answers. ' + QUESTION_POLICY + ANSWER_PRESENTATION_POLICY), **kwargs) for handler in (zpwiki_rag, zpwiki_graphrag, zpwiki_health): server.add_tool(handler, annotations=ToolAnnotations( readOnlyHint=True, destructiveHint=False, openWorldHint=True)) diff --git a/packages/claude/rag/SKILL.md b/packages/claude/rag/SKILL.md index c5e541f..aef6d4e 100644 --- a/packages/claude/rag/SKILL.md +++ b/packages/claude/rag/SKILL.md @@ -10,6 +10,15 @@ For `/rag `, send the text after the leading `/rag` as the question to `zpwiki_rag`. Preserve every condition. If no question follows, ask for one. Use this text-search tool rather than GraphRAG when the user explicitly selects `/rag`. +For every call, copy the latest user question verbatim after removing only its leading +command. Do not reuse an earlier question or an example, paraphrase, silently correct a +name, add a year or drop a condition. Before calling, check names, work types, years, +negations and comparisons against the current message. Before presenting the result, +check that its `query` matches that question. A mismatch is not a usable answer. +If clarification is needed, ask specifically for the missing or ambiguous condition. +For multiple explicit commands in one message, handle each question separately in order; +keep all conditions belonging to each question together. + 1. If a ZP Wiki MCP connector is available, call `zpwiki_rag` with the complete question. 2. Otherwise, in a Python execution environment with securely configured backend access, run `python scripts/query.py "the complete question"` relative to this skill's directory. diff --git a/scripts/build_claude_packages.py b/scripts/build_claude_packages.py index 0dd4f7b..c24a8b4 100644 --- a/scripts/build_claude_packages.py +++ b/scripts/build_claude_packages.py @@ -11,7 +11,7 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] BASE = ROOT / 'packages/claude' ENGINES = ('rag', 'graphrag') -PACKAGE_VERSION = '1.3.0' +PACKAGE_VERSION = '1.4.0' def json_bytes(value): diff --git a/test/test_answer_presentation.py b/test/test_answer_presentation.py index ff39ed8..ce2dbeb 100644 --- a/test/test_answer_presentation.py +++ b/test/test_answer_presentation.py @@ -81,6 +81,37 @@ def test_progress_negations_and_unknown_prose_are_not_reinterpreted(text): assert text in rendered and 'tieto ciele' not in rendered +@pytest.mark.parametrize('command',['vyhodnoťte','vyhodnotte','vyhodnnotte']) +def test_known_coordinated_command_variants_are_natural_without_claiming_completion(command): + raw='Preštudujte si spôsoby vyhodnotenia RAG - RAGAS, DeepEval a '+command+' systém.' + text,converted=goal_text(raw) + assert converted and text=='Preštudovať si spôsoby vyhodnotenia RAG - RAGAS, DeepEval a vyhodnotiť systém.' + assert 'vyhodnotený' not in text and 'dokončený' not in text + + +def test_goal_formatting_protects_urls_code_attributes_and_conditions(): + raw=('Vypracujte prehľad iba pre slovenské texty a vyhodnnotte systém podľa ' + 'https://example.test/implementujte/vyhodnnotte?x=navrhnite ' + 's kódom `implementujte("vyhodnnotte")` a odkazom.') + text,converted=goal_text(raw) + assert converted and 'Vypracovať prehľad iba pre slovenské texty a vyhodnotiť systém' in text + assert 'https://example.test/implementujte/vyhodnnotte?x=navrhnite' in text + assert '`implementujte("vyhodnnotte")`' in text and '' in text + + +@pytest.mark.parametrize('complete',[True,False]) +def test_shared_document_authors_have_a_grammatical_bounded_summary(complete): + people=['Eva Nová','Boris Nový'];urls=['https://example.test/eva','https://example.test/boris'] + claims=[{'person':person,'document_author':'Daniel Hladek'} for person in people] + c={'status':'ok','facts':[{**claim,'source_url':url} for claim,url in zip(claims,urls)], + 'sources':[{'source_url':url} for url in urls], 'complete':complete,'truncated':not complete, + 'shared_properties':[{'field':'document_author','value':'Daniel Hladek','people':people}]} + rendered=render_graph_answer({'status':'ok','claims':claims,'source_urls':urls},c) + prefix='Spoloční autori dokumentov:' if complete else 'Doložení spoloční autori dokumentov:' + assert rendered['answer'].startswith(prefix) + assert answer_context_issues(rendered,c)==[] + + def test_assignment_heading_does_not_keep_imperative_style(): lines = content_lines(['*Zadanie diplomovej práce*:', '1. Vypracujte prehľad metód.', '2. Implementujte systém.'], 'Ján Holp', 'diploma_thesis') diff --git a/test/test_comparison_answers.py b/test/test_comparison_answers.py index 8a9f56c..08e6450 100644 --- a/test/test_comparison_answers.py +++ b/test/test_comparison_answers.py @@ -74,10 +74,13 @@ def test_goal_headings_are_not_extra_evidence_and_sentences_keep_capitals(): assert 'Ďalšie podklady' not in text and 'Zlepšiť proces. Zlepšiť extrakciu.' in text -@pytest.mark.parametrize('literal', ['Použite https://example.test/pouzite/vypracujte.', - 'Implementujte parser pre `navrhnite a implementujte`.']) -def test_grammar_does_not_change_urls_or_code_literals(literal): - assert goal_text(literal)==(literal,False) +@pytest.mark.parametrize('literal,expected', [ + ('Použite https://example.test/pouzite/vypracujte.', + 'Použiť https://example.test/pouzite/vypracujte.'), + ('Implementujte parser pre `navrhnite a implementujte`.', + 'Implementovať parser pre `navrhnite a implementujte`.')]) +def test_grammar_does_not_change_urls_or_code_literals(literal,expected): + assert goal_text(literal)==(expected,True) @pytest.fixture diff --git a/test/test_question_delivery.py b/test/test_question_delivery.py new file mode 100644 index 0000000..da9e2e3 --- /dev/null +++ b/test/test_question_delivery.py @@ -0,0 +1,81 @@ +"""Question/engine binding across actual HTTP/MCP boundaries, without an LLM.""" +import asyncio +import json +import os +import sys + +import pytest + +from test_claude_packages import backend, ROOT +from packages.claude import client + + +QUESTIONS=[ + 'Čomu sa venuje diplomová práca Jána Holpa? Stručne ju predstav a uveď zdroj.', + 'Porovnaj zadanie a výsledky Ondreja Megelu a Jána Holpa.\nPlán nie je výsledok; uveď zdroje.', + 'Ktorí študenti majú tag "RAG" a zároveň diplomový projekt z roku 2026?'] + + +@pytest.mark.parametrize('engine',['rag','graphrag']) +def test_latest_question_and_all_conditions_cross_http_unchanged(backend,engine): + for question in QUESTIONS: + r=client.query_backend(question,engine,limit=7) + assert r['accepted_for_delivery'] and r['query']==question and r['engine']==engine + path,body,_=backend['requests'][-1] + assert path=='/answer' and body['query']==question and body['engine']==engine and body['limit']==7 + assert len(backend['requests'])==len(QUESTIONS) + + +@pytest.mark.parametrize('engine',['rag','graphrag']) +def test_only_a_matching_leading_command_is_removed(backend,engine): + question='Aké práce má Ján Holp v roku 2020? Zachovaj výraz "RAG".' + r=client.query_backend(' /'+engine.upper()+' '+question+' ',engine) + assert r['accepted_for_delivery'] and r['query']==question + assert backend['requests'][-1][1]['query']==question + other='graphrag' if engine=='rag' else 'rag' + r=client.query_backend('/'+other+' '+question,engine) + assert r['reason']=='engine_mismatch' and not r['accepted_for_delivery'] + assert len(backend['requests'])==1 + + +@pytest.mark.parametrize('question',['/rag','/rag ','/graphrag']) +def test_command_without_a_question_never_reaches_backend(backend,question): + engine='graphrag' if question.startswith('/graphrag') else 'rag' + assert client.query_backend(question,engine)['reason']=='invalid_query' + assert not backend['requests'] + + +@pytest.mark.parametrize('change',[ + {'query':'A previous unrelated question','engine':'rag'}, + {'query':QUESTIONS[0],'engine':'graphrag'}, +]) +def test_stale_question_or_wrong_engine_cannot_deliver_another_answer(backend,change): + backend['raw']=json.dumps({**backend['checked'],'answer':'OTHER_ANSWER_SHOULD_NOT_ESCAPE',**change}).encode() + r=client.query_backend(QUESTIONS[0],'rag') + assert r['reason']=='question_mismatch' and not r['accepted_for_delivery'] + assert 'OTHER_ANSWER' not in json.dumps(r) + + +def test_missing_question_echo_is_not_accepted(backend): + backend['raw']=json.dumps({**backend['checked'],'engine':'rag'}).encode() + r=client.query_backend(QUESTIONS[0],'rag') + assert r['reason']=='invalid_backend_response' and not r['accepted_for_delivery'] + + +def test_consecutive_mcp_calls_do_not_reuse_previous_queries(backend): + from mcp import ClientSession,StdioServerParameters + from mcp.client.stdio import stdio_client + from packages.claude.mcp_server import QUESTION_POLICY + async def run(): + params=StdioServerParameters(command=sys.executable,args=[str(ROOT/'packages/claude/mcp_server.py')], + env={**os.environ,'PYTHONDONTWRITEBYTECODE':'1'}) + async with stdio_client(params) as (read,write): + async with ClientSession(read,write) as session: + initialized=await session.initialize() + assert QUESTION_POLICY in initialized.instructions + for engine,question in [('rag',QUESTIONS[0]),('graphrag',QUESTIONS[2]),('rag',QUESTIONS[1])]: + result=await session.call_tool('zpwiki_'+engine,{'query':question}) + assert not result.isError and result.structuredContent['query']==question + assert backend['requests'][-1][1]['query']==question + assert backend['requests'][-1][1]['engine']==engine + asyncio.run(run()) diff --git a/test/test_verified_rag.py b/test/test_verified_rag.py index 6a2b16c..621798f 100644 --- a/test/test_verified_rag.py +++ b/test/test_verified_rag.py @@ -109,6 +109,29 @@ def test_bad_answer_is_repaired_once_and_only_verified_fields_are_rendered(paylo assert 'unsupported_person_work_type_year_binding' in result['attempts'][0]['issues'] +def test_scalar_claims_require_real_source_urls_without_graph_record_ids(payload,answer): + empty=copy.deepcopy(answer);empty['source_urls']=[] + outputs=iter([empty,answer]);calls=[] + def complete(messages): + calls.append(copy.deepcopy(messages)) + assert 'source_urls pri kladnej odpovedi nesmie byť prázdne' in messages[0]['content'] + sent=json.loads(messages[1]['content'])['context'] + assert all('text' not in row for row in sent['sources']) + return completion(next(outputs)) + r=generate_verified_rag(QUESTION,payload,complete) + assert r['accepted_for_delivery'] and len(calls)==2 + assert 'missing_required_citations' in r['attempts'][0]['issues'] + assert 'skutočnú wiki URL' in calls[1][-1]['content'] + assert r['source_urls']==answer['source_urls'] + + +def test_rag_claim_schema_does_not_accept_a_source_id_as_a_graph_record(payload,answer): + wrong=copy.deepcopy(answer);wrong['claims'][0]['record_id']='S1' + r=generate_verified_rag(QUESTION,payload,lambda _:completion(wrong)) + assert not r['accepted_for_delivery'] + assert all(attempt['issues']==['invalid_answer_format'] for attempt in r['attempts']) + + def test_two_bad_attempts_never_return_unverified_model_text(payload, answer): answer['claims'][0]['year'] = 2021 calls = [] diff --git a/tools/answer_evaluation/presentation.py b/tools/answer_evaluation/presentation.py index 9501e14..fd4c34c 100644 --- a/tools/answer_evaluation/presentation.py +++ b/tools/answer_evaluation/presentation.py @@ -107,15 +107,29 @@ GOAL_VERBS = { 'natrénujte': 'Natrénovať', 'optimalizujte': 'Optimalizovať', 'identifikujte': 'Identifikovať', 'preštudujte': 'Preštudovať', 'pripravte': 'Pripraviť', + 'vyhodnotte': 'Vyhodnotiť', 'vyhodnnotte': 'Vyhodnotiť', } def goal_text(unit: str) -> tuple[str, bool]: + """Format only known goal verbs, keeping URLs and code byte-for-byte intact.""" + prefix='_ZP_LITERAL_' + while prefix in unit:prefix+='X' + literals={} + def protect(value): + key=prefix+str(len(literals))+'_';literals[key]=value + return key + text=re.sub(r'(`+)([\s\S]*?)\1',lambda m:protect(m[0]),unit) + text=re.sub(r'<[^>]+>',lambda m:protect(m[0]),text) + for url in sorted(text_urls(text),key=len,reverse=True): + text=text.replace(url,protect(url)) + result,is_goal=_goal_text(text) + for key,value in literals.items():result=result.replace(key,value) + return result,is_goal + + +def _goal_text(unit: str) -> tuple[str, bool]: """Only bounded grammatical substitutions; unknown prose stays verbatim.""" - if text_urls(unit) or '`' in unit: - # A word inside a URL/code literal must never be conjugated. Preserve - # technical excerpts exactly rather than changing their references. - return unit.strip(), False text = re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+', '', unit.strip()) for original, replacement in GOAL_NOUNS: if re.match(re.escape(original) + r'\b', text, re.I): diff --git a/tools/answer_evaluation/render_answer.py b/tools/answer_evaluation/render_answer.py index e2dc034..c4c34da 100644 --- a/tools/answer_evaluation/render_answer.py +++ b/tools/answer_evaluation/render_answer.py @@ -19,7 +19,9 @@ def render_graph_answer(answer: dict, context: dict) -> dict: for person in row['people'])}) if values: shared_fields.add(field) - prefix='Spoločné ' if context.get('complete') is True and not context.get('truncated') else 'Doložené spoločné ' + full=context.get('complete') is True and not context.get('truncated') + prefix=(('Spoloční ' if full else 'Doložení spoloční ') if field=='document_author' else + ('Spoločné ' if full else 'Doložené spoločné ')) lines.append(prefix+label+': '+', '.join('**'+v+'**' for v in values)+'.') for row in context.get('shared_properties',[]): if row['field']==field and row['value'] in values: diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index b945307..8ea7a8c 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -8,7 +8,6 @@ from typing import Literal from pydantic import BaseModel, ConfigDict, Field -from .answer_format import Claim from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized, text_urls from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections, quote_matches_passage from .answer_coverage import answer_coverage, coverage_notices @@ -19,7 +18,16 @@ from .evidence_selection import (resolve_selection, EvidenceSelectionError, missing_required_evidence, overview_units) -class RagClaim(Claim): +class RagClaim(BaseModel): + # RAG facts carry source_id, not the graph's record_id/work_label fields. + model_config = ConfigDict(extra='forbid', strict=True) + person: str + title: str | None = None + year: int | None = None + start_year: int | None = None + keyword: str | None = None + category: str | None = None + document_author: str | None = None work_type: Literal['bachelor_thesis', 'diploma_thesis', 'doctoral_thesis', 'bachelor_project', 'diploma_project', 'team_project', 'research_project'] | None = None @@ -503,6 +511,9 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: sent['content_passages'] = context['content_passages'] elif not context['requirements']['academic']: sent['context'] = payload.get('context', '') + elif context['requirements']['response_mode']=='claims': + sent['sources']=[{key:source[key] for key in ['source_id','source_url','title','document_path'] if key in source} + for source in context['sources']] if context['requirements']['response_mode'] == 'quotes': sent.pop('facts') if 'content_candidates' in context: @@ -528,6 +539,12 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: 'Aplikácia konkrétne pomenuje chýbajúce informácie. no_answer tu nesmie odmietnuť ' 'aj dostupnú časť. Plánovaný experiment alebo vyhodnotenie označené phase=planned ' 'odpovedá na otázku, čo mali pripraviť; nie je to dosiahnutý výsledok.') + elif context['requirements']['response_mode']=='claims': + selection_policy=('\nPre tento dotaz je response_mode=claims: vyber požadované údaje z facts. ' + 'Každý claim MUSÍ mať citáciu v source_urls: použi source_url príslušného faktu alebo zdroja. ' + 'source_urls pri kladnej odpovedi nesmie byť prázdne. evidence_quotes=[] a evidence_ids=[]. ' + 'source_id ako S1 je len označenie podkladu; nevkladaj ho do claims. ' + 'Polia record_id a work_label patria iba grafovému nástroju, tu sa nepoužívajú.') messages = [{'role': 'system', 'content': PROMPT + selection_policy + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)}, {'role': 'user', 'content': json.dumps({'question': question, 'context': sent}, ensure_ascii=False)}] attempts = [] @@ -577,7 +594,10 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: if context['requirements'].get('content') else '') + (' Použi evidence_ids z content_candidates a evidence_quotes=[]. Vyber všetky required=true; ' 'doslovné texty aj ich citácie dosadí aplikácia. Pri response_mode=quotes musí byť claims=[].' - if 'content_candidates' in context else '')}] + if 'content_candidates' in context else '') + + (' Ku každému claimu uveď vo source_urls jeho skutočnú wiki URL zo sources/facts. ' + 'Prázdne source_urls ani source_id=S1 nie sú citácia. Do claims nepridávaj record_id ani work_label.' + if context['requirements']['response_mode']=='claims' else '')}] return {'status': 'error', 'reason': 'answer_rejected', 'accepted_for_delivery': False, 'answer': 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.', 'attempts': attempts, 'index_version': context['index_version']}