jp170na/dp-zp-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve trusted metadata scope through final answer validation

Browse Source
This commit is contained in:
Ján Pták 2026-11-10 08:58:38 +01:00
parent 2da46418c7
commit 16d1eb72ac
7 changed files with 591 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

477
experiments/delivery_contract_v1/post_test_dev.json Normal file
View File

@ -0,0 +1,477 @@
{
"scope": "Frozen TEST cases reused as known DEV after a source-binding repair. NOT new independent TEST accuracy.",
"frozen_test_remains_unchanged": true,
"frozen_test_score_does_not_describe_changed_runtime": true,
"repair": "Preserve trusted work identity through RAG preparation and graph presentation field selection; require source-backed work witnesses for person-metadata conjunctions.",
"checks": [
"HTTP status ok and accepted",
"source URL equals the gold profile",
"nonempty claims",
"every claim has the literal work heading year and the expected work type"
],
"passed": 20,
"total": 20,
"runtime_files_sha256": "f2e53317b1e0371b08bb2c12084d674cf532cedbb5b53de87f322de2369933d2",
"results": [
{
"id": "C001",
"query": "U osoby Kamil Tomčufčík uveď typ a evidovaný rok záznamu „Diplomová práca 2025“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Kamil Tomčufčík",
"work_type": "diploma_thesis",
"year": 2025
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2020/kamil_tomcufcik"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "40b1ea380c6cfeb870b0a8f309cf6895df6b2e1cec80d2ae22230bad0d3a90a7"
}
},
{
"id": "C001",
"query": "U osoby Kamil Tomčufčík uveď typ a evidovaný rok záznamu „Diplomová práca 2025“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Kamil Tomčufčík",
"year": 2025,
"work_type": "diploma_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2020/kamil_tomcufcik"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "8d70bbd70f4c117117fc2b013920d7e37c76c14f38359f7b2cb010c14174c33d"
}
},
{
"id": "C009",
"query": "U osoby Vladimír Ferko uveď typ a evidovaný rok záznamu „Bakalárska práca 2024“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Vladimír Ferko",
"work_type": "bachelor_thesis",
"year": 2024
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "af44359ab30c1eb16eaa4e4ac972cde0cb7fe36f0b8fbc786f910f75dcef9ebe"
}
},
{
"id": "C009",
"query": "U osoby Vladimír Ferko uveď typ a evidovaný rok záznamu „Bakalárska práca 2024“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Vladimír Ferko",
"year": 2024,
"work_type": "bachelor_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "a362e5e6a86dd6e8653e1c379a0bdbf6f06bb1a1922eadee332ce41ffd1ef6bb"
}
},
{
"id": "C017",
"query": "U osoby Dávid Omasta uveď typ a evidovaný rok záznamu „Diplomová práca 2024“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Dávid Omasta",
"work_type": "diploma_thesis",
"year": 2024
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2018/david_omasta"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "df566cba179452b6ec761abf6e11ce5d46c1b622a35eb4f704f510cc6579a05a"
}
},
{
"id": "C017",
"query": "U osoby Dávid Omasta uveď typ a evidovaný rok záznamu „Diplomová práca 2024“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Dávid Omasta",
"year": 2024,
"work_type": "diploma_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2018/david_omasta"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "3e70a1633a926ed3e1c6c1b86601a587b6cd9f48b80c7b002aea56ee154548bd"
}
},
{
"id": "C025",
"query": "U osoby Matej Ščišľak uveď typ a evidovaný rok záznamu „Bakalárska práca 2024“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Matej Ščišľak",
"work_type": "bachelor_thesis",
"year": 2024
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "9817ffbd759ab19997a4ebc5c6ee86e8482bf9d687a91851002cb45e29f7e703"
}
},
{
"id": "C025",
"query": "U osoby Matej Ščišľak uveď typ a evidovaný rok záznamu „Bakalárska práca 2024“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Matej Ščišľak",
"year": 2024,
"work_type": "bachelor_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "21abb9ffd01fab89d9b8cd5d198348f35d67b6f50869cdb529d2f0f459057273"
}
},
{
"id": "C033",
"query": "U osoby Yevhenii Leonov uveď typ a evidovaný rok záznamu „Bakalárska práca 2025“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Yevhenii Leonov",
"work_type": "bachelor_thesis",
"year": 2025
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "20998da90b31479cbb52deed9c295f506ff7f0ad2ced6e67c99fbb040f108431"
}
},
{
"id": "C033",
"query": "U osoby Yevhenii Leonov uveď typ a evidovaný rok záznamu „Bakalárska práca 2025“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Yevhenii Leonov",
"year": 2025,
"work_type": "bachelor_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "c03cd06974d8a3e12ed0585aac192112b2a435ae0d19d324e504c896e3fb37a5"
}
},
{
"id": "C041",
"query": "U osoby Serhii Yemets uveď typ a evidovaný rok záznamu „Bakalárska práca 2025“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Serhii Yemets",
"work_type": "bachelor_thesis",
"year": 2025
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "7e4abcd74aceee1b9970bdea9af2078f91d863ed9e3752df7ca2c97fd29bdab7"
}
},
{
"id": "C041",
"query": "U osoby Serhii Yemets uveď typ a evidovaný rok záznamu „Bakalárska práca 2025“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Serhii Yemets",
"year": 2025,
"work_type": "bachelor_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "22ddb6a34f509f28734bc307ef84aea5f22eb47ab39d4b184492b3b1e5b6f8ce"
}
},
{
"id": "C049",
"query": "U osoby Artur Hyrenko uveď typ a evidovaný rok záznamu „Diplomová práca 2026“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Artur Hyrenko",
"work_type": "diploma_thesis",
"year": 2026
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/artur_hyrenko"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "f308ccc97c0bb5a93d109477f3e977eb67b887480ea5f952801ef4fe4f23b59b"
}
},
{
"id": "C049",
"query": "U osoby Artur Hyrenko uveď typ a evidovaný rok záznamu „Diplomová práca 2026“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Artur Hyrenko",
"year": 2026,
"work_type": "diploma_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/artur_hyrenko"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "db6a19db9eca2a43c0483c239d8184b0fd1163f48bd5cd201a0b0a778df60af5"
}
},
{
"id": "C057",
"query": "U osoby Nikita Bodnar uveď typ a evidovaný rok záznamu „Bakalárska práca 2024“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Nikita Bodnar",
"work_type": "bachelor_thesis",
"year": 2024
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/nikita_bodnar"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "563cd03a1b203ffe0bbbac07d9e8d202a1d02f0fdce017a2506165c2b4638300"
}
},
{
"id": "C057",
"query": "U osoby Nikita Bodnar uveď typ a evidovaný rok záznamu „Bakalárska práca 2024“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Nikita Bodnar",
"year": 2024,
"work_type": "bachelor_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/nikita_bodnar"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "a3da35c1976f1117d52a48f2ffdd280b7ebbccd2020d3eda1be16f059084f261"
}
},
{
"id": "C065",
"query": "U osoby Dávid Kostilník uveď typ a evidovaný rok záznamu „Diplomová práca 2025“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Dávid Kostilník",
"work_type": "diploma_thesis",
"year": 2025
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2020/david_kostilnik"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "644f469d2be50b94d6458cb53cbb2e781f2ce0ef34d94e05966139db9754dd94"
}
},
{
"id": "C065",
"query": "U osoby Dávid Kostilník uveď typ a evidovaný rok záznamu „Diplomová práca 2025“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Dávid Kostilník",
"year": 2025,
"work_type": "diploma_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2020/david_kostilnik"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "bc9ecfd809e4acaad3a468d538637f651b3e94b6737d8858664bb9db2ec9ada2"
}
},
{
"id": "C073",
"query": "U osoby Valerii Kutsenko uveď typ a evidovaný rok záznamu „Bakalárska práca 2025“.",
"engine": "rag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Valerii Kutsenko",
"work_type": "bachelor_thesis",
"year": 2025
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "48b8745ac97968457129947083dd7a7230b2ff4e7c3c1884b9d3fd726f244085"
}
},
{
"id": "C073",
"query": "U osoby Valerii Kutsenko uveď typ a evidovaný rok záznamu „Bakalárska práca 2025“.",
"engine": "graphrag",
"passed": true,
"status": "ok",
"reason": null,
"claims": [
{
"person": "Valerii Kutsenko",
"year": 2025,
"work_type": "bachelor_thesis"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko"
],
"delivery_check": {
"layer": "delivery",
"code": "complete",
"request_fingerprint": "e97f121dc3d87415b2eeaeff02a089e65a612727b84fb96d0b20775dae865566"
}
}
]
}

71
test/test_delivery_bindings.py Normal file
View File

@ -0,0 +1,71 @@
"""Source identity survives presentation field selection in both retrievers."""
import copy
import pytest
from tools.answers import delivery_contract as contract
from tools.answers.graph_context import delivery_facts
from tools.answers.question_plan import QuestionPlan, PersonRequest, WorkRequest
from tools.answers.verified_rag import generate_verified_rag
URL='https://example.test/eva'
def plan():
return QuestionPlan(question='Aký typ a rok má záznam „Diplomová práca 2027“ osoby Eva Nová?',
people=[PersonRequest(name='Eva Nová',source_document='pages/students/2020/eva/README.md')],
works=[WorkRequest(person='Eva Nová',work_type='diploma_thesis',years=[2027],labels=['diplomova praca 2027'])],
fields=['work_type','year'],scope='single_work')
def output(claims):
return {'status':'ok','accepted_for_delivery':True,'answer':'Eva Nová: diplomová práca, rok 2027.\nZdroj: '+URL,
'claims':claims,'source_urls':[URL],'evidence':[]}
def test_graph_source_scope_is_preserved_without_adding_unrequested_fields_to_prompt():
fact={'person':'Eva Nová','work_type':'diploma_thesis','year':2027,'source_url':URL}
context={'facts':[fact]};before=copy.deepcopy(context)
raw={'facts':[{'person':'Eva Nová','thesis_type':'diploma_thesis','year':2027,
'heading':'Diplomová práca 2027','thesis_id':'work-1','source_url':URL,'fact_type':'person_thesis'}]}
p=plan();result=output([{'person':'Eva Nová','work_type':'diploma_thesis','year':2027}])
assert not contract.enforce(p.question,'graphrag',p,result,facts=context['facts'])['accepted_for_delivery']
bindings=delivery_facts(context,raw)
assert contract.enforce(p.question,'graphrag',p,result,facts=bindings)['accepted_for_delivery']
assert context==before and bindings[-1]['record_id']=='work-1'
def test_person_tag_requires_separate_source_backed_work_for_a_conjunction():
p=plan();p.question='Ktorí študenti majú tag NLP a diplomovú prácu z roku 2027?'
p.fields=['keyword'];p.presentation='list'
fact={'person':'Eva Nová','keyword':'NLP','source_url':URL}
result=output([{'person':'Eva Nová','keyword':'NLP'}])
result['answer']='Eva Nová má tag NLP.\nZdroj: '+URL
assert not contract.enforce(p.question,'graphrag',p,result,facts=[fact])['accepted_for_delivery']
work={'person':'Eva Nová','work_type':'diploma_thesis','year':2027,'work_label':'Diplomová práca 2027','source_url':URL}
assert contract.enforce(p.question,'graphrag',p,result,facts=[fact,work])['accepted_for_delivery']
for change in [{'year':2024},{'source_url':'https://example.test/other'},{'person':'Dana Iná'}]:
assert not contract.enforce(p.question,'graphrag',p,result,facts=[fact,{**work,**change}])['accepted_for_delivery']
def test_rag_binds_actual_prepared_facts_instead_of_incomplete_model_claims():
p=plan()
payload={'status':'ok','reason':None,'index_version':{'version':'fixture'},'complete':True,'truncated':False,
'truncation_reasons':[],'named_scope_complete':True,'question_plan':p.model_dump(),
'sources':[{'source_id':'S1','title':'Eva Nová','document_path':'pages/students/2020/eva/README.md',
'source_url':URL,'categories':['dp2027'],'text':'# Diplomová práca 2027\nNázov: Pokusný systém.\n'}]}
from tools.rag.rag_utils import build_context_text
payload['context']=build_context_text(payload['sources'])
def forbidden(_):pytest.fail('Exact metadata needs no language model')
plain=generate_verified_rag(p.question,payload,forbidden)
assert '_delivery_facts' not in plain
result=generate_verified_rag(p.question,payload,forbidden,include_bindings=True)
facts=result.pop('_delivery_facts')
assert any(f.get('work_label')=='Diplomová práca 2027' for f in facts)
assert contract.enforce(p.question,'rag',p,result,facts=facts)['accepted_for_delivery']
def test_rejected_verification_is_not_diagnosed_as_provider_unavailability():
rejected=contract.diagnose({'status':'error','reason':'answer_rejected'},'q','rag')
unavailable=contract.diagnose({'status':'error','reason':'verification_unavailable'},'q','rag')
assert rejected['delivery_check']['code']=='request_not_fulfilled'
assert unavailable['delivery_check']['code']=='unavailable'

2
test/test_person_names.py
View File

@ -357,7 +357,7 @@ def test_both_engines_use_same_resolution_and_notice(names_db, people, monkeypat
monkeypatch.setattr(delivery, "prepare_named_content", lambda db, q, p: p)
monkeypatch.setattr(delivery, "compact_rag_payload", lambda p: p)
def rag(q, p, model):
def rag(q, p, model, **kwargs):
assert q == resolution["resolved_query"]
return dict(checked)

8
tools/answers/answer_delivery.py
View File

@ -9,7 +9,7 @@ from tools.rag.common import DB_FILE
from tools.rag.rag_utils import build_rag_context
from tools.rag.llm_context import compact_rag_payload
from tools.graphrag.graph_search import search_graph
from tools.answers.graph_context import graph_context
from tools.answers.graph_context import graph_context, delivery_facts
from tools.answers.checked_generation import generate_checked_answer
from tools.answers.verified_rag import generate_verified_rag
from tools.answers.rag_evidence import expand_comparison_sources
@ -204,7 +204,8 @@ def _deliver_answer(
retrieved = compact_rag_payload(retrieved)
except Exception:
return retrieval_failure(engine)
result = generate_verified_rag(question, retrieved, model)
result = generate_verified_rag(question, retrieved, model, include_bindings=True)
facts = result.pop('_delivery_facts', [])
context = retrieved
elif engine == "graphrag":
if plan.spec.intent != "metadata":
@ -224,6 +225,7 @@ def _deliver_answer(
return retrieval_failure(engine)
retrieved["name_resolution"] = plan.resolution
context = graph_context(retrieved)
facts = delivery_facts(context, retrieved)
except Exception:
return retrieval_failure(engine)
if context.get("clarification"):
@ -269,6 +271,6 @@ def _deliver_answer(
**result,
"engine": engine,
**({"retrieval_context": context, "question_plan": plan.model_dump()} if include_diagnostics else {}),
},context.get('facts',[]))
},facts)
finally:
_capacity.release()

12
tools/answers/delivery_contract.py
View File

@ -75,7 +75,8 @@ def diagnose(result,query,engine):
'planning' if result.get('status')=='ambiguous' else
'retrieval' if result.get('status')=='no_answer' or reason in {'retrieval_unavailable','content_index_unavailable'} else
'verification' if reason in {'answer_rejected','verification_unavailable'} else 'service')
code={'ambiguous':'clarification_required','no_answer':'evidence_not_found'}.get(result.get('status'),'unavailable')
code=('request_not_fulfilled' if reason=='answer_rejected' else
{'ambiguous':'clarification_required','no_answer':'evidence_not_found'}.get(result.get('status'),'unavailable'))
return {**result,'delivery_check':{'layer':layer,'code':code,'request_fingerprint':fingerprint(query,engine)}}
@ -138,7 +139,14 @@ def mechanical_check(plan,result,index=None,*,facts=()):
k not in claim or claim[k] is None or r.get(k) is not None and _same(claim[k],r[k])
for k in ['record_id','work_label','work_type','title','year','start_year','keyword','category','document_author'])]
if candidates:
if not any(plan.matches_work(person,r.get('work_label'),r.get('work_type'),r.get('year')) for r in candidates):
# Person-level metadata (e.g. a tag) can be conjoined with a work
# condition, but cannot itself prove that work exists. Require a
# separate scoped fact from the same person and source document.
scoped = candidates
if not any(claim.get(k) is not None for k in ['record_id','work_label','work_type','title','year']):
urls={r.get('source_url') for r in candidates if r.get('source_url')}
scoped=[r for r in facts if r.get('person')==person and (not urls or r.get('source_url') in urls)]
if not any(plan.matches_work(person,r.get('work_label'),r.get('work_type'),r.get('year')) for r in scoped):
raise ContractError('scope','wrong_work_or_meeting')
elif facts:
raise ContractError('evidence','altered_evidence')

18
tools/answers/graph_context.py
View File

@ -3,6 +3,24 @@
from tools.answers.context_policy import joined_candidates
def delivery_facts(context, result):
"""Preserve source work identity independently of the model's field subset.
A question can ask for type/year without asking to display a record ID or
heading. The final scope check still needs those trusted source fields.
These extra bindings are not added to the language model's payload.
"""
facts = list(context.get('facts', []))
for row in [*result.get('facts', []), *result.get('supporting_facts', [])]:
if not row.get('person') or row.get('fact_type') not in {'person_thesis', 'person_work'}:
continue
fact = {k: row[k] for k in ['person', 'title', 'year', 'start_year', 'source_url', 'document_path'] if k in row}
fact.update(work_type=row.get('thesis_type') or row.get('work_type'),
work_label=row.get('heading'), record_id=row.get('thesis_id') or row.get('work_id'))
facts.append(fact)
return facts
def graph_context(result: dict) -> dict:
"""Remove storage/profile metadata that cannot answer the actual question.

10
tools/answers/verified_rag.py
View File

@ -792,8 +792,16 @@ def render_rag_answer(answer: dict, context: dict) -> str:
return clean_answer(lines)
def generate_verified_rag(question: str, payload: dict, complete) -> dict:
def generate_verified_rag(question: str, payload: dict, complete, *, include_bindings=False) -> dict:
context = rag_evidence_context(question, payload)
result = _generate_from_context(question, payload, complete, context)
if include_bindings:
# Server-only provenance used by the final contract, never model output.
result = {**result, '_delivery_facts': context['facts']}
return result
def _generate_from_context(question, payload, complete, context):
if payload.get("status") == "ambiguous" or context.get("clarification"):
return {
"status": "ambiguous",