281 lines
17 KiB
Python
281 lines
17 KiB
Python
"""Meeting prose and hypothetical recommendations retain source-bound facts."""
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools.answers import synthesis as s, recommendations as rec
|
|
from tools.answers.notices import clean_answer_notices,LIMIT_NOTICE
|
|
from tools.answers.question_plan import build_question_plan,QuestionPlan,PersonRequest
|
|
from tools.knowledge.sources import profile_units
|
|
from tools.knowledge.academic import answer_academic
|
|
|
|
CATALOG=[{'kind':'Person','name':name,'source_document':f'pages/students/2022/{slug}/README.md'}
|
|
for name,slug in [('Ján Pták','jan_ptak'),('Jakub Schwarc','jakub_schwarc')]]
|
|
|
|
|
|
def corpus(texts=None):
|
|
texts=texts or ['Funguje agent RAG.','Funguje dotrénovanie modelu.']
|
|
rows=[]
|
|
for p,text in zip(CATALOG,texts):
|
|
rows+=profile_units({'title':p['name'],'path':p['source_document'],
|
|
'content':'# Diplomová práca 2027\nStretnutie 5.11.2026\nStav:\n- '+text},include_records=True)
|
|
return {'index_version':{'version':'v'},'version':'content','academic_schema':1,'units':rows}
|
|
|
|
|
|
def meeting_result(data=None):
|
|
data=data or corpus();q='Kedy mal Ján Pták posledné stretnutie a o čom bolo?'
|
|
bound=build_question_plan(q,CATALOG)
|
|
return answer_academic(q,data,engine='rag',mode='meeting',plan=bound),bound.spec
|
|
|
|
|
|
def reviewed_draft(messages):
|
|
payload=json.loads(messages[-1]['content'])
|
|
if 'claims' in payload:
|
|
return {'choices':[{'message':{'content':json.dumps({'coverage':'supported','verdicts':[
|
|
{'claim_id':c['claim_id'],'decision':'supported','reason':'entailed'} for c in payload['claims']]})}}]}
|
|
return {'choices':[{'message':{'content':json.dumps({'claims':[
|
|
{'text':r['text'].lstrip('- '),'kind':'description','evidence_ids':[r['id']]} for r in payload['evidence']]})}}]}
|
|
|
|
|
|
def test_meeting_synthesis_ignores_structural_markers_but_retains_them_in_result():
|
|
result,plan=meeting_result();before=copy.deepcopy(result)
|
|
out=s.synthesize(result,plan,complete=reviewed_draft,diagnostics=True)
|
|
assert out['synthesis']['status']=='accepted'
|
|
assert 'Podľa zápisu zo stretnutia 2026-11-05' in out['answer']
|
|
assert 'Výsledky' in out['answer'] and 'agent RAG' in out['answer']
|
|
assert out['evidence']==result['evidence'] and any(r['kind']=='meeting' for r in out['evidence'])
|
|
assert result==before
|
|
|
|
|
|
def test_meeting_prose_retains_planned_vs_executed_without_copying_imperatives():
|
|
data=corpus();path=CATALOG[0]['source_document']
|
|
data['units']+=profile_units({'title':'Ján Pták','path':path,
|
|
'content':'# Diplomová práca 2027\nStretnutie 5.11.2026\nÚlohy:\n- Vyhodnoťte systém.'},include_records=True)[-1:]
|
|
result,plan=meeting_result(data)
|
|
def model(messages):
|
|
payload=json.loads(messages[-1]['content'])
|
|
if 'claims' in payload:return reviewed_draft(messages)
|
|
value={'claims':[{'kind':'description','evidence_ids':[r['id']],
|
|
'text':'Má vyhodnotiť systém.' if r['phase']=='planned' else 'Funguje agent RAG.'} for r in payload['evidence']]}
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
out=s.synthesize(result,plan,complete=model,diagnostics=True)
|
|
assert out['synthesis']['status']=='accepted' and 'Ďalší postup: Má vyhodnotiť systém.' in out['answer']
|
|
assert 'Vyhodnoťte' not in out['answer'] and 'Vyhodnotil' not in out['answer']
|
|
|
|
|
|
def test_descriptions_may_merge_same_record_but_never_different_meetings():
|
|
result,plan=meeting_result();rows=s.prepare_evidence(result);a=next(iter(rows.values()))
|
|
a['phase']='planned';a['text']='Implementujte agenta RAG.'
|
|
rows['second']={**a,'id':'second','text':'Pridajte vyhľadávanie.'}
|
|
draft=s.Draft(claims=[s.ProseClaim(text='Má implementovať agenta RAG a pridať vyhľadávanie.',kind='description',evidence_ids=[a['id'],'second'])])
|
|
s.validate_draft(draft,rows,plan)
|
|
rows['second']['meeting_id']='another-record';rows['second']['record_date']='2026-10-01'
|
|
with pytest.raises(s.SynthesisRejected,match='non_atomic'):s.validate_draft(draft,rows,plan)
|
|
|
|
|
|
def test_separate_record_dates_are_not_pooled_into_one_ambiguous_phase_header():
|
|
result,plan=meeting_result();rows=s.prepare_evidence(result);a=next(iter(rows.values()))
|
|
rows['old']={**a,'id':'old','record_date':'2026-10-01','meeting_id':'old'}
|
|
draft=s.Draft(claims=[s.ProseClaim(text='Funguje agent RAG.',kind='description',evidence_ids=[key]) for key in rows])
|
|
text,_=s.render(draft,rows,result,plan)
|
|
assert 'zo stretnutia 2026-11-05' in text and 'zo stretnutia 2026-10-01' in text
|
|
assert '2026-11-05, 2026-10-01' not in text
|
|
|
|
|
|
def proposal_input(data=None,engine='rag'):
|
|
data=data or corpus()
|
|
bound=build_question_plan('Čo by sa dalo spojiť z práce Jána Ptáka a Jakuba Schwarca?',CATALOG)
|
|
return rec.collect_evidence(bound,data,data['units'],engine=engine,limit=5),bound.spec,data
|
|
|
|
|
|
def proposal_model(mutate=None,review_ok=True):
|
|
def call(messages):
|
|
p=json.loads(messages[-1]['content'])
|
|
if 'draft' in p:
|
|
value={k:review_ok for k in rec.ProposalReview.model_fields}
|
|
elif 'claims' in p:
|
|
return reviewed_draft(messages)
|
|
else:
|
|
ids=[r['id'] for r in p['evidence']]
|
|
by_id={r['id']:r for r in p['evidence']}
|
|
value={'claims':[{'text':' '.join(by_id[k]['text'].lstrip('- ') for k in g['evidence_ids']),'group_id':g['group_id']} for g in p['scope_groups']],
|
|
'proposal':{'text':'Mohlo by sa vyskúšať dotrénovaný model ako súčasť agenta RAG.','evidence_ids':ids},
|
|
'requirements':[{'text':'Overiť možnosť spustenia modelu a kompatibilitu s rozhraním agenta.','evidence_ids':ids}],
|
|
'evaluation':[{'text':'Porovnať odpovede agenta pri rovnakých otázkach a zaznamenať správnosť aj čas odpovede.','evidence_ids':ids}]}
|
|
if mutate:mutate(value)
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
return call
|
|
|
|
|
|
@pytest.mark.parametrize('engine',['rag','graphrag'])
|
|
def test_grounded_proposal_has_four_parts_citations_and_no_graph_or_index_mutation(engine):
|
|
result,plan,data=proposal_input(engine=engine);before=copy.deepcopy((result,data))
|
|
out=rec.generate(result,plan,complete=proposal_model(),known_people=['Ján Pták','Jakub Schwarc'],diagnostics=True)
|
|
assert out['recommendation_status']=='accepted' and out['answer_kind']=='recommendation'
|
|
assert all(s in out['answer'] for s in ['Doložené východiská','Návrh na zváženie','Podmienky realizácie','Ako návrh overiť','Ide o návrh.'])
|
|
assert len(out['source_urls'])==2 and 'spoluprácu ani zlepšenie' in out['answer']
|
|
assert before==(result,data) and 'recommendation' not in data
|
|
|
|
|
|
@pytest.mark.parametrize('mutation',[
|
|
lambda v:v['proposal'].update(text='Prepojenie už funguje a garantuje lepšie výsledky.'),
|
|
lambda v:v['proposal'].update(text='Návrh zaručene zlepší presnosť o 99 percent.'),
|
|
lambda v:v['proposal'].update(text='Mohlo by sa spustiť overenie v roku 2099.'),
|
|
lambda v:v['proposal'].update(text='Mohlo by sa použiť <a href="/invented">riešenie</a>.'),
|
|
lambda v:v['proposal'].update(evidence_ids=['invented']),
|
|
lambda v:v['proposal'].update(evidence_ids=v['proposal']['evidence_ids'][:1]),
|
|
])
|
|
def test_invalid_or_falsely_achieved_proposals_are_not_published(mutation):
|
|
result,plan,_=proposal_input()
|
|
out=rec.generate(result,plan,complete=proposal_model(mutation),diagnostics=True)
|
|
assert out['recommendation_status']=='unavailable' and out['answer_kind']=='grounded_facts'
|
|
assert 'Návrh na zváženie' not in out['answer'] and '99 percent' not in out['answer']
|
|
assert '2099' not in out['answer'] and '/invented' not in out['answer']
|
|
|
|
|
|
def test_independent_proposal_review_rejects_a_plausible_but_unfounded_bridge():
|
|
result,plan,_=proposal_input()
|
|
out=rec.generate(result,plan,complete=proposal_model(review_ok=False),diagnostics=True)
|
|
assert out['recommendation_status']=='unavailable'
|
|
assert out['recommendation_review']['reason']=='proposal_review_rejected'
|
|
|
|
|
|
def test_missing_participant_gets_specific_partial_answer_without_invented_proposal():
|
|
data=corpus();data['units']=[r for r in data['units'] if r['person']=='Ján Pták']
|
|
result,plan,_=proposal_input(data)
|
|
out=rec.generate(result,plan,complete=lambda _:pytest.fail('No generation with a missing person'))
|
|
assert out['recommendation_status']=='insufficient_grounding' and out['answer_coverage']['status']=='partial'
|
|
assert 'Jakub Schwarc: chýbajú' in out['answer'] and 'Funguje agent RAG' in out['answer']
|
|
|
|
|
|
def test_recommendation_never_upgrades_a_planned_component_to_an_existing_one():
|
|
data=corpus();data['units']=[{**r,'phase':'planned','text':'Implementujte agenta RAG.'} for r in data['units'] if r['kind']!='meeting']
|
|
result,plan,_=proposal_input(data)
|
|
def upgrade(value):
|
|
value['claims'][0]['text']='Implementoval agenta RAG.'
|
|
out=rec.generate(result,plan,complete=proposal_model(upgrade),diagnostics=True)
|
|
assert out['recommendation_status']=='unavailable'
|
|
assert out['recommendation_review']['reason']=='phase_upgrade'
|
|
assert 'Implementoval' not in out['answer']
|
|
|
|
|
|
def test_proposal_evidence_is_bound_to_requested_people_work_year_and_meeting():
|
|
data=corpus()
|
|
for year,stamp in [(2026,'5.11.2026'),(2027,'1.10.2026')]:
|
|
data['units']+=profile_units({'title':'Ján Pták','path':CATALOG[0]['source_document'],
|
|
'content':f'# Diplomová práca {year}\nStretnutie {stamp}\nStav:\n- Funguje iný systém.'},include_records=True)
|
|
bound=build_question_plan('Čo by sa dalo zlepšiť pri diplomovej práci Jána Ptáka z roku 2027 na stretnutí 5.11.2026?',CATALOG)
|
|
assert bound.spec.intent=='proposal'
|
|
out=rec.collect_evidence(bound,data,data['units'],engine='rag',limit=5)
|
|
assert out['evidence']
|
|
assert all(r['person']=='Ján Pták' and r['work_year']==2027 and r['record_date']=='2026-11-05' for r in out['evidence'])
|
|
|
|
|
|
def test_comparison_labels_commonality_and_difference_but_keeps_each_work_bound():
|
|
data=corpus();rows={r['id']:r for r in data['units'] if r['kind'] not in {'meeting','work_record'}}
|
|
ids=list(rows)
|
|
plan=QuestionPlan(question='Porovnaj práce Jána Ptáka a Jakuba Schwarca.',intent='facets',presentation='comparison',
|
|
people=[PersonRequest(name=p['name'],role='student') for p in CATALOG])
|
|
draft=s.Draft(claims=[s.ProseClaim(text=r['text'].lstrip('- '),kind='description',evidence_ids=[r['id']]) for r in rows.values()]+[
|
|
s.ProseClaim(text='V oboch prípadoch je doložené fungujúce riešenie.',kind='comparison',comparison_kind='common',evidence_ids=ids),
|
|
s.ProseClaim(text='Jedna práca sa týka agenta RAG, druhá dotrénovania modelu.',kind='comparison',comparison_kind='difference',evidence_ids=ids)])
|
|
s.validate_draft(draft,rows,plan)
|
|
text,urls=s.render(draft,rows,{'presentation_notes':[]},plan)
|
|
assert 'Spoločné črty:' in text and 'Rozdiely:' in text and len(urls)==2
|
|
|
|
|
|
def test_proposal_for_latest_meeting_never_uses_old_or_undated_activities():
|
|
data=corpus()
|
|
for stamp in ['1.10.2026','7.11.']:
|
|
data['units']+=profile_units({'title':'Ján Pták','path':CATALOG[0]['source_document'],
|
|
'content':f'# Diplomová práca 2027\nStretnutie {stamp}\nStav:\n- Funguje iný systém.'},include_records=True)
|
|
bound=build_question_plan('Navrhni ďalší postup Jána Ptáka podľa posledného stretnutia.',CATALOG)
|
|
assert not bound.clarification and bound.spec.intent=='proposal' and bound.spec.meeting.selection=='latest'
|
|
out=rec.collect_evidence(bound,data,data['units'],engine='rag',limit=5)
|
|
assert out['evidence'] and all(r['record_date']=='2026-11-05' for r in out['evidence'])
|
|
assert any('bez úplného dátumu' in n for n in out['presentation_notes'])
|
|
|
|
|
|
def test_grouped_generator_does_not_waive_semantic_coverage_of_original_units():
|
|
result,plan=meeting_result();first=next(r for r in result['evidence'] if r['kind'] not in {'meeting','work_record'})
|
|
first['phase']='planned'
|
|
first['text']='Vyhodnoťte agenta RAG a pripravte experimenty.'
|
|
calls=[]
|
|
def model(messages):
|
|
payload=json.loads(messages[-1]['content']);calls.append(payload)
|
|
if 'claims' in payload:
|
|
assert len(payload['evidence'])==1
|
|
value={'coverage':'unsupported','verdicts':[{'claim_id':0,'decision':'supported','reason':'entailed'}]}
|
|
else:
|
|
assert len(payload['evidence'])==1
|
|
value={'claims':[{'text':'Má vyhodnotiť agenta RAG.','kind':'description','evidence_ids':['E1']}]}
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
out=s.synthesize(result,plan,complete=model,diagnostics=True)
|
|
assert out['answer']==result['answer'] and out['synthesis']['reason']=='support_rejected'
|
|
|
|
|
|
def test_duplicate_text_in_two_works_is_kept_under_both_but_not_repeated_within_one():
|
|
result,plan=meeting_result();row=next(iter(s.prepare_evidence(result).values()))
|
|
other={**row,'id':'other','work_id':'different','work_label':'Diplomový projekt 2026'}
|
|
rows={row['id']:row,'other':other}
|
|
draft=s.Draft(claims=[s.ProseClaim(text='Funguje agent RAG.',kind='description',evidence_ids=[key]) for key in [row['id'],row['id'],'other']])
|
|
text,_=s.render(draft,rows,{'presentation_notes':[]},plan)
|
|
assert text.count('Funguje agent RAG.')==2
|
|
assert 'Diplomová práca 2027' in text and 'Diplomový projekt 2026' in text
|
|
|
|
|
|
def test_semantic_repair_is_freshly_reviewed_and_never_publishes_first_bad_claim():
|
|
result,plan=meeting_result();calls=[]
|
|
def model(messages):
|
|
payload=json.loads(messages[-1]['content']);calls.append(payload)
|
|
if 'claims' in payload:
|
|
bad=any('celú prácu' in c['text'] for c in payload['claims'])
|
|
value={'coverage':'unsupported' if bad else 'supported','verdicts':[
|
|
{'claim_id':c['claim_id'],'decision':'unsupported' if bad else 'supported',
|
|
'reason':'unsupported_fact' if bad else 'entailed'} for c in payload['claims']]}
|
|
else:
|
|
value={'claims':[{'kind':'description','evidence_ids':['E1'],
|
|
'text':'Funguje agent RAG.' if 'support_repair' in payload else 'Úspešne dokončil celú prácu.'}]}
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
out=s.synthesize(result,plan,complete=model,diagnostics=True)
|
|
assert out['synthesis']['status']=='accepted' and 'celú prácu' not in out['answer']
|
|
# draft, two concurrent reviews, one fresh repair draft, two concurrent reviews
|
|
assert len(calls)==6 and 'support_repair' in calls[3]
|
|
assert all('support_repair' not in c for c in calls[:3])
|
|
|
|
|
|
def test_unfounded_optional_comparison_cannot_discard_valid_per_work_answers():
|
|
data=corpus();plan=build_question_plan('Porovnaj výsledky Jána Ptáka a Jakuba Schwarca.',CATALOG).spec
|
|
result={'status':'ok','accepted_for_delivery':True,'index_version':data['index_version'],
|
|
'knowledge_version':data['version'],'answer':'Safe original','evidence':data['units'],
|
|
'source_urls':list(dict.fromkeys(r['source_url'] for r in data['units'])),'presentation_notes':[]}
|
|
def model(messages):
|
|
payload=json.loads(messages[-1]['content'])
|
|
if 'claims' in payload:return reviewed_draft(messages)
|
|
claims=[{'text':r['text'].lstrip('- '),'kind':'description','evidence_ids':[r['id']]} for r in payload['evidence']]
|
|
claims.append({'text':'Prvá práca je o 99 percent úspešnejšia.','kind':'comparison','comparison_kind':'difference',
|
|
'evidence_ids':[r['id'] for r in payload['evidence']]})
|
|
return {'choices':[{'message':{'content':json.dumps({'claims':claims})}}]}
|
|
out=s.synthesize(result,plan,complete=model,diagnostics=True)
|
|
assert out['synthesis']['status']=='accepted' and '99' not in out['answer']
|
|
assert 'agent RAG' in out['answer'] and 'dotrénovanie modelu' in out['answer']
|
|
|
|
|
|
def test_source_count_fits_output_schemas_and_input_budget_never_silently_clips():
|
|
result,plan=meeting_result();row=next(iter(s.prepare_evidence(result).values()))
|
|
result['evidence']=[{**row,'id':f'unit-{i}','text':f'Funguje modul {i}.'} for i in range(1,49)]
|
|
draft=s.Draft(claims=[s.ProseClaim(text=r['text'],kind='description',evidence_ids=[r['id']]) for r in result['evidence']])
|
|
review=s.Review(coverage='supported',verdicts=[s.Verdict(claim_id=i,decision='supported',reason='entailed') for i in range(48)])
|
|
assert len(draft.claims)==len(review.verdicts)==48
|
|
out=s.synthesize(result,plan,complete=reviewed_draft,diagnostics=True)
|
|
assert out['synthesis']=={'status':'fallback','reason':'input_budget'}
|
|
assert out['answer']==result['answer'] and len(out['evidence'])==48
|
|
|
|
|
|
def test_generic_limit_notices_are_combined_but_specific_missing_facts_stay():
|
|
out=clean_answer_notices({'answer':'Prechod grafom dosiahol limit výberu. Zúž otázku.\n\n'
|
|
'Zobrazené sú príklady z obmedzeného výberu textových dôkazov.\n\n'
|
|
'Jakub Schwarc: chýba dátum vykonania experimentu.'})
|
|
assert out['answer'].count(LIMIT_NOTICE)==1 and 'Jakub Schwarc: chýba dátum' in out['answer']
|