322 lines
18 KiB
Python
322 lines
18 KiB
Python
"""Question semantics, conservative LLM assistance and actual shared execution."""
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from tools.answers.question_plan import (QuestionPlan,build_question_plan,validate_model_plan,PlanRejected,PlanModelUnavailable)
|
|
from tools.knowledge.sources import profile_units
|
|
from tools.knowledge.planned_answers import answer_facets
|
|
|
|
CATALOG=[{'kind':'Person','name':name,'source_document':path} for name,path in [
|
|
('Ondrej Megela','pages/students/2018/ondrej_megela/README.md'),
|
|
('Ján Holp','pages/students/2016/jan_holp/README.md')]]
|
|
|
|
|
|
def plan(q,**kwargs):return build_question_plan(q,CATALOG,**kwargs)
|
|
|
|
|
|
def test_general_question_is_an_overview_but_specific_duplicate_project_needs_selection():
|
|
broad=plan('Aké technické problémy riešil Ondrej Megela?')
|
|
assert broad.spec.intent=='facets' and broad.spec.scope=='across_works' and broad.spec.facets==['problems']
|
|
specific=plan('Aké metódy používal Ján Holp pri diplomovom projekte v roku 2020?')
|
|
assert specific.spec.scope=='single_work' and specific.spec.works[0].years==[2020]
|
|
assert specific.spec.works[0].work_type=='diploma_project'
|
|
|
|
|
|
def test_two_people_type_and_year_remain_bound_to_their_own_work():
|
|
p=plan('Porovnaj zadanie bakalárskej práce Ondreja Megelu z roku 2020 a diplomovej práce Jána Holpa z roku 2021.').spec
|
|
assert p.matches_work('Ondrej Megela','Bakalárska práca 2020','bachelor_thesis',2020)
|
|
assert not p.matches_work('Ondrej Megela','Bakalárska práca 2021','bachelor_thesis',2021)
|
|
assert p.matches_work('Ján Holp','Diplomová práca 2021','diploma_thesis',2021)
|
|
assert not p.matches_work('Ján Holp','Diplomová práca 2020','diploma_thesis',2020)
|
|
|
|
|
|
@pytest.mark.parametrize('q,reason',[
|
|
('Aké práce má Xaver Neexistujúci?','unknown_person'),
|
|
('Aké práce má Ján?','incomplete_person_name'),
|
|
('Kto týmto tempom dokončí prácu v roku 2027?','unsupported_constraint'),
|
|
('Daj obe.','missing_followup_context'),
|
|
])
|
|
def test_ambiguity_unknown_identity_and_unsupported_conditions_never_call_model(q,reason):
|
|
result=plan(q,complete=lambda _:pytest.fail('No model may invent missing identity/context'))
|
|
assert result.reason==reason and result.clarification
|
|
|
|
|
|
def test_work_interval_covers_intermediate_years():
|
|
p=plan('Aké práce má Ján Holp od roku 2019 do roku 2021?').spec
|
|
assert p.period.field=='work_year'
|
|
assert p.matches_work('Ján Holp','Diplomový projekt 2020','diploma_project',2020)
|
|
assert not p.matches_work('Ján Holp','Diplomový projekt 2022','diploma_project',2022)
|
|
|
|
|
|
def test_record_period_is_not_work_year():
|
|
p=plan('Ako pokročil Ondrej Megela v roku 2021?').spec
|
|
assert p.period.field=='record_date' and p.works[0].years==[]
|
|
|
|
|
|
@pytest.mark.parametrize('field,value',[
|
|
('question','Aké práce má niekto iný?'),('people',[]),('works',[]),('years',[2099]),
|
|
('conditions',[{'kind':'tag','quote':'tag RAG'}]),('scope','across_works'),('facets',[]),
|
|
])
|
|
def test_a_structurally_valid_model_plan_cannot_change_known_meaning(field,value):
|
|
baseline=plan('Aké problémy riešil Ondrej Megela pri bakalárskej práci v roku 2020?').spec
|
|
altered=baseline.model_dump();altered[field]=value
|
|
with pytest.raises(PlanRejected):validate_model_plan(QuestionPlan.model_validate(altered),baseline)
|
|
|
|
|
|
@pytest.mark.parametrize('intent',['facets','progress'])
|
|
def test_complex_paraphrase_uses_typed_plan_not_a_rewritten_question(intent):
|
|
calls=[]
|
|
def model(messages):
|
|
calls.append(messages)
|
|
value=json.loads(messages[1]['content'])['baseline']
|
|
value.update(intent=intent,facets=['problems'])
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
result=plan('Povedz mi, s čím zápasil Ondrej Megela počas svojej roboty.',complete=model,verify_complete=lambda _: {'choices':[{'message':{'content':'{"decision":"supported"}'}}]})
|
|
assert result.origin=='model' and result.spec.intent=='facets'
|
|
assert len(calls)==1 and result.spec.people[0].name=='Ondrej Megela'
|
|
|
|
|
|
def test_invalid_model_json_retries_but_never_executes_an_unvalidated_plan():
|
|
calls=[]
|
|
def model(messages):calls.append(1);return {'choices':[{'message':{'content':'not JSON'}}]}
|
|
result=plan('Povedz mi, s čím zápasil Ondrej Megela počas svojej roboty.',complete=model)
|
|
# No model plan is executed; the user gets a concrete question instead of a rejection.
|
|
assert result.reason=='unsupported_content_question' and 'Ondrej Megela' in result.clarification
|
|
assert result.origin=='direct' and result.spec.intent=='metadata'
|
|
assert len(calls)==2
|
|
|
|
|
|
def test_provider_failure_is_not_no_answer():
|
|
def model(messages):raise RuntimeError('PRIVATE_ENDPOINT')
|
|
with pytest.raises(PlanModelUnavailable):plan('Povedz mi, s čím zápasil Ondrej Megela počas svojej roboty.',complete=model)
|
|
|
|
|
|
def source_rows():
|
|
return profile_units({'title':'Ján Holp','path':CATALOG[1]['source_document'],
|
|
'content':'# Diplomový projekt 2020\nStav:\n- Vyriešený problém Cassandra.\n# Diplomový projekt 2 2020\nStav:\n- Vznikol problém BERT.\nÚlohy:\n- Otestujte model.'},include_records=True)
|
|
|
|
|
|
@pytest.mark.parametrize('engine',['rag','graphrag'])
|
|
def test_actual_facet_executor_groups_works_and_preserves_source_bindings(engine):
|
|
p=plan('Aké problémy riešil Ján Holp?')
|
|
rows=source_rows();index={'index_version':{'version':'v'},'version':'content','units':rows}
|
|
result=answer_facets(p,index,rows,engine=engine,limit=5)
|
|
assert result['status']=='ok' and result['accepted_for_delivery']
|
|
assert 'Cassandra' in result['answer'] and 'BERT' in result['answer']
|
|
assert 'Otestujte' not in result['answer']
|
|
assert result['source_urls']==['https://zp.kemt.fei.tuke.sk/students/2016/jan_holp']
|
|
assert {r['work_label'] for r in result['evidence']}=={'Diplomový projekt 2020','Diplomový projekt 2 2020'}
|
|
|
|
|
|
def test_single_project_asks_for_concrete_choice_but_both_projects_are_returned():
|
|
rows=source_rows();index={'index_version':{'version':'v'},'version':'content','units':rows}
|
|
a=answer_facets(plan('Aké problémy riešil Ján Holp pri diplomovom projekte v roku 2020?'),index,rows,engine='rag',limit=5)
|
|
assert a['reason']=='ambiguous_work' and 'projekt 2' in a['answer']
|
|
b=answer_facets(plan('Aké problémy riešil Ján Holp pri oboch diplomových projektoch v roku 2020?'),index,rows,engine='rag',limit=5)
|
|
assert b['status']=='ok' and 'Cassandra' in b['answer'] and 'BERT' in b['answer']
|
|
|
|
|
|
@pytest.mark.parametrize('question',[
|
|
'Aké problémy riešil Ján Holp?',
|
|
'Aké problémy riešil Ján Holp pri oboch diplomových projektoch v roku 2020?',
|
|
'Porovnaj problémy Jána Holpa pri oboch diplomových projektoch v roku 2020.',
|
|
])
|
|
def test_planned_facet_output_passes_the_existing_mcp_client_contract(monkeypatch,question):
|
|
from packages.claude import client
|
|
rows=source_rows();index={'index_version':{'version':'v'},'version':'content','units':rows}
|
|
result=answer_facets(plan(question),index,rows,engine='rag',limit=5)
|
|
assert result['status']=='ok'
|
|
monkeypatch.setattr(client,'request_json',lambda *a,**kw:{**result,'query':question,'engine':'rag'})
|
|
delivered=client.query_backend(question,'rag',config=object())
|
|
assert delivered['status']=='ok' and delivered['accepted_for_delivery']
|
|
assert delivered['answer']==result['answer'] and delivered['source_urls']==result['source_urls']
|
|
|
|
|
|
def test_no_unsupported_fields_in_provider_schema_and_full_local_validation():
|
|
from tools.answers.model_client import strict_output_schema
|
|
schema=strict_output_schema(QuestionPlan.model_json_schema())
|
|
assert schema['additionalProperties'] is False and set(schema['required'])==set(schema['properties'])
|
|
with pytest.raises(ValidationError):QuestionPlan(question='q',intent='execute_cypher')
|
|
|
|
|
|
@pytest.mark.parametrize('verdict',['unsupported','uncertain'])
|
|
def test_plausible_but_semantically_wrong_route_is_vetoed(verdict):
|
|
def proposer(messages):
|
|
value=json.loads(messages[1]['content'])['baseline']
|
|
value.update(intent='facets',facets=['results'])
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
def judge(messages):
|
|
return {'choices':[{'message':{'content':json.dumps({'decision':verdict})}}]}
|
|
result=plan('Povedz mi, s čím zápasil Ondrej Megela počas svojej roboty.',complete=proposer,verify_complete=judge)
|
|
assert result.reason=='unsupported_content_question' and result.clarification
|
|
assert result.origin=='direct' and result.spec.intent!='facets' and not result.spec.facets
|
|
|
|
|
|
def test_malformed_provider_message_is_rejected_not_a_retrieval_error():
|
|
result=plan('Povedz mi, s čím zápasil Ondrej Megela počas svojej roboty.',complete=lambda _: {'choices':[{'message':{'content':None}}]})
|
|
assert result.reason=='unsupported_content_question' and result.origin=='direct'
|
|
|
|
|
|
def test_explicit_staff_role_does_not_disappear_when_staff_has_a_student_profile():
|
|
p=plan('Ktoré práce vedie Ján Holp?')
|
|
assert p.spec.people[0].role=='supervisor'
|
|
|
|
|
|
@pytest.mark.parametrize('provider,key',[('anthropic','output_config'),('openai_compatible','response_format')])
|
|
def test_schema_is_sent_using_provider_specific_transport(monkeypatch,provider,key):
|
|
from tools.answers import model_client
|
|
sent=[]
|
|
class Reply:
|
|
def __enter__(self):return self
|
|
def __exit__(self,*args):pass
|
|
def read(self,*args):
|
|
return json.dumps({'stop_reason':'end_turn','content':[{'type':'text','text':'{}'}]} if provider=='anthropic' else {'choices':[{'finish_reason':'stop','message':{'content':'{}'}}]}).encode()
|
|
class Opener:
|
|
def open(self,request,**kwargs):sent.append(json.loads(request.data));return Reply()
|
|
monkeypatch.setattr(model_client.urllib.request,'build_opener',lambda *a:Opener())
|
|
config={'provider':provider,'model':'test','key':'private','endpoint':'https://example.test/api'}
|
|
model_client._complete_request([{'role':'user','content':'q'}],config,output_schema={'type':'object','properties':{},'additionalProperties':False})
|
|
assert key in sent[0]
|
|
|
|
|
|
def test_requested_table_and_real_limit_are_respected():
|
|
rows=source_rows();index={'index_version':{'version':'v'},'version':'content','units':rows}
|
|
p=plan('Aké problémy riešil Ján Holp? Uveď ich v tabuľke.')
|
|
value=answer_facets(p,index,rows,engine='rag',limit=5)
|
|
assert '| Oblasť |' in value['answer']
|
|
limited=answer_facets(p,index,rows,engine='rag',limit=1)
|
|
assert limited['truncated'] and 'Zúž otázku' in limited['answer']
|
|
assert len({r['work_label'] for r in limited['evidence']})==1
|
|
|
|
|
|
def test_current_status_is_preserved_without_inference_from_old_notes():
|
|
p=plan('Aké problémy teraz riešil Ján Holp?')
|
|
assert any(c.kind=='current_status' for c in p.spec.conditions)
|
|
rows=source_rows();value=answer_facets(p,{'index_version':{},'version':'v'},rows,engine='rag',limit=5)
|
|
assert 'nepotvrdzujú aktuálny stav' in value['answer']
|
|
|
|
|
|
def test_explicit_record_dates_are_not_silently_replaced_by_work_years():
|
|
p=plan('Aké problémy riešil Ondrej Megela od 1.9.2020 do 31.1.2021?').spec
|
|
assert p.period.field=='record_date' and p.period.start=='2020-09-01' and p.period.end=='2021-01-31'
|
|
assert p.works[0].years==[]
|
|
|
|
|
|
def test_work_year_is_distinct_from_a_record_interval():
|
|
p=plan('Aké problémy riešil Ondrej Megela pri bakalárskej práci 2020 od 1.1.2021 do 31.12.2021?').spec
|
|
assert p.period.field=='record_date' and p.works[0].years==[2020]
|
|
|
|
|
|
def test_structural_format_never_licenses_added_sql_or_cypher():
|
|
value=plan('Aké práce má Ján Holp?').spec.model_dump();value['cypher']='MATCH (n) DELETE n'
|
|
with pytest.raises(ValidationError):QuestionPlan.model_validate(value)
|
|
|
|
|
|
def test_plain_work_question_with_year_is_direct_not_an_llm_paraphrase():
|
|
p=plan('Akú prácu mal Ján Holp v roku 1999?',complete=lambda _:pytest.fail('Clear year/work query must be direct'))
|
|
assert p.origin=='direct' and p.spec.intent=='metadata' and p.spec.years==[1999]
|
|
|
|
|
|
@pytest.mark.parametrize('q',[
|
|
'Aké problémy riešil Ján Holp pri diplomovej práci v roku 99?',
|
|
'Aké problémy riešil Ján Holp bez Cassandry?',
|
|
'Aké problémy riešil Ján Holp v marci?',
|
|
])
|
|
def test_recognized_but_unimplemented_filter_never_expands_to_general_results(q):
|
|
p=plan(q,complete=lambda _:pytest.fail('Unsupported filters need clarification, not a model'))
|
|
assert p.clarification and p.spec.unsupported
|
|
|
|
|
|
def test_explicit_consultant_is_a_known_identity_not_inferred_from_authorship():
|
|
catalog=CATALOG+[{'kind':'DocumentAuthor','name':'Daniel Hladek'}]
|
|
p=build_question_plan('Ktoré práce konzultoval Daniel Hladek?',catalog)
|
|
assert p.resolution['status']=='ok' and p.spec.people[0].role=='consultant'
|
|
|
|
|
|
def test_plural_work_request_is_not_forced_to_pick_one_work():
|
|
p=plan('Aké diplomové práce má Ján Holp?').spec
|
|
assert p.scope=='selected_works'
|
|
|
|
|
|
def test_graph_adapter_filters_each_persons_work_year_before_counting():
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
|
q='Aké názvy majú bakalárska práca Ondrej Megela 2020 a diplomová práca Ján Holp 2021?'
|
|
spec=plan(q).spec
|
|
rows=[{'person':name,'person_element_id':name,'thesis_element_id':str(i),'thesis_id':str(i),'thesis_type':kind,'heading':label,'year':year,'title':'Téma '+str(i),'document_path':path,'source_url':'https://example.test/'+str(i)}
|
|
for i,(name,kind,label,year,path) in enumerate([
|
|
('Ondrej Megela','bachelor_thesis','Bakalárska práca 2020',2020,CATALOG[0]['source_document']),
|
|
('Ondrej Megela','bachelor_thesis','Bakalárska práca 2021',2021,CATALOG[0]['source_document']),
|
|
('Ján Holp','diploma_thesis','Diplomová práca 2021',2021,CATALOG[1]['source_document'])])]
|
|
catalog=[{**r,'element_id':r['name']} for r in CATALOG]
|
|
result=gs.search_graph_with_session(FakeSession({gs.CATALOG_QUERY:catalog,gs.THESIS_FACTS_QUERY:rows}),q,question_plan=spec)
|
|
assert not any(f.get('person')=='Ondrej Megela' and f.get('year')==2021 for f in result['facts'])
|
|
assert any(f.get('person')=='Ondrej Megela' and f.get('year')==2020 for f in result['facts'])
|
|
|
|
|
|
def test_llm_can_bind_supplied_years_but_never_invent_or_drop_them():
|
|
q='Porovnaj zadania: Ondrej Megela bakalárska práca 2020, Ján Holp diplomová práca 2021.'
|
|
def proposer(messages):
|
|
value=json.loads(messages[1]['content'])['baseline']
|
|
value['works'][0]['years']=[2020];value['works'][1]['years']=[2021]
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
def judge(messages):return {'choices':[{'message':{'content':'{"decision":"supported"}'}}]}
|
|
p=plan(q,complete=proposer,verify_complete=judge)
|
|
assert p.origin=='model' and p.spec.works[0].years==[2020] and p.spec.works[1].years==[2021]
|
|
base=plan(q).spec;invented=p.spec.model_copy(deep=True);invented.works[0].years=[2099]
|
|
with pytest.raises(PlanRejected):validate_model_plan(invented,base,allow_work_binding=True)
|
|
|
|
|
|
def test_slovak_summary_imperative_is_not_misclassified_as_gibberish():
|
|
p=plan('Zhrň zadanie a výsledky diplomovej práce Jána Holpa.')
|
|
assert not p.clarification and p.spec.intent=='facets'
|
|
|
|
|
|
@pytest.mark.parametrize('alias,kind',[('BP','bachelor_thesis'),('DP','diploma_thesis'),('PhD','doctoral_thesis'),('TP','team_project')])
|
|
def test_existing_graph_work_type_aliases_remain_explicit_in_the_plan(alias,kind):
|
|
p=plan('Aký názov má '+alias+' Jána Holpa z roku 2021?').spec
|
|
assert p.works[0].work_type==kind and p.works[0].years==[2021]
|
|
|
|
|
|
def test_model_repairs_changed_per_person_scope_without_relaxing_validation():
|
|
q='Porovnaj zadania: Ondrej Megela bakalárska práca 2020, Ján Holp diplomová práca 2021.'
|
|
calls=[]
|
|
def propose(messages):
|
|
calls.append(messages[-1]['content'])
|
|
value=json.loads(messages[1]['content'])['baseline']
|
|
value['works'][0]['years']=[2020];value['works'][1]['years']=[2021]
|
|
if len(calls)==1:value['scope']='selected_works'
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
judge=lambda _: {'choices':[{'message':{'content':'{"decision":"supported"}'}}]}
|
|
result=plan(q,complete=propose,verify_complete=judge)
|
|
assert result.spec.scope=='single_work' and len(calls)==2
|
|
assert 'Changed work selection scope' in calls[1] and 'scope=single_work' in calls[1]
|
|
|
|
|
|
def test_work_limit_does_not_starve_the_second_requested_person():
|
|
first=profile_units({'title':'Ondrej Megela','path':CATALOG[0]['source_document'],
|
|
'content':'# Bakalárska práca 2020\nStav:\n- Vznikol problém s Pytorch.\n# Diplomová práca 2023\nStav:\n- Vznikol problém s vyhodnotením.'},include_records=True)
|
|
rows=first+source_rows()
|
|
p=plan('Aké problémy riešili Ondrej Megela a Ján Holp?')
|
|
result=answer_facets(p,{'index_version':{},'version':'v'},rows,engine='rag',limit=2)
|
|
assert {r['person'] for r in result['evidence']}=={'Ondrej Megela','Ján Holp'}
|
|
assert result['truncated']
|
|
|
|
|
|
def test_budget_exhaustion_is_not_misreported_as_absent_evidence():
|
|
text='\n'.join('# Diplomová práca '+str(y)+'\nStav:\n- Vznikol problém. '+('Podrobný záznam. '*350) for y in range(2020,2024))
|
|
rows=profile_units({'title':'Ján Holp','path':CATALOG[1]['source_document'],'content':text},include_records=True)
|
|
result=answer_facets(plan('Aké problémy riešil Ján Holp?'),{'index_version':{},'version':'v'},rows,engine='rag',limit=4)
|
|
assert result['status']=='ambiguous' and result['reason']=='content_budget_exceeded'
|
|
assert 'údaje chýbajú' in result['answer']
|
|
|
|
|
|
@pytest.mark.parametrize('suffix',['Uveď to v tabuľke.','Odpovedz stručne.'])
|
|
def test_model_cannot_drop_an_explicit_presentation_requirement(suffix):
|
|
baseline=plan('Aké problémy riešil Ján Holp? '+suffix).spec
|
|
changed=baseline.model_copy(update={'presentation':'list'})
|
|
with pytest.raises(PlanRejected):validate_model_plan(changed,baseline)
|