241 lines
14 KiB
Python
241 lines
14 KiB
Python
"""Previously failed user formulations are regressions, never a fresh TEST."""
|
|
import base64
|
|
import zlib
|
|
from types import SimpleNamespace
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from tools.answers.question_plan import build_question_plan,QuestionPlan,PersonRequest
|
|
from tools.retrieval.person_names import resolve_person_names
|
|
from tools.knowledge.text_retrieval import retrieve_units
|
|
|
|
CATALOG=[{'kind':'Person','name':name,'source_document':f'pages/students/2024/{i}/README.md'}
|
|
for i,name in enumerate(['Adrián Ondra','Sebastián Kišš','Samuel Štucka','Samuel Džurina'])]
|
|
|
|
|
|
@pytest.mark.parametrize('suffix',[
|
|
'Zhrň všetky doložené body a nezamieňaj ich za hotové výsledky.',
|
|
'Vysvetli konkrétne prepojenie a dolož obe osoby.',
|
|
'Pri každej osobe príslušné práce uveď v bodoch.',
|
|
])
|
|
def test_presentation_clause_is_not_an_additional_unknown_identity(suffix):
|
|
r=resolve_person_names('Na čom pracoval Adrián Ondra? '+suffix,CATALOG)
|
|
assert r['status']=='ok' and [m['name'] for m in r['matches']]==['Adrián Ondra']
|
|
|
|
|
|
def test_inflected_partial_first_name_asks_for_real_candidates_without_model():
|
|
p=build_question_plan('Čo sa podarilo Samuelovi pri jeho práci?',CATALOG)
|
|
assert p.reason=='incomplete_person_name' and p.resolution['status']=='ambiguous'
|
|
assert {c['name'] for c in p.resolution['candidates']}=={'Samuel Štucka','Samuel Džurina'}
|
|
|
|
|
|
def test_one_short_surname_typo_still_requires_an_exact_given_name_anchor():
|
|
assert resolve_person_names('Akú prácu má Sebastián Kiš?',CATALOG)['matches'][0]['name']=='Sebastián Kišš'
|
|
assert resolve_person_names('Akú prácu má Sebastan Kš?',CATALOG)['status']!='ok'
|
|
|
|
|
|
@pytest.mark.parametrize('question',[
|
|
'Kto už vyskúšal hierarchickú sumarizáciu na problém s dlhým kontextom?',
|
|
'U koho je doložené vytvorenie crawlera pomocou Scrapy?',
|
|
'Ktorý študent sa venoval agentovi pre špeciálneho pedagóga? Bez domýšľania výsledkov.',
|
|
])
|
|
def test_global_discovery_does_not_require_a_known_student(question):
|
|
p=build_question_plan(question,CATALOG)
|
|
assert not p.clarification and p.spec.intent=='discovery' and not p.spec.people
|
|
|
|
|
|
def test_grounding_directive_does_not_add_a_results_facet_to_goals():
|
|
p=build_question_plan('Aké sú ciele projektu osoby Adrián Ondra? Nezamieňaj ich za hotové výsledky.',CATALOG)
|
|
assert p.spec.facets==['assignment'] and not p.clarification
|
|
|
|
|
|
def index_rows():
|
|
rows=[{'id':str(i),'person':'Eva Nová','source_path':'eva','line':i+1,
|
|
'text':'Všeobecná poznámka.' if i else 'Použitá technológia Cassandra.',
|
|
'concepts':[],'work_label':'Diplomová práca 2026','work_type':'diploma_thesis','work_year':2026}
|
|
for i in range(100)]
|
|
matrix=np.array([[0.,1.]]+[[1.,0.]]*99,dtype='<f4')
|
|
return {'version':'unit-v1','units':rows,'embedding_fingerprint':'fp','vector_shape':[100,2],
|
|
'vectors':base64.b64encode(zlib.compress(matrix.tobytes())).decode()}
|
|
|
|
|
|
def test_semantic_candidates_survive_a_successful_lexical_hit(monkeypatch):
|
|
import tools.rag.embedding_utils as embedding
|
|
monkeypatch.setattr(embedding,'model_profile',lambda _: {'fingerprint':'fp'})
|
|
monkeypatch.setattr(embedding,'embed_query',lambda _:np.array([1.,0.]))
|
|
plan=QuestionPlan(question='Cassandra',intent='discovery')
|
|
rows,scores,trace=retrieve_units(plan.question,index_rows(),plan)
|
|
assert '0' in {r['id'] for r in rows}
|
|
assert any(r['id']!='0' for r in rows) and scores['1']>scores['0']
|
|
assert trace['lexical_candidates']==1 and trace['vector_candidates']==80
|
|
|
|
|
|
def test_named_scope_expansion_preserves_every_goal_and_excludes_other_people(monkeypatch):
|
|
index=index_rows();index.pop('vectors')
|
|
index['units'].append({**index['units'][0],'id':'other','person':'Dana Iná','source_path':'dana'})
|
|
plan=QuestionPlan(question='Všetky ciele Evy',intent='facets',people=[PersonRequest(name='Eva Nová',source_document='eva')])
|
|
rows,_,trace=retrieve_units(plan.question,index,plan)
|
|
assert len(rows)==100 and all(r['person']=='Eva Nová' for r in rows)
|
|
assert trace['named_scope_expansion']
|
|
|
|
|
|
def metadata_payload(presentation='list',fields=None):
|
|
from tools.rag.rag_utils import build_context_text
|
|
source={'source_id':'S1','title':'Adrián Ondra','document_path':CATALOG[0]['source_document'],
|
|
'source_url':'https://example.test/adrian','tags':['rag','nlp'],
|
|
'text':'# Bakalársky projekt 2027\nAutomatická tvorba myšlienkových máp\nMožné ciele semestrálneho projektu:\n- Vytvoriť vizualizáciu.\n# Diplomový projekt 2029\nStav:\n- Vyskúšaný model.'}
|
|
plan=QuestionPlan(question='Zostav prehľad prác Adriána Ondru.',people=[PersonRequest(name='Adrián Ondra',source_document=source['document_path'])],
|
|
fields=fields if fields is not None else ['work_type','title','year'],presentation=presentation)
|
|
return {'sources':[source],'question_plan':plan.model_dump(),'context':build_context_text([source]),
|
|
'named_scope_complete':True,'index_version':{'version':'fixture'},'truncation_reasons':[]}
|
|
|
|
|
|
def test_exact_metadata_keeps_distinct_untitled_records_without_a_model():
|
|
from tools.answers.verified_rag import generate_verified_rag
|
|
result=generate_verified_rag('Zostav prehľad prác Adriána Ondru.',metadata_payload(),lambda _:pytest.fail('No model for exact complete records'))
|
|
assert result['status']=='ok' and len(result['claims'])==2
|
|
assert len({r['record_id'] for r in result['claims']})==2
|
|
assert 'myšlienkových máp' in result['answer'] and 'názov v zdroji nie je uvedený' in result['answer']
|
|
|
|
|
|
def test_exact_count_counts_records_not_source_windows_or_people():
|
|
from tools.answers.verified_rag import generate_verified_rag
|
|
payload=metadata_payload('count',[])
|
|
result=generate_verified_rag('Zostav prehľad prác Adriána Ondru.',payload,lambda _:pytest.fail('No model for exact count'))
|
|
assert result['count']==2 and 'prác a projektov' in result['answer']
|
|
|
|
|
|
@pytest.mark.parametrize('presentation',['list','count'])
|
|
def test_exact_metadata_is_accepted_by_the_actual_claude_client(monkeypatch,presentation):
|
|
from packages.claude import client
|
|
from tools.answers.verified_rag import generate_verified_rag
|
|
question='Zostav prehľad prác Adriána Ondru.'
|
|
result=generate_verified_rag(question,metadata_payload(presentation),lambda _:pytest.fail('No model'))
|
|
monkeypatch.setattr(client,'request_json',lambda *a,**kw:{**result,'query':question,'engine':'rag'})
|
|
delivered=client.query_backend(question,'rag',config=client.Config('http://localhost:8000','test'))
|
|
assert delivered['status']=='ok' and delivered['accepted_for_delivery']
|
|
assert delivered['answer']==result['answer']
|
|
|
|
|
|
def test_complete_scope_flag_cannot_override_context_clipping():
|
|
from tools.answers.rag_metadata import exact_metadata_answer
|
|
from tools.answers.rag_evidence import rag_evidence_context
|
|
payload=metadata_payload();payload['truncation_reasons']=['llm_context_budget']
|
|
assert exact_metadata_answer(payload,rag_evidence_context('Zostav prehľad prác Adriána Ondru.',payload)) is None
|
|
|
|
|
|
def test_document_authorship_is_not_interpreted_as_an_advisor_filter():
|
|
p=build_question_plan('Kto vedie bakalársku prácu Adriána Ondru a kto je autor dokumentu?',CATALOG)
|
|
assert p.spec.intent=='advisors' and not p.clarification
|
|
assert 'document_author' in p.spec.fields and not p.spec.conditions
|
|
|
|
|
|
def test_profile_metadata_has_its_own_time_role_and_snapshot(monkeypatch,tmp_path):
|
|
import sqlite3
|
|
from contextlib import nullcontext
|
|
import tools.sync.index_state as state
|
|
from tools.knowledge.academic import with_profile_metadata
|
|
db=tmp_path/'db.sqlite'
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute('CREATE TABLE documents(path,author,published)')
|
|
conn.execute('CREATE TABLE chunks(chunk_id,document_path,text,published)')
|
|
path=CATALOG[0]['source_document']
|
|
conn.execute('INSERT INTO documents VALUES (?,?,1)',(path,'Daniel Hladek'))
|
|
conn.execute('INSERT INTO chunks VALUES (?,?,?,1)',('c1',path,'rok začiatku štúdia: 2024'))
|
|
monkeypatch.setattr(state,'index_snapshot',lambda *a,**k:nullcontext())
|
|
monkeypatch.setattr(state,'read_version',lambda *a:{'version':'v'})
|
|
spec=QuestionPlan(question='Odlíš roly a roky.',people=[PersonRequest(name='Adrián Ondra',source_document=path)],fields=['document_author','start_year','year'])
|
|
plan=SimpleNamespace(spec=spec)
|
|
original={'status':'ok','accepted_for_delivery':True,'index_version':{'version':'v'},
|
|
'answer':'Vedúci: Matúš Pleva. Rok práce: 2027.\n\nZdroje: https://example.test/adrian','source_urls':['https://example.test/adrian']}
|
|
result=with_profile_metadata(original,plan,db)
|
|
assert 'Matúš Pleva' in result['answer'] and 'Autor dokumentu profilu Adrián Ondra: Daniel Hladek.' in result['answer']
|
|
assert 'začiatku štúdia — Adrián Ondra: 2024' in result['answer']
|
|
assert any(r.get('quote')=='rok začiatku štúdia: 2024' for r in result['metadata_evidence'])
|
|
with pytest.raises(RuntimeError,match='snapshot'):
|
|
with_profile_metadata({**original,'index_version':{'version':'other'}},plan,db)
|
|
|
|
|
|
def test_graph_work_count_is_not_the_number_of_authors():
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
|
from tools.graphrag.tests.test_graph_search_entity_v2 import catalog,thesis
|
|
q='Koľko prác má Alena Nová?'
|
|
bound=build_question_plan(q,catalog())
|
|
rows=[{**thesis(),'thesis_id':'a','heading':'Bakalárska práca 2024'},
|
|
{**thesis(),'thesis_id':'b','heading':'Bakalárska práca 2025','year':2025}]
|
|
session=FakeSession({gs.CATALOG_QUERY:catalog(),gs.THESIS_FACTS_QUERY:rows})
|
|
result=gs.search_graph_with_session(session,q,question_plan=bound.spec)
|
|
assert result['aggregation']['count']==2 and result['aggregation']['people']==['Alena Nová']
|
|
assert result['aggregation']['count_domain']=='academic_records'
|
|
|
|
|
|
def test_declarative_stack_is_a_method_description_without_promoting_its_phase():
|
|
from tools.answers.content_selection import facets_for
|
|
q='Aké technológie má študent na frontende, backende a čo je doložené o funkčnosti?'
|
|
assert 'methods' in facets_for('Frontend React, backend Flask, DB Postgres.','other',q,'unknown')
|
|
assert 'methods' not in facets_for('Frontend bude React, backend bude Flask.','task',q,'planned')
|
|
assert 'results' in facets_for('Aplikácia funguje.','progress',q,'result')
|
|
|
|
|
|
@pytest.mark.parametrize('extra_work,expected', [('',True),('\n# Diplomová práca 2029\nStav:\n- Funguje systém.',False)])
|
|
def test_profile_header_role_is_qualified_and_never_broadcast_to_multiple_works(extra_work,expected):
|
|
from tools.knowledge.sources import profile_units
|
|
from tools.knowledge.academic import answer_academic
|
|
path=CATALOG[0]['source_document']
|
|
units=profile_units({'title':'Adrián Ondra','path':path,'content':'Vedúci: Matúš Pleva\n# Bakalárska práca 2027\nStav:\n- Funguje aplikácia.'+extra_work},include_records=True)
|
|
bound=build_question_plan('Kto vedie bakalársku prácu Adriána Ondru?',CATALOG)
|
|
index={'index_version':{'version':'v'},'version':'content','academic_schema':1,'units':units}
|
|
result=answer_academic(bound.spec.question,index,engine='rag',mode='advisors',plan=bound)
|
|
assert ('Matúš Pleva' in result['answer']) is expected
|
|
if expected:
|
|
assert 'v hlavičke profilu' in result['answer']
|
|
assert any(r['kind']=='supervisor' and r.get('work_id') is None for r in result['evidence'])
|
|
|
|
|
|
@pytest.mark.parametrize('consultant_profile',[False,True])
|
|
def test_mentioning_a_consultant_does_not_turn_the_supervisor_into_a_consultant(consultant_profile):
|
|
consultant={'kind':'DocumentAuthor','name':'Kristián Sopkovič'}
|
|
if consultant_profile:consultant.update(kind='Person',source_document='pages/students/2020/kristian/README.md')
|
|
catalog=[{'kind':'DocumentAuthor','name':'Daniel Hladek'}, consultant]
|
|
plan=build_question_plan('Ktoré práce vedie Daniel Hladek a konzultuje Kristián Sopkovič?',catalog)
|
|
assert [(p.name,p.role) for p in plan.spec.people]==[('Daniel Hladek','supervisor'),('Kristián Sopkovič','consultant')]
|
|
|
|
|
|
def test_created_chatbot_is_not_satisfied_by_a_future_assignment():
|
|
from tools.knowledge.query import activity_supported
|
|
q='Kto vytvoril chatbota?'
|
|
assert not activity_supported(q,{'text':'Vytvorte chatbota.','phase':'assignment'})
|
|
assert activity_supported(q,{'text':'Vytvorený modelový chatbot.','phase':'result'})
|
|
|
|
|
|
def test_ner_is_not_proved_by_generic_form_slot_recognition_or_model_concepts():
|
|
from tools.knowledge.query import activity_supported
|
|
q='Kto sa venoval rozpoznávaniu pomenovaných entít?'
|
|
row={'text':'Nedarí sa rozpoznať typ priznania za psa vo Form.','phase':'progress','concepts':['NER']}
|
|
assert not activity_supported(q,row)
|
|
assert activity_supported(q,{'text':'Vyskúšaný model SlovakBERT-NER na rozpoznávanie pomenovaných entít.','phase':'result'})
|
|
|
|
|
|
def test_possible_goals_keep_the_literal_qualification_for_prose_rendering():
|
|
from tools.knowledge.sources import profile_units
|
|
from tools.knowledge.planned_answers import answer_facets
|
|
rows=profile_units({'title':'Adrián Ondra','path':CATALOG[0]['source_document'],
|
|
'content':'# Bakalársky projekt 2027\nMožné ciele semestrálneho projektu:\n- Vytvoriť vizualizáciu.'},include_records=True)
|
|
bound=build_question_plan('Aké sú ciele projektu Adriána Ondru?',CATALOG)
|
|
index={'version':'c','index_version':{'version':'v'},'units':rows}
|
|
result=answer_facets(bound,index,rows,engine='rag',limit=5)
|
|
assert 'Vytvoriť vizualizáciu' in result['answer']
|
|
assert any('možné/navrhované ciele' in note for note in result['presentation_notes'])
|
|
|
|
|
|
def test_comparing_time_roles_does_not_dump_the_meeting_agenda():
|
|
from tools.knowledge.sources import profile_units
|
|
from tools.knowledge.academic import answer_academic
|
|
path=CATALOG[0]['source_document']
|
|
rows=profile_units({'title':'Adrián Ondra','path':path,'content':'# Bakalársky projekt 2027\nStretnutie 5.11.2026\nStav:\n- Implementovaný nevyžiadaný detail.'},include_records=True)
|
|
p=build_question_plan('Odlíš rok začiatku štúdia Adriána Ondru, rok práce a najnovší dátum stretnutia.',CATALOG)
|
|
result=answer_academic(p.spec.question,{'index_version':{'version':'v'},'version':'c','academic_schema':1,'units':rows},engine='rag',mode='meeting',plan=p)
|
|
assert '2026-11-05' in result['answer'] and '2027' in result['answer']
|
|
assert 'nevyžiadaný detail' not in result['answer']
|