"""Exact/both/negated work selection and proof of actual use, not future tasks.""" import json import sqlite3 import pytest from tools.answer_evaluation.rag_preparation import prepare_named_content from tools.answer_evaluation.rag_evidence import rag_evidence_context,project_number,content_sections from tools.answer_evaluation.verified_rag import generate_verified_rag from tools.answer_evaluation.content_selection import facets_for from tools.retrieval.work_selector import parse_work_selector from tools.retrieval.question_constraints import requested_years,unsupported_constraint from tools.answers import answer_delivery from tools.rag.llm_context import compact_rag_payload @pytest.fixture def projects(tmp_path): db=tmp_path/'index.sqlite';path='pages/students/2016/eva_nova/README.md' with sqlite3.connect(db) as c: c.execute('CREATE TABLE documents(path,title,author,published)') c.execute('CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)') c.execute('INSERT INTO documents VALUES(?,?,?,?)',(path,'Eva Nová','Autor dokumentu',1)) for i,(label,text) in enumerate([ ('Diplomový projekt 2020','Hotové:\n- Funguje databáza SQLite.\nÚlohy na ďalšie stretnutie:\n- Naštudujte si ako sa používa Redis.'), ('Diplomový projekt 2 2020','Stav:\n- Implementovaný parser pomocou Python.\nÚlohy na ďalšie stretnutie:\n- urobte návrh metódy PageRank.\nDo ďalšieho stretnutia:\n- Naštudujte si ako sa používa Promise.')]): c.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path,text,json.dumps([['Eva Nová',label]]),1,i)) return db def prepare(db,q): raw={'status':'ok','reason':None,'sources':[],'context':'','complete':False,'truncated':True,'truncation_reasons':['result_limit']} return compact_rag_payload(prepare_named_content(db,q,raw)) def select(messages): context=json.loads(messages[1]['content'])['context'] a={'status':'ok','claims':[],'source_urls':[], 'evidence_ids':[u['evidence_id'] for u in context.get('content_candidates',[])]} return {'choices':[{'message':{'content':json.dumps(a)}}]} @pytest.mark.parametrize('phrase',[ 'pri práci "Diplomový projekt 2020" (nie Diplomový projekt 2)', 'pri nečíslovanom diplomovom projekte v roku 2020', 'pri diplomovom projekte bez čísla v roku 2020', 'pri diplomovom projekte v roku 2020 okrem projektu 2', ]) def test_exact_unnumbered_and_excluded_number_select_the_actual_work(projects,phrase): q='Aké metódy používala Eva Nová '+phrase+'?' assert not unsupported_constraint(q) and requested_years(q)==[2020] p=prepare(projects,q);assert p['status']=='ok' assert [s['work_label'] for s in p['work_scopes']]==['Diplomový projekt 2020'] r=generate_verified_rag(q,p,select) assert r['accepted_for_delivery'] and 'SQLite' in r['answer'] assert all(x not in r['answer'] for x in ['Python','PageRank','Promise','Redis']) def test_numbered_work_keeps_actual_methods_and_rejects_future_task_evidence(projects): q='Aké metódy používala Eva Nová pri diplomovom projekte 2 v roku 2020?' p=prepare(projects,q);c=rag_evidence_context(q,p) assert all(u['phase'] not in {'planned','assignment'} for u in c['content_candidates']) r=generate_verified_rag(q,p,select) assert r['accepted_for_delivery'] and 'Python' in r['answer'] assert 'PageRank' not in r['answer'] and 'Promise' not in r['answer'] and 'SQLite' not in r['answer'] bad={'status':'ok','claims':[],'source_urls':[s['source_url'] for s in c['sources']], 'evidence_quotes':[{'source_id':c['sources'][0]['source_id'],'quote':'- urobte návrh metódy PageRank.'}]} r=generate_verified_rag(q,p,lambda _:{'choices':[{'message':{'content':json.dumps(bad)}}]}) assert not r['accepted_for_delivery'] and 'quote_outside_delivered_candidates' in r['attempts'][-1]['issues'] @pytest.mark.parametrize('word',['oboch','obidvoch','všetkých']) def test_each_requested_work_has_its_own_methods_and_heading(projects,word): q=f'Aké metódy používala Eva Nová pri {word} diplomových projektoch v roku 2020?' p=prepare(projects,q);c=rag_evidence_context(q,p) assert c['requirements']['separate_works'] and len(p['work_scopes'])==2 r=generate_verified_rag(q,p,select) assert r['accepted_for_delivery'] and r['answer'].count('**Eva Nová — Diplomový projekt')==2 first,second=r['answer'].split('**Eva Nová — Diplomový projekt 2 2020') assert 'SQLite' in first and 'Python' not in first and 'Python' in second assert 'PageRank' not in r['answer'] and 'Promise' not in r['answer'] def test_omitting_a_second_work_with_evidence_is_rejected_even_for_the_same_person_and_year(projects): q='Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?' p=prepare(projects,q) def incomplete(messages): c=json.loads(messages[1]['content'])['context'] a={'status':'ok','claims':[],'source_urls':[],'evidence_ids':[u['evidence_id'] for u in c['content_candidates'] if u['work_label']=='Diplomový projekt 2020']} return {'choices':[{'message':{'content':json.dumps(a)}}]} r=generate_verified_rag(q,p,incomplete) assert not r['accepted_for_delivery'] and 'missing_methods_evidence' in r['attempts'][-1]['issues'] def test_missing_methods_are_reported_for_the_right_work_without_stealing_another_works_evidence(projects): with sqlite3.connect(projects) as c: c.execute("UPDATE chunks SET text='Úlohy na ďalšie stretnutie:\n- Použite metódu BERT.' WHERE chunk_index=1") q='Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?' r=generate_verified_rag(q,prepare(projects,q),select) assert r['accepted_for_delivery'] and r['answer_coverage']['status']=='partial' assert 'SQLite' in r['answer'] and 'BERT' not in r['answer'] assert 'Diplomový projekt 2 2020' in r['answer'] and 'skutočne použitých metódach' in r['answer'] def test_plans_remain_available_as_plans_with_natural_verbs(projects): q='Aké plánované úlohy mala Eva Nová pri oboch diplomových projektoch v roku 2020?' r=generate_verified_rag(q,prepare(projects,q),select) assert r['accepted_for_delivery'] assert 'Urobiť návrh metódy PageRank.' in r['answer'] and 'Naštudovať si ako sa používa Promise.' in r['answer'] assert 'urobte' not in r['answer'] and 'Naštudujte' not in r['answer'] def test_unknown_requested_work_is_not_silently_dropped(projects): q='Porovnaj metódy Evy Novej pri "Diplomový projekt 2020" a "Diplomový projekt 3 2020".' p=prepare(projects,q) assert p['status']=='ambiguous' and p['reason']=='unresolved_work_selection' def test_short_followup_needs_context_instead_of_global_retrieval(monkeypatch): def forbidden(*a,**kw):pytest.fail('A bare follow-up must not search or call an LLM') monkeypatch.setattr(answer_delivery,'build_rag_context',forbidden) r=answer_delivery.deliver_answer('daj obe',engine='rag',complete=forbidden) assert r['status']=='ambiguous' and r['reason']=='missing_followup_context' def test_negative_selector_is_not_a_positive_number_and_literal_title_years_are_not_filters(): q='Aké metódy používal Ján Holp pri "Diplomový projekt 2020" (nie Diplomový projekt 2)?' selector=parse_work_selector(q) assert project_number(q) is None and selector.excluded_numbers==(2,) and requested_years(q)==[2020] assert selector.matches('Diplomový projekt 2020') and not selector.matches('Diplomový projekt 2 2020') assert requested_years('Kto má názov "Projekt 2 o svete 2099"?')==[] assert not parse_work_selector('Kto má názov "Projekt 2 o svete 2099"?').specific assert unsupported_constraint('Aké metódy používal Ján Holp, nie v roku 2020?')=='exclusion' def test_unimplemented_method_is_not_proof_of_actual_use(): assert 'methods' not in facets_for('Metóda BERT ešte nie je implementovaná.','progress', 'Aké metódy skutočne použila Eva Nová?') def test_scalar_question_also_preserves_exact_work_and_exclusion(projects): q='Aký názov má "Diplomový projekt 2020" Evy Novej (nie Diplomový projekt 2)?' p=prepare(projects,q) assert len(p['work_scopes'])==1 and p['work_scopes'][0]['work_label']=='Diplomový projekt 2020' assert 'Python' not in p['context'] and 'Promise' not in p['context'] @pytest.mark.parametrize('heading',['Úlohy na ďalšie stretnutie:','Do ďalšieho stretnutia:','Do ďalšiehio stretnutia:']) def test_source_task_headings_cannot_inherit_a_completed_state(heading): sections=content_sections('Stav:\n- Implementovaný parser.\n'+heading+'\n- urobte návrh metódy PageRank.') assert [s['kind'] for s in sections]==['progress','task'] assert sections[-1]['phase']=='planned'