153 lines
8.7 KiB
Python
153 lines
8.7 KiB
Python
"""Exact/both/negated work selection and proof of actual use, not future tasks."""
|
|
import json
|
|
import sqlite3
|
|
|
|
import pytest
|
|
|
|
from tools.answer_evaluation.rag_preparation import prepare_named_content
|
|
from tools.answer_evaluation.rag_evidence import rag_evidence_context,project_number,content_sections
|
|
from tools.answer_evaluation.verified_rag import generate_verified_rag
|
|
from tools.answer_evaluation.content_selection import facets_for
|
|
from tools.retrieval.work_selector import parse_work_selector
|
|
from tools.retrieval.question_constraints import requested_years,unsupported_constraint
|
|
from tools.answers import answer_delivery
|
|
from tools.rag.llm_context import compact_rag_payload
|
|
|
|
|
|
@pytest.fixture
|
|
def projects(tmp_path):
|
|
db=tmp_path/'index.sqlite';path='pages/students/2016/eva_nova/README.md'
|
|
with sqlite3.connect(db) as c:
|
|
c.execute('CREATE TABLE documents(path,title,author,published)')
|
|
c.execute('CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)')
|
|
c.execute('INSERT INTO documents VALUES(?,?,?,?)',(path,'Eva Nová','Autor dokumentu',1))
|
|
for i,(label,text) in enumerate([
|
|
('Diplomový projekt 2020','Hotové:\n- Funguje databáza SQLite.\nÚlohy na ďalšie stretnutie:\n- Naštudujte si ako sa používa Redis.'),
|
|
('Diplomový projekt 2 2020','Stav:\n- Implementovaný parser pomocou Python.\nÚlohy na ďalšie stretnutie:\n- urobte návrh metódy PageRank.\nDo ďalšieho stretnutia:\n- Naštudujte si ako sa používa Promise.')]):
|
|
c.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path,text,json.dumps([['Eva Nová',label]]),1,i))
|
|
return db
|
|
|
|
|
|
def prepare(db,q):
|
|
raw={'status':'ok','reason':None,'sources':[],'context':'','complete':False,'truncated':True,'truncation_reasons':['result_limit']}
|
|
return compact_rag_payload(prepare_named_content(db,q,raw))
|
|
|
|
|
|
def select(messages):
|
|
context=json.loads(messages[1]['content'])['context']
|
|
a={'status':'ok','claims':[],'source_urls':[],
|
|
'evidence_ids':[u['evidence_id'] for u in context.get('content_candidates',[])]}
|
|
return {'choices':[{'message':{'content':json.dumps(a)}}]}
|
|
|
|
|
|
@pytest.mark.parametrize('phrase',[
|
|
'pri práci "Diplomový projekt 2020" (nie Diplomový projekt 2)',
|
|
'pri nečíslovanom diplomovom projekte v roku 2020',
|
|
'pri diplomovom projekte bez čísla v roku 2020',
|
|
'pri diplomovom projekte v roku 2020 okrem projektu 2',
|
|
])
|
|
def test_exact_unnumbered_and_excluded_number_select_the_actual_work(projects,phrase):
|
|
q='Aké metódy používala Eva Nová '+phrase+'?'
|
|
assert not unsupported_constraint(q) and requested_years(q)==[2020]
|
|
p=prepare(projects,q);assert p['status']=='ok'
|
|
assert [s['work_label'] for s in p['work_scopes']]==['Diplomový projekt 2020']
|
|
r=generate_verified_rag(q,p,select)
|
|
assert r['accepted_for_delivery'] and 'SQLite' in r['answer']
|
|
assert all(x not in r['answer'] for x in ['Python','PageRank','Promise','Redis'])
|
|
|
|
|
|
def test_numbered_work_keeps_actual_methods_and_rejects_future_task_evidence(projects):
|
|
q='Aké metódy používala Eva Nová pri diplomovom projekte 2 v roku 2020?'
|
|
p=prepare(projects,q);c=rag_evidence_context(q,p)
|
|
assert all(u['phase'] not in {'planned','assignment'} for u in c['content_candidates'])
|
|
r=generate_verified_rag(q,p,select)
|
|
assert r['accepted_for_delivery'] and 'Python' in r['answer']
|
|
assert 'PageRank' not in r['answer'] and 'Promise' not in r['answer'] and 'SQLite' not in r['answer']
|
|
bad={'status':'ok','claims':[],'source_urls':[s['source_url'] for s in c['sources']],
|
|
'evidence_quotes':[{'source_id':c['sources'][0]['source_id'],'quote':'- urobte návrh metódy PageRank.'}]}
|
|
r=generate_verified_rag(q,p,lambda _:{'choices':[{'message':{'content':json.dumps(bad)}}]})
|
|
assert not r['accepted_for_delivery'] and 'quote_outside_delivered_candidates' in r['attempts'][-1]['issues']
|
|
|
|
|
|
@pytest.mark.parametrize('word',['oboch','obidvoch','všetkých'])
|
|
def test_each_requested_work_has_its_own_methods_and_heading(projects,word):
|
|
q=f'Aké metódy používala Eva Nová pri {word} diplomových projektoch v roku 2020?'
|
|
p=prepare(projects,q);c=rag_evidence_context(q,p)
|
|
assert c['requirements']['separate_works'] and len(p['work_scopes'])==2
|
|
r=generate_verified_rag(q,p,select)
|
|
assert r['accepted_for_delivery'] and r['answer'].count('**Eva Nová — Diplomový projekt')==2
|
|
first,second=r['answer'].split('**Eva Nová — Diplomový projekt 2 2020')
|
|
assert 'SQLite' in first and 'Python' not in first and 'Python' in second
|
|
assert 'PageRank' not in r['answer'] and 'Promise' not in r['answer']
|
|
|
|
|
|
def test_omitting_a_second_work_with_evidence_is_rejected_even_for_the_same_person_and_year(projects):
|
|
q='Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?'
|
|
p=prepare(projects,q)
|
|
def incomplete(messages):
|
|
c=json.loads(messages[1]['content'])['context']
|
|
a={'status':'ok','claims':[],'source_urls':[],'evidence_ids':[u['evidence_id'] for u in c['content_candidates'] if u['work_label']=='Diplomový projekt 2020']}
|
|
return {'choices':[{'message':{'content':json.dumps(a)}}]}
|
|
r=generate_verified_rag(q,p,incomplete)
|
|
assert not r['accepted_for_delivery'] and 'missing_methods_evidence' in r['attempts'][-1]['issues']
|
|
|
|
|
|
def test_missing_methods_are_reported_for_the_right_work_without_stealing_another_works_evidence(projects):
|
|
with sqlite3.connect(projects) as c:
|
|
c.execute("UPDATE chunks SET text='Úlohy na ďalšie stretnutie:\n- Použite metódu BERT.' WHERE chunk_index=1")
|
|
q='Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?'
|
|
r=generate_verified_rag(q,prepare(projects,q),select)
|
|
assert r['accepted_for_delivery'] and r['answer_coverage']['status']=='partial'
|
|
assert 'SQLite' in r['answer'] and 'BERT' not in r['answer']
|
|
assert 'Diplomový projekt 2 2020' in r['answer'] and 'skutočne použitých metódach' in r['answer']
|
|
|
|
|
|
def test_plans_remain_available_as_plans_with_natural_verbs(projects):
|
|
q='Aké plánované úlohy mala Eva Nová pri oboch diplomových projektoch v roku 2020?'
|
|
r=generate_verified_rag(q,prepare(projects,q),select)
|
|
assert r['accepted_for_delivery']
|
|
assert 'Urobiť návrh metódy PageRank.' in r['answer'] and 'Naštudovať si ako sa používa Promise.' in r['answer']
|
|
assert 'urobte' not in r['answer'] and 'Naštudujte' not in r['answer']
|
|
|
|
|
|
def test_unknown_requested_work_is_not_silently_dropped(projects):
|
|
q='Porovnaj metódy Evy Novej pri "Diplomový projekt 2020" a "Diplomový projekt 3 2020".'
|
|
p=prepare(projects,q)
|
|
assert p['status']=='ambiguous' and p['reason']=='unresolved_work_selection'
|
|
|
|
|
|
def test_short_followup_needs_context_instead_of_global_retrieval(monkeypatch):
|
|
def forbidden(*a,**kw):pytest.fail('A bare follow-up must not search or call an LLM')
|
|
monkeypatch.setattr(answer_delivery,'build_rag_context',forbidden)
|
|
r=answer_delivery.deliver_answer('daj obe',engine='rag',complete=forbidden)
|
|
assert r['status']=='ambiguous' and r['reason']=='missing_followup_context'
|
|
|
|
|
|
def test_negative_selector_is_not_a_positive_number_and_literal_title_years_are_not_filters():
|
|
q='Aké metódy používal Ján Holp pri "Diplomový projekt 2020" (nie Diplomový projekt 2)?'
|
|
selector=parse_work_selector(q)
|
|
assert project_number(q) is None and selector.excluded_numbers==(2,) and requested_years(q)==[2020]
|
|
assert selector.matches('Diplomový projekt 2020') and not selector.matches('Diplomový projekt 2 2020')
|
|
assert requested_years('Kto má názov "Projekt 2 o svete 2099"?')==[]
|
|
assert not parse_work_selector('Kto má názov "Projekt 2 o svete 2099"?').specific
|
|
assert unsupported_constraint('Aké metódy používal Ján Holp, nie v roku 2020?')=='exclusion'
|
|
|
|
|
|
def test_unimplemented_method_is_not_proof_of_actual_use():
|
|
assert 'methods' not in facets_for('Metóda BERT ešte nie je implementovaná.','progress',
|
|
'Aké metódy skutočne použila Eva Nová?')
|
|
|
|
|
|
def test_scalar_question_also_preserves_exact_work_and_exclusion(projects):
|
|
q='Aký názov má "Diplomový projekt 2020" Evy Novej (nie Diplomový projekt 2)?'
|
|
p=prepare(projects,q)
|
|
assert len(p['work_scopes'])==1 and p['work_scopes'][0]['work_label']=='Diplomový projekt 2020'
|
|
assert 'Python' not in p['context'] and 'Promise' not in p['context']
|
|
|
|
|
|
@pytest.mark.parametrize('heading',['Úlohy na ďalšie stretnutie:','Do ďalšieho stretnutia:','Do ďalšiehio stretnutia:'])
|
|
def test_source_task_headings_cannot_inherit_a_completed_state(heading):
|
|
sections=content_sections('Stav:\n- Implementovaný parser.\n'+heading+'\n- urobte návrh metódy PageRank.')
|
|
assert [s['kind'] for s in sections]==['progress','task']
|
|
assert sections[-1]['phase']=='planned'
|