dp-zp-agent/test/test_knowledge.py

413 lines
21 KiB
Python

"""Acceptance, adversarial extraction and source-bound temporal regressions."""
import copy
import json
from pathlib import Path
import pytest
from tools.knowledge.sources import profile_units, meeting_date
from tools.knowledge.extraction import extract_batch
from tools.knowledge.query import answer_from_index, intent, invalid_question, select_topic_evidence
def response(value):
return {'choices':[{'message':{'content':json.dumps(value,ensure_ascii=False)}}]}
def doc(name='Alena Nová', content=None):
return {'title':name, 'path':'pages/students/2020/alena/README.md',
'content':content or '''# Diplomová práca 2026
Zadanie:
1. Vytvorte systém RAG na vyhľadávanie dokumentov.
Stretnutie 5.2.2026
Stav:
- Implementovaný systém RAG s vyhľadávaním dokumentov.
Úlohy:
- Vyhodnoťte systém pomocou DeepEval.
Stretnutie 6.3.
Stav:
- Vyhodnotenie modelu zatiaľ nefunguje.
'''}
def fixture_index():
units=profile_units(doc())
second=profile_units({**doc(), 'title':'Boris Starý', 'path':'pages/students/2020/boris/README.md',
'content':'# Diplomová práca 2025\nStretnutie 3.4.2025\nStav:\n- Implementovaný systém RAG pre vzdelávanie.\n'})
for unit in [*units,*second]:
unit.update(concepts=['RAG'] if 'RAG' in unit['text'] else [], relations=[])
return {'units':[*units,*second], 'index_version':{'version':'fixture'}, 'version':'knowledge-fixture'}
def test_dates_are_explicit_and_invalid_dates_are_not_repaired():
assert meeting_date('Stretnutie 5.2.2026')['date']=='2026-02-05'
assert meeting_date('Virtuálne stretnutie 12.3.')=={
'date':None,'date_label':'12.3.','date_precision':'day_month'}
assert meeting_date('Stretnutie 31.2.2026')['date'] is None
assert meeting_date('Diplomová práca 2026') is None
assert meeting_date('Virtuálne stretnutie 13.11:2020:')['date']=='2020-11-13'
def test_source_units_keep_person_work_phase_and_date_together():
units=profile_units(doc())
assignment=next(u for u in units if 'Vytvorte' in u['text'])
done=next(u for u in units if 'Implementovaný' in u['text'])
planned=next(u for u in units if 'Vyhodnoťte' in u['text'])
partial=next(u for u in units if 'nefunguje' in u['text'])
assert assignment['phase']=='assignment' and assignment['date'] is None
assert done['phase']=='result' and done['date']=='2026-02-05'
assert planned['phase']=='planned' and planned['date']=='2026-02-05'
assert partial['phase']=='progress' and partial['date'] is None
assert partial['date_precision']=='day_month' and partial['work_year']==2026
assert len({u['id'] for u in units})==len(units)
def test_same_quote_in_different_works_is_not_a_single_evidence():
units=profile_units(doc(content='# Diplomový projekt 2025\nStav:\n- Funguje systém RAG.\n'
'# Diplomová práca 2026\nStav:\n- Funguje systém RAG.\n'))
assert len(units)==2 and units[0]['id']!=units[1]['id']
assert {u['work_year'] for u in units}=={2025,2026}
def test_sibling_nonacademic_heading_does_not_inherit_previous_work():
units=profile_units(doc(content='# Diplomová práca 2026\nStav:\n- Funguje systém RAG.\n'
'# Príprava na projekt\n- Iná poznámka mimo diplomovej práce.\n'))
assert units[-1]['work_label'] is None and units[-1]['phase']=='unknown'
def test_reference_urls_and_topic_examples_are_not_promoted_to_results_or_goals():
units=profile_units(doc(content='# Diplomová práca 2026\nZadanie:\n- Vytvorte systém RAG.\n'
'Napr.\n- téma pre základné školy.\nStav:\n- Implementovaný systém RAG.\n'
'Zdroje dokumentov:\n- https://example.test/research\n'))
assert not any('https://' in u['text'] for u in units)
example=next(u for u in units if 'téma' in u['text'])
assert example['phase']=='unknown'
def test_partial_implementation_is_not_a_completed_result_even_under_urobene():
units=profile_units(doc(content='# Diplomová práca 2026\nUrobené:\n- rozrobené web rozhranie\n'))
assert units[0]['phase']=='progress' and units[0]['kind']=='progress'
def test_explicit_outputs_and_checked_tasks_remain_distinguishable():
units=profile_units(doc(content='# Bakalársky projekt 2020\nVýstupy:\n- Aplikácia vie rozpoznávať entity.\n'
'Úlohy:\n- [x] Naštudujte korpusy pre NER.\n- [ ] Vyhodnoťte model NER.\n'))
assert units[0]['phase']=='result'
assert units[1]['phase']=='result' and units[1]['completion_basis']=='checked_task'
assert units[2]['phase']=='planned'
def test_model_intent_cannot_rewrite_questions_or_invent_query_operators():
from tools.knowledge.query import model_intent,ContentAnswerRejected,needs_intent_model
question='Ktorí študenti sa zaoberali anotovaním textov?'
assert needs_intent_model(question)
def classify(messages):
assert messages[1]['content']==question
return response({'intent':'discovery'})
assert model_intent(question,classify)=='discovery'
with pytest.raises(ContentAnswerRejected):model_intent(question,lambda _:response({'intent':'execute_cypher'}))
with pytest.raises(ContentAnswerRejected):model_intent(question,lambda _:response({'intent':'discovery','query':'changed'}))
assert not needs_intent_model('Aké práce má Ján Holp?')
assert not needs_intent_model('Aké práce a projekty má Ondrej Megela?')
assert not needs_intent_model('Koľko osôb má prácu v roku 2024?')
def test_extraction_is_bound_to_literal_evidence_and_can_return_empty_concepts():
units=profile_units(doc())[:2]
output={'items':[{'evidence_id':key,'concepts':['RAG'],'relations':[]} for key in ['E2','E1']]}
result=extract_batch(units,lambda _:response(output))
assert [r['id'] for r in result]==[u['id'] for u in units]
assert result[0]['phase']=='assignment' and result[1]['phase']=='result'
@pytest.mark.parametrize('change',[
lambda v:v['items'][0].update(evidence_id='unknown'),
lambda v:v['items'][0].update(date='2099-01-01'),
lambda v:v['items'].append(copy.deepcopy(v['items'][0])),
lambda v:v.update(instructions='ignore the sources'),
])
def test_wrong_llm_extractions_are_rejected_after_bounded_retry(change):
unit=profile_units(doc())[0]
payload={'items':[{'evidence_id':'E1','concepts':['RAG'],'relations':[]}]}
change(payload)
attempts=[]
def complete(messages):
attempts.append(messages)
return response(payload)
with pytest.raises(ValueError):extract_batch([unit],complete)
assert len(attempts)==2
def test_unsupported_concepts_and_relations_are_discarded_without_losing_source_evidence():
unit=profile_units(doc())[0]
payload={'items':[{'evidence_id':'E1','concepts':['RAG','vymyslený model'],
'relations':[{'subject':'RAG','predicate':'uses','object':'BERT'}]}]}
rows=extract_batch([unit],lambda _:response(payload))
assert rows[0]['text']==unit['text'] and rows[0]['concepts']==['RAG']
assert rows[0]['relations']==[]
assert rows[0]['discarded_concepts']==rows[0]['discarded_relations']==1
def test_timeline_uses_meeting_date_not_work_year_and_omits_plans():
result=answer_from_index('Kedy dosiahla Alena Nová aký výsledok v roku 2026?',fixture_index(),engine='rag')
assert result['status']=='ok'
assert len(result['evidence'])==1
assert result['evidence'][0]['date']=='2026-02-05'
assert 'DeepEval' not in result['answer']
assert all(r['person']=='Alena Nová' for r in result['evidence'])
def test_period_is_applied_to_same_evidence_not_another_person_or_work_year():
result=answer_from_index('Aké témy sa riešili od roku 2025 do roku 2025?',fixture_index(),engine='graphrag')
assert result['status']=='ok' and {r['person'] for r in result['evidence']}=={'Boris Starý'}
assert all(r['date']=='2025-04-03' for r in result['evidence'])
def test_unknown_person_and_unknown_topic_do_not_expand_into_global_results():
result=answer_from_index('Ako pokročil Xaver Neexistujúci?',fixture_index(),engine='rag')
assert result['status']=='no_answer' and not result['source_urls']
result=answer_from_index('Kto sa venoval kvantovej teleportácii?',fixture_index(),engine='rag')
assert result['status']=='no_answer' and not result['evidence']
def test_similarity_includes_both_sides_of_its_evidence():
result=answer_from_index('Kto pracoval na podobnej téme ako Alena Nová?',fixture_index(),engine='graphrag')
assert result['status']=='ok'
assert {r['person'] for r in result['evidence']}=={'Alena Nová','Boris Starý'}
assert len(result['source_urls'])==2
assert 'nie dôkaz spolupráce' in result['answer']
def test_shared_interface_or_webhook_alone_does_not_prove_topic_similarity():
index=fixture_index()
for row in index['units']:row['concepts']=['webhook','rozhranie']
result=answer_from_index('Kto pracoval na podobnej téme ako Alena Nová?',index,engine='graphrag')
assert result['status']=='no_answer' and not result['evidence']
def test_shared_graph_concepts_are_relevance_checked_before_delivery():
selected=[]
def choose(messages):
payload=json.loads(messages[1]['content'])
assert payload['reference_evidence'] and payload['evidence']
selected.append(True)
return response({'evidence_ids':['R1']})
result=answer_from_index('Kto pracoval na podobnej téme ako Alena Nová?',fixture_index(),
engine='graphrag',complete=choose)
assert selected and result['status']=='ok'
assert {u['person'] for u in result['evidence']}=={'Alena Nová','Boris Starý'}
def test_year_constraint_is_not_dropped_while_resolving_a_typo():
result=answer_from_index('Ako pokročila Alena Novvá pri diplomovej práci v roku 1999?',fixture_index(),engine='rag')
assert result['status']=='no_answer' and not result['evidence']
def test_metadata_filters_are_never_silently_dropped():
result=answer_from_index('Kto pracoval na podobnej téme ako Alena Nová s tagom NLP?',fixture_index(),engine='rag')
assert result['status']=='ambiguous' and result['reason']=='unsupported_content_join'
def test_topic_model_cannot_add_unprovided_evidence():
candidates=fixture_index()['units'][:1]
with pytest.raises(ValueError):
select_topic_evidence('Kto vytvoril chatbota?',candidates,lambda _:response({'evidence_ids':['invented']}))
def test_selection_accepts_fenced_json_and_empty_selection_but_never_unbound_lists():
candidates=fixture_index()['units'][:1]
fenced={'choices':[{'message':{'content':'```json\n{"evidence_ids":["R1"]}\n```'}}]}
assert select_topic_evidence('Kto pracoval na RAG?',candidates,lambda _:fenced)==candidates
assert select_topic_evidence('Kto pracoval na teleportácii?',candidates,lambda _:response([]))==[]
with pytest.raises(ValueError):
select_topic_evidence('Kto pracoval na RAG?',candidates,lambda _:response(['R1']))
def test_work_context_cannot_join_another_person_or_another_work():
from tools.knowledge.sources import with_work_context
index=fixture_index()
topic={**index['units'][0],'id':'topic','kind':'topic','text':'Názov: Agent pre RAG.'}
unrelated={**topic,'id':'unrelated','work_label':'Diplomový projekt 2025','text':'Názov: Iná práca.'}
other={**topic,'id':'other','person':'Boris Starý','source_path':'other','text':'Názov: Cudzia práca.'}
rows=with_work_context([*index['units'],topic,unrelated,other])
unit=next(r for r in rows if r['id']==index['units'][1]['id'])
assert [r['id'] for r in unit['context_quotes']]==['topic']
def test_annotation_of_another_target_or_an_analogy_is_not_named_entity_annotation():
from tools.knowledge.query import activity_supported
question='Kto sa venoval anotácii pomenovaných entít?'
assert not activity_supported(question,{'text':'Vytvorené anotácie do databázy, cca 1700 jednotiek.'})
assert not activity_supported(question,{'text':'Anotácia urážlivých častí. Dáta anotujete podobne ako named entities.'})
assert not activity_supported(question,{'text':'Rozpoznávanie pomenovaných entít pomocou BERT.'})
assert activity_supported(question,{'text':'Rozbehaná slovenská anotačná schéma pre pomenované entity.'})
assert activity_supported(question,{'text':'Vytvorené anotácie do databázy.',
'context_quotes':[{'text':'Názov: Anotácia NER.'}]})
def test_explicit_dates_can_resolve_a_semester_without_inventing_boundaries():
result=answer_from_index('Aké témy sa riešili v zimnom semestri od 1.1.2026 do 1.3.2026?',
fixture_index(),engine='rag',mode='period')
assert result['status']=='ok' and all(r['date']=='2026-02-05' for r in result['evidence'])
@pytest.mark.parametrize('query',['CRZP','HDFS systémy','NLP projekty'])
def test_real_acronyms_are_not_keyboard_gibberish(query):
assert not invalid_question(query)
def test_discovery_selects_recorded_implementation_before_a_bare_topic_link():
index=fixture_index()
rows=[{**index['units'][0],'id':'link','phase':'unknown','kind':'context','text':'- [Chatbot](/topics/chatbot)'},
{**index['units'][0],'id':'done','phase':'result','kind':'result','text':'- Chatbot funguje pre agendu mesta.'}]
def select(messages):
supplied=json.loads(messages[1]['content'])['evidence']
assert supplied[0]['text']=='- Chatbot funguje pre agendu mesta.'
return response({'evidence_ids':['R1']})
result=answer_from_index('Kto vytvoril chatbota?',{**index,'units':rows},engine='rag',complete=select)
assert result['status']=='ok' and result['evidence'][0]['id']=='done'
@pytest.mark.parametrize('exception,reason',[
(__import__('tools.knowledge.query',fromlist=['ContentModelUnavailable']).ContentModelUnavailable,'model_unavailable'),
(__import__('tools.knowledge.query',fromlist=['ContentAnswerRejected']).ContentAnswerRejected,'answer_rejected'),
(RuntimeError,'content_index_unavailable'),
])
def test_content_failures_keep_model_rejection_and_index_errors_distinct(monkeypatch,exception,reason):
import tools.knowledge.query as query
from tools.answers.answer_delivery import deliver_answer
def failed(*a,**kw):raise exception('private diagnostic must not escape')
monkeypatch.setattr(query,'deliver_content',failed)
result=deliver_answer('Ako pokročil Ján Holp?')
assert result['reason']==reason and not result['accepted_for_delivery']
assert 'private diagnostic' not in result['answer']
@pytest.mark.parametrize('query',['xdzf','asdfgh','bcdfrt','x y z'])
def test_unintelligible_input_requests_clarification(query):
assert invalid_question(query)
def test_intent_does_not_intercept_existing_metadata_questions():
assert intent('Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?') is None
assert intent('Aké spoločné kľúčové slová majú Ján Pták a Ján Malinovský?') is None
def test_real_markdown_acceptance_sources_are_independent_of_extraction_parser():
from tools.rag.common import ZPWIKI_ROOT
expectations=[
('2016/jan_holp','funguje databáza Cassandra'),
('2018/ondrej_megela','Dotrenovany t5 model na generovanie otazok.'),
('2022/jan_malinovsky','Extrahovaný text pomocou pypdf a docx.'),
('2021/martin_sarissky','Problém s kvalitou vyhľadávania.'),
]
if not ZPWIKI_ROOT.is_dir():pytest.skip('Local wiki is not installed')
for path,quote in expectations:
raw=(ZPWIKI_ROOT/'pages/students'/path/'README.md').read_text()
assert quote in raw
def test_stale_extraction_is_not_used(tmp_path):
from tools.knowledge.store import load_index
folder=tmp_path/'knowledge';folder.mkdir()
(folder/'index.json').write_text(json.dumps({'index_version':{'version':'old'}}))
with pytest.raises(RuntimeError):load_index(tmp_path,{'version':'new'})
def test_shared_embedding_cache_can_be_reused_without_writes(tmp_path):
import numpy as np
from tools.rag.embedding_cache import EmbeddingCache
path=tmp_path/'cache.sqlite'
writable=EmbeddingCache(path)
writable.save({'fixture':np.array([1.,0.],dtype=np.float32)})
before=path.read_bytes()
readonly=EmbeddingCache(path,readonly=True)
assert readonly.load(['fixture'])['fixture'].tolist()==[1.,0.]
with pytest.raises(RuntimeError):readonly.save({'other':np.array([0.,1.])})
assert path.read_bytes()==before
def test_build_is_cached_and_source_changes_cannot_activate_it(tmp_path):
from tools.knowledge.build import build
from tools.sync.index_state import durable_json
from tools.sync.sync_indexes import source_files
import hashlib
wiki=tmp_path/'wiki';data=tmp_path/'data';data.mkdir()
path=wiki/'pages/students/2020/alena/README.md';path.parent.mkdir(parents=True)
path.write_text('---\ntitle: Alena Nová\npublished: true\n---\n'+doc()['content'])
version={'version':'fixture','source_sha256':hashlib.sha256(json.dumps(source_files(wiki),sort_keys=True).encode()).hexdigest()}
durable_json(data/'index_version.json',version)
calls=[]
def extract(messages):
payload=json.loads(messages[1]['content']);calls.append(1)
return response({'items':[{'evidence_id':r['evidence_id'],'concepts':[],'relations':[]} for r in payload]})
first=build(wiki,data,extract,{'model':'fixture'},graph=False,vectors=False)
assert first['profiles']==1 and first['llm_batches']==1
activated=(data/'knowledge/index.json').read_bytes()
second=build(wiki,data,extract,{'model':'fixture'},graph=False,vectors=False)
assert second['llm_batches']==0 and second['cached_batches']==1 and len(calls)==1
assert activated==(data/'knowledge/index.json').read_bytes()
path.write_text(path.read_text()+'\nNové podklady.\n')
with pytest.raises(RuntimeError):build(wiki,data,extract,{'model':'fixture'},graph=False,vectors=False)
assert activated==(data/'knowledge/index.json').read_bytes()
@pytest.mark.parametrize('enabled',[False,True])
def test_index_sync_refreshes_content_only_after_it_has_been_enabled(tmp_path,monkeypatch,enabled):
import tools.rag.rebuild_index as rebuild
import tools.sync.sync_indexes as sync
from tools.knowledge import build as knowledge
target=tmp_path/'data';target.mkdir()
if enabled:
(target/'knowledge').mkdir()
(target/'knowledge/index.json').write_text('{}')
calls=[];phases=[]
monkeypatch.setattr(sync,'synchronize_indexes',lambda *a,**kw:{'counts':{'documents':1}})
monkeypatch.setattr(knowledge,'refresh_existing',lambda *a:calls.append(a) or {'cached_batches':1})
result=rebuild.rebuild_index(wiki_root=tmp_path/'wiki',data_dir=target,progress=phases.append)
assert bool(calls)==enabled
assert ('knowledge' in result)==enabled
if enabled:assert phases[-1]=='extracting_knowledge'
def test_explicit_work_year_is_not_misused_as_an_event_date():
index=fixture_index()
row=next(r for r in index['units'] if r['phase']=='result' and r['person']=='Alena Nová')
row['date']='2025-12-01'
result=answer_from_index('Kedy dosiahla Alena Nová výsledok pri diplomovej práci z roku 2026?',index,engine='rag')
assert result['status']=='ok' and result['evidence'][0]['date']=='2025-12-01'
def test_reversed_time_interval_requires_clarification():
result=answer_from_index('Aké témy sa riešili od roku 2026 do roku 2020?',fixture_index(),engine='rag')
assert result['status']=='ambiguous' and result['reason']=='invalid_period'
@pytest.mark.skipif(__import__('os').getenv('RUN_NEO4J_TESTS')!='1',reason='Requires local Neo4j')
def test_live_content_graph_roundtrip_is_idempotent_and_separate(monkeypatch):
import uuid
from tools.knowledge import store
from tools.graphrag.graph_db import create_neo4j_driver,get_neo4j_settings
scope='knowledge-test-'+uuid.uuid4().hex
monkeypatch.setattr(store,'SCOPE',scope)
index=fixture_index()
try:
store.write_graph(index)
first=store.read_graph(index)
store.write_graph(index)
again=store.read_graph(index)
assert len(first)==len(again)==len(index['units'])
assert {r['id']:r['text'] for r in first}=={r['id']:r['text'] for r in index['units']}
assert any(r.get('date')=='2026-02-05' for r in again)
with pytest.raises(RuntimeError):store.read_graph({**index,'version':'wrong-version'})
broken=copy.deepcopy(index)
broken['units'][0]['source_path']={'not':'a Neo4j property'}
from neo4j.exceptions import Neo4jError
with pytest.raises(Neo4jError):store.write_graph(broken)
assert {r['id'] for r in store.read_graph(index)}=={r['id'] for r in index['units']}
finally:
settings=get_neo4j_settings()
with create_neo4j_driver(settings) as driver,driver.session(database=settings.database) as session:
session.run('MATCH (n {graph_scope:$scope}) DETACH DELETE n',scope=scope).consume()
session.run('MATCH (n:KnowledgeState {scope:$scope}) DELETE n',scope=scope).consume()