72 lines
4.2 KiB
Python
72 lines
4.2 KiB
Python
"""Explicit opt-in acceptance of real profiles and configured model API.
|
|
|
|
This is DEV/regression coverage, not an independent estimate of accuracy.
|
|
RUN_KNOWLEDGE_LIVE=1 is required; the normal suite never sends wiki texts out.
|
|
"""
|
|
import json
|
|
import re
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
CASES=json.loads((ROOT/'tools/knowledge/questions.json').read_text())['questions']
|
|
pytestmark=pytest.mark.skipif(os.getenv('RUN_KNOWLEDGE_LIVE')!='1',reason='Explicit permission and model API required')
|
|
|
|
|
|
@pytest.mark.parametrize('engine',['rag','graphrag'])
|
|
@pytest.mark.parametrize('case',CASES,ids=lambda c:c['id'])
|
|
def test_real_profile_answer_preserves_evidence(case,engine,monkeypatch):
|
|
if case['id']=='content-17' and engine!='rag' or case['id']=='content-18' and engine!='graphrag':
|
|
pytest.skip('Metadata regression belongs to its reference engine')
|
|
from dotenv import dotenv_values
|
|
allowed={'OPENWEBUI_API_KEY','ANSWER_PROVIDER','ANSWER_MODEL','ANSWER_LLM_URL','ANSWER_API_KEY',
|
|
'ANTHROPIC_API_KEY','NEO4J_URI','NEO4J_USER','NEO4J_PASSWORD','NEO4J_DATABASE'}
|
|
for key,value in dotenv_values(ROOT/'.env').items():
|
|
if key in allowed and value is not None:monkeypatch.setenv(key,value)
|
|
from tools.answers.answer_delivery import deliver_answer
|
|
from tools.answer_evaluation.context_policy import normalized
|
|
result=deliver_answer(case['query'],engine=engine,limit=10)
|
|
assert result['accepted_for_delivery'] and result['status']!='error',result.get('reason')
|
|
if 'expected_status' in case:assert result['status']==case['expected_status']
|
|
for text in case.get('answer_contains',[]):assert text in result['answer']
|
|
for text in case.get('answer_excludes',[]):assert text not in result['answer']
|
|
evidence=result.get('evidence',[])
|
|
for row in evidence:
|
|
source=ROOT.parent/'zpwiki'/row['source_path']
|
|
assert row['text'] in source.read_text(),row['id']
|
|
assert row['source_url'] in result['source_urls']
|
|
if case.get('expected_person'):
|
|
assert any(row['person']==case['expected_person'] for row in evidence),result['answer']
|
|
if case.get('expected_date'):
|
|
assert any(row['date']==case['expected_date'] and case['evidence_contains'] in row['text']
|
|
for row in evidence)
|
|
if case.get('missing_results'):
|
|
assert 'nedokladá' in result['answer']
|
|
assert not any(row['phase']=='result' and row['person']=='Ján Pták' for row in evidence)
|
|
if case.get('requires_both_sides'):
|
|
people={r['person'] for r in evidence}
|
|
assert 'Ján Pták' in people and people & set(case['expected_any_related_people'])
|
|
# A bounded discovery query does not prescribe one ranking. Require
|
|
# source proof of the research topic, not only a shared UI or webhook.
|
|
candidate_text=' '.join(r['text'] for r in evidence if r['person']!='Ján Pták')
|
|
assert re.search(r'\brag\b|\b(?:graph|light)rag\b|\bznalostn\w* graf\w*\b|'
|
|
r'\bgrafov\w* vyhladav\w*\b|\bknowledge graph\b',normalized(candidate_text))
|
|
if case.get('must_disclaim_current_activity'):
|
|
assert 'nepotvrdzuje' in result['answer'] and 'aktívne' in result['answer']
|
|
if case.get('requires_explicit_meeting_date'):
|
|
assert evidence and all(row['date'] and '2020-01-01'<=row['date']<='2021-12-31' for row in evidence)
|
|
if case.get('requires_recognized_name_notice'):
|
|
assert 'Rozpoznané meno' in result['answer']
|
|
if case['id']=='content-07' and result['status']=='ok':
|
|
# Literal titles of the same work are supporting context, not claims
|
|
# that the title itself proves annotation was actually performed.
|
|
primary=[r for r in evidence if r.get('evidence_role')!='work_context']
|
|
assert primary and all('anot' in normalized(row['text']) and
|
|
('ner' in normalized(row['text']).split() or 'entit' in normalized(row['text'])) for row in primary)
|
|
if case['id']=='content-17':
|
|
assert 'Systém získavania informácií v slovenskom jazyku' in result['answer'] and '2021' in result['answer']
|
|
if case['id']=='content-18':
|
|
assert 'NLP' in result['answer'] and 'RAG' in result['answer']
|