"""Explicit opt-in acceptance of real profiles and configured model API. This is DEV/regression coverage, not an independent estimate of accuracy. RUN_KNOWLEDGE_LIVE=1 is required; the normal suite never sends wiki texts out. """ import json import re import os from pathlib import Path import pytest ROOT=Path(__file__).resolve().parents[1] CASES=json.loads((ROOT/'tools/knowledge/questions.json').read_text())['questions'] pytestmark=pytest.mark.skipif(os.getenv('RUN_KNOWLEDGE_LIVE')!='1',reason='Explicit permission and model API required') @pytest.mark.parametrize('engine',['rag','graphrag']) @pytest.mark.parametrize('case',CASES,ids=lambda c:c['id']) def test_real_profile_answer_preserves_evidence(case,engine,monkeypatch): if case['id']=='content-17' and engine!='rag' or case['id']=='content-18' and engine!='graphrag': pytest.skip('Metadata regression belongs to its reference engine') from dotenv import dotenv_values allowed={'OPENWEBUI_API_KEY','ANSWER_PROVIDER','ANSWER_MODEL','ANSWER_LLM_URL','ANSWER_API_KEY', 'ANTHROPIC_API_KEY','NEO4J_URI','NEO4J_USER','NEO4J_PASSWORD','NEO4J_DATABASE'} for key,value in dotenv_values(ROOT/'.env').items(): if key in allowed and value is not None:monkeypatch.setenv(key,value) from tools.answers.answer_delivery import deliver_answer from tools.answer_evaluation.context_policy import normalized result=deliver_answer(case['query'],engine=engine,limit=10) assert result['accepted_for_delivery'] and result['status']!='error',result.get('reason') if 'expected_status' in case:assert result['status']==case['expected_status'] for text in case.get('answer_contains',[]):assert text in result['answer'] for text in case.get('answer_excludes',[]):assert text not in result['answer'] evidence=result.get('evidence',[]) for row in evidence: source=ROOT.parent/'zpwiki'/row['source_path'] assert row['text'] in source.read_text(),row['id'] assert row['source_url'] in result['source_urls'] if case.get('expected_person'): assert any(row['person']==case['expected_person'] for row in evidence),result['answer'] if case.get('expected_date'): assert any(row['date']==case['expected_date'] and case['evidence_contains'] in row['text'] for row in evidence) if case.get('missing_results'): assert 'nedokladá' in result['answer'] assert not any(row['phase']=='result' and row['person']=='Ján Pták' for row in evidence) if case.get('requires_both_sides'): people={r['person'] for r in evidence} assert 'Ján Pták' in people and people & set(case['expected_any_related_people']) # A bounded discovery query does not prescribe one ranking. Require # source proof of the research topic, not only a shared UI or webhook. candidate_text=' '.join(r['text'] for r in evidence if r['person']!='Ján Pták') assert re.search(r'\brag\b|\b(?:graph|light)rag\b|\bznalostn\w* graf\w*\b|' r'\bgrafov\w* vyhladav\w*\b|\bknowledge graph\b',normalized(candidate_text)) if case.get('must_disclaim_current_activity'): assert 'nepotvrdzuje' in result['answer'] and 'aktívne' in result['answer'] if case.get('requires_explicit_meeting_date'): assert evidence and all(row['date'] and '2020-01-01'<=row['date']<='2021-12-31' for row in evidence) if case.get('requires_recognized_name_notice'): assert 'Rozpoznané meno' in result['answer'] if case['id']=='content-07' and result['status']=='ok': # Literal titles of the same work are supporting context, not claims # that the title itself proves annotation was actually performed. primary=[r for r in evidence if r.get('evidence_role')!='work_context'] assert primary and all('anot' in normalized(row['text']) and ('ner' in normalized(row['text']).split() or 'entit' in normalized(row['text'])) for row in primary) if case['id']=='content-17': assert 'Systém získavania informácií v slovenskom jazyku' in result['answer'] and '2021' in result['answer'] if case['id']=='content-18': assert 'NLP' in result['answer'] and 'RAG' in result['answer']