"""Explicit opt-in acceptance of real profiles and configured model API. This is DEV/regression coverage, not an independent estimate of accuracy. RUN_KNOWLEDGE_LIVE=1 is required; the normal suite never sends wiki texts out. """ import json import re import os from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[1] CASES = json.loads((ROOT / "tools/knowledge/questions.json").read_text())["questions"] pytestmark = pytest.mark.skipif( os.getenv("RUN_KNOWLEDGE_LIVE") != "1", reason="Explicit permission and model API required" ) @pytest.mark.parametrize("engine", ["rag", "graphrag"]) @pytest.mark.parametrize("case", CASES, ids=lambda c: c["id"]) def test_real_profile_answer_preserves_evidence(case, engine, monkeypatch): if ( case["id"] == "content-17" and engine != "rag" or case["id"] == "content-18" and engine != "graphrag" ): pytest.skip("Metadata regression belongs to its reference engine") from dotenv import dotenv_values allowed = { "OPENWEBUI_API_KEY", "ANSWER_PROVIDER", "ANSWER_MODEL", "ANSWER_LLM_URL", "ANSWER_API_KEY", "ANTHROPIC_API_KEY", "NEO4J_URI", "NEO4J_USER", "NEO4J_PASSWORD", "NEO4J_DATABASE", } for key, value in dotenv_values(ROOT / ".env").items(): if key in allowed and value is not None: monkeypatch.setenv(key, value) from tools.answers.answer_delivery import deliver_answer from tools.answers.context_policy import normalized result = deliver_answer(case["query"], engine=engine, limit=10) assert result["accepted_for_delivery"] and result["status"] != "error", result.get("reason") if "expected_status" in case: assert result["status"] == case["expected_status"] for text in case.get("answer_contains", []): assert text in result["answer"] for text in case.get("answer_excludes", []): assert text not in result["answer"] evidence = result.get("evidence", []) for row in evidence: source = ROOT.parent / "zpwiki" / row["source_path"] assert row["text"] in source.read_text(), row["id"] assert row["source_url"] in result["source_urls"] if case.get("expected_person"): assert any(row["person"] == case["expected_person"] for row in evidence), result["answer"] if case.get("expected_date"): assert any( row["date"] == case["expected_date"] and case["evidence_contains"] in row["text"] for row in evidence ) if case.get("missing_results"): assert "nedokladá" in result["answer"] assert not any(row["phase"] == "result" and row["person"] == "Ján Pták" for row in evidence) if case.get("requires_both_sides"): people = {r["person"] for r in evidence} assert "Ján Pták" in people and people & set(case["expected_any_related_people"]) # A bounded discovery query does not prescribe one ranking. Require # source proof of the research topic, not only a shared UI or webhook. candidate_text = " ".join(r["text"] for r in evidence if r["person"] != "Ján Pták") assert re.search( r"\brag\b|\b(?:graph|light)rag\b|\bznalostn\w* graf\w*\b|" r"\bgrafov\w* vyhladav\w*\b|\bknowledge graph\b", normalized(candidate_text), ) if case.get("must_disclaim_current_activity"): assert "nepotvrdzuje" in result["answer"] and "aktívne" in result["answer"] if case.get("requires_explicit_meeting_date"): assert evidence and all( row["date"] and "2020-01-01" <= row["date"] <= "2021-12-31" for row in evidence ) if case.get("requires_recognized_name_notice"): assert "Rozpoznané meno" in result["answer"] if case["id"] == "content-07" and result["status"] == "ok": # Literal titles of the same work are supporting context, not claims # that the title itself proves annotation was actually performed. primary = [r for r in evidence if r.get("evidence_role") != "work_context"] assert primary and all( "anot" in normalized(row["text"]) and ("ner" in normalized(row["text"]).split() or "entit" in normalized(row["text"])) for row in primary ) if case["id"] == "content-17": assert ( "Systém získavania informácií v slovenskom jazyku" in result["answer"] and "2021" in result["answer"] ) if case["id"] == "content-18": assert "NLP" in result["answer"] and "RAG" in result["answer"]