109 lines
4.5 KiB
Python
109 lines
4.5 KiB
Python
"""Explicit opt-in acceptance of real profiles and configured model API.
|
|
|
|
This is DEV/regression coverage, not an independent estimate of accuracy.
|
|
RUN_KNOWLEDGE_LIVE=1 is required; the normal suite never sends wiki texts out.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CASES = json.loads((ROOT / "tools/knowledge/questions.json").read_text())["questions"]
|
|
pytestmark = pytest.mark.skipif(
|
|
os.getenv("RUN_KNOWLEDGE_LIVE") != "1", reason="Explicit permission and model API required"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("engine", ["rag", "graphrag"])
|
|
@pytest.mark.parametrize("case", CASES, ids=lambda c: c["id"])
|
|
def test_real_profile_answer_preserves_evidence(case, engine, monkeypatch):
|
|
if (
|
|
case["id"] == "content-17"
|
|
and engine != "rag"
|
|
or case["id"] == "content-18"
|
|
and engine != "graphrag"
|
|
):
|
|
pytest.skip("Metadata regression belongs to its reference engine")
|
|
from dotenv import dotenv_values
|
|
|
|
allowed = {
|
|
"OPENWEBUI_API_KEY",
|
|
"ANSWER_PROVIDER",
|
|
"ANSWER_MODEL",
|
|
"ANSWER_LLM_URL",
|
|
"ANSWER_API_KEY",
|
|
"ANTHROPIC_API_KEY",
|
|
"NEO4J_URI",
|
|
"NEO4J_USER",
|
|
"NEO4J_PASSWORD",
|
|
"NEO4J_DATABASE",
|
|
}
|
|
for key, value in dotenv_values(ROOT / ".env").items():
|
|
if key in allowed and value is not None:
|
|
monkeypatch.setenv(key, value)
|
|
from tools.answers.answer_delivery import deliver_answer
|
|
from tools.answers.context_policy import normalized
|
|
|
|
result = deliver_answer(case["query"], engine=engine, limit=10)
|
|
assert result["accepted_for_delivery"] and result["status"] != "error", result.get("reason")
|
|
if "expected_status" in case:
|
|
assert result["status"] == case["expected_status"]
|
|
for text in case.get("answer_contains", []):
|
|
assert text in result["answer"]
|
|
for text in case.get("answer_excludes", []):
|
|
assert text not in result["answer"]
|
|
evidence = result.get("evidence", [])
|
|
for row in evidence:
|
|
source = ROOT.parent / "zpwiki" / row["source_path"]
|
|
assert row["text"] in source.read_text(), row["id"]
|
|
assert row["source_url"] in result["source_urls"]
|
|
if case.get("expected_person"):
|
|
assert any(row["person"] == case["expected_person"] for row in evidence), result["answer"]
|
|
if case.get("expected_date"):
|
|
assert any(
|
|
row["date"] == case["expected_date"] and case["evidence_contains"] in row["text"]
|
|
for row in evidence
|
|
)
|
|
if case.get("missing_results"):
|
|
assert "nedokladá" in result["answer"]
|
|
assert not any(row["phase"] == "result" and row["person"] == "Ján Pták" for row in evidence)
|
|
if case.get("requires_both_sides"):
|
|
people = {r["person"] for r in evidence}
|
|
assert "Ján Pták" in people and people & set(case["expected_any_related_people"])
|
|
# A bounded discovery query does not prescribe one ranking. Require
|
|
# source proof of the research topic, not only a shared UI or webhook.
|
|
candidate_text = " ".join(r["text"] for r in evidence if r["person"] != "Ján Pták")
|
|
assert re.search(
|
|
r"\brag\b|\b(?:graph|light)rag\b|\bznalostn\w* graf\w*\b|"
|
|
r"\bgrafov\w* vyhladav\w*\b|\bknowledge graph\b",
|
|
normalized(candidate_text),
|
|
)
|
|
if case.get("must_disclaim_current_activity"):
|
|
assert "nepotvrdzuje" in result["answer"] and "aktívne" in result["answer"]
|
|
if case.get("requires_explicit_meeting_date"):
|
|
assert evidence and all(
|
|
row["date"] and "2020-01-01" <= row["date"] <= "2021-12-31" for row in evidence
|
|
)
|
|
if case.get("requires_recognized_name_notice"):
|
|
assert "Rozpoznané meno" in result["answer"]
|
|
if case["id"] == "content-07" and result["status"] == "ok":
|
|
# Literal titles of the same work are supporting context, not claims
|
|
# that the title itself proves annotation was actually performed.
|
|
primary = [r for r in evidence if r.get("evidence_role") != "work_context"]
|
|
assert primary and all(
|
|
"anot" in normalized(row["text"])
|
|
and ("ner" in normalized(row["text"]).split() or "entit" in normalized(row["text"]))
|
|
for row in primary
|
|
)
|
|
if case["id"] == "content-17":
|
|
assert (
|
|
"Systém získavania informácií v slovenskom jazyku" in result["answer"]
|
|
and "2021" in result["answer"]
|
|
)
|
|
if case["id"] == "content-18":
|
|
assert "NLP" in result["answer"] and "RAG" in result["answer"]
|