dp-zp-agent/test/test_rag_work_content.py

"""Content questions must get source passages, not a metadata-only substitute."""

import copy
import json

import pytest

from tools.answers.rag_evidence import rag_evidence_context
from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues
from tools.rag.llm_context import compact_rag_payload
from tools.rag.rag_utils import build_context_text

QUESTION = (
    "Čomu sa venuje diplomová práca Jána Holpa? Odpovedz podľa dostupných podkladov a uveď zdroj."
)
ASSIGNMENT = (
    "1. Vypracujte prehľad metód ohodnotenia dokumentov v systémoch získavania informácií.\n"
    "2. Implementujte systém pre získavanie informácií v slovenskom jazyku."
)
URL = "https://example.test/jan"


@pytest.fixture
def payload():
    sources = [
        {
            "source_id": "S1",
            "title": "Ján Holp",
            "author": "Daniel Hladek",
            "document_path": "pages/students/2016/jan_holp/README.md",
            "source_url": URL,
            "text": "Dokument: Ján Holp\nSekcia: Diplomová práca 2021\n"
            "Názov: Systém získavania informácií v slovenskom jazyku.\n"
            "Meno vedúceho: Ing. Daniel Hladek, PhD.\nZadanie diplomovej práce:\n"
            + ASSIGNMENT
            + "\n## Metódy\nVyhodnotenie využíva PageRank.\n"
            "## Bakalárska práca 2019\nNázov: Vyhľadávač na slovenskom internete.\n"
            "Práca implementuje prehľadávanie slovenského webu.\n"
            "## Diplomový projekt 2020\nNázov: Pokusy s databázou.\n"
            "Projekt využíva databázu Cassandra.",
        },
        {
            "source_id": "S2",
            "title": "Boris Nový",
            "document_path": "pages/students/2020/boris_novy/README.md",
            "source_url": "https://example.test/boris",
            "text": "Sekcia: Diplomová práca 2021\nNázov: Spracovanie obrázkov.\n"
            "Práca skúma klasifikáciu medicínskych obrázkov.",
        },
    ]
    return {
        "status": "ok",
        "reason": None,
        "sources": sources,
        "context": build_context_text(sources),
        "complete": False,
        "truncated": True,
        "truncation_reasons": ["result_limit"],
        "index_version": {"version": "fixture"},
    }


def description():
    return {
        "answer": "",
        "status": "ok",
        "claims": [],
        "count": None,
        "source_urls": [URL],
        "evidence_quotes": [{"source_id": "S1", "quote": ASSIGNMENT}],
    }


def metadata():
    return {
        "answer": "",
        "status": "ok",
        "claims": [
            {
                "person": "Ján Holp",
                "work_type": "diploma_thesis",
                "title": "Systém získavania informácií v slovenskom jazyku",
                "year": 2021,
            }
        ],
        "evidence_quotes": [],
        "source_urls": [URL],
    }


def completion(answer):
    return {"choices": [{"message": {"content": json.dumps(answer, ensure_ascii=False)}}]}


@pytest.mark.parametrize(
    "question",
    [
        QUESTION,
        "O čom je diplomová práca Jána Holpa?",
        "Čomu sa venovala diplomová práca Jána Holpa?",
        "Čo rieši diplomová práca Jána Holpa?",
        "Stručne opíš diplomovú prácu Jána Holpa.",
        "Zhrň obsah diplomovej práce Jána Holpa.",
        "Aké sú ciele diplomovej práce Jána Holpa?",
        "Aké je zadanie diplomovej práce Jána Holpa?",
        "Aké metódy používa diplomová práca Jána Holpa?",
    ],
)
def test_content_intent_is_not_a_title_or_year_lookup(payload, question):
    c = rag_evidence_context(question, payload)
    assert c["requirements"]["content"]
    assert c["requirements"]["response_mode"] == "quotes"
    assert not c["requirements"]["title"] and not c["requirements"]["year"]
    assert any(ASSIGNMENT in p["text"] for p in c["content_passages"])


def test_named_work_content_without_explicit_work_type_still_needs_text(payload):
    c = rag_evidence_context("Opíš obsah práce Jána Holpa.", payload)
    assert c["requirements"]["content"] and c["requirements"]["response_mode"] == "quotes"
    assert "missing_content_evidence" in rag_answer_issues(metadata(), c)


@pytest.mark.parametrize(
    "question",
    [
        "Aký je názov diplomovej práce Jána Holpa?",
        "Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?",
        "Uveď názov a rok diplomovej práce Jána Holpa.",
        "V ktorom roku je diplomová práca Jána Holpa?",
    ],
)
def test_identity_questions_keep_the_existing_typed_binding(payload, question):
    c = rag_evidence_context(question, payload)
    assert c["requirements"]["response_mode"] == "claims"
    assert not c["requirements"]["content"]
    assert rag_answer_issues(metadata(), c) == []
    bad = metadata()
    bad["claims"][0]["year"] = 2019
    assert "unsupported_person_work_type_year_binding" in rag_answer_issues(bad, c)


@pytest.mark.parametrize(
    "question",
    [
        "Aký je názov diplomovej práce Jána Holpa, ktorá používa PageRank?",
        "Aký je názov diplomovej práce Jána Holpa o metódach vyhľadávania?",
        "Uveď názov diplomovej práce Jána Holpa „Metódy a obsah implementácie“.",
    ],
)
def test_topic_words_in_identity_question_do_not_request_a_description(payload, question):
    c = rag_evidence_context(question, payload)
    assert not c["requirements"]["content"] and c["requirements"]["response_mode"] == "claims"


def test_passages_exclude_other_people_and_other_work_types(payload):
    passages = rag_evidence_context(QUESTION, payload)["content_passages"]
    assert passages and all(
        p["source_id"] == "S1" and p["work_type"] == "diploma_thesis" for p in passages
    )
    assert all("Cassandra" not in p["text"] and "medicínskych" not in p["text"] for p in passages)


@pytest.mark.parametrize("year,expected", [(2021, True), (1999, False)])
def test_year_constraint_also_applies_to_content(payload, year, expected):
    c = rag_evidence_context(f"Opíš obsah diplomovej práce Jána Holpa v roku {year}.", payload)
    assert bool(c["content_passages"]) == expected
    assert all(p["year"] == year for p in c["content_passages"])
    if not expected:
        assert "quote_outside_requested_work" in rag_answer_issues(description(), c)
        refusal = {"status": "no_answer", "claims": [], "evidence_quotes": [], "source_urls": []}
        assert rag_answer_issues(refusal, c) == []


@pytest.mark.parametrize(
    "quote",
    [
        "Názov: Systém získavania informácií v slovenskom jazyku.",
        "Systém získavania informácií v slovenskom jazyku.",
        "Meno vedúceho: Ing. Daniel Hladek, PhD.",
        "Zadanie diplomovej práce:",
    ],
)
def test_metadata_or_an_assignment_heading_does_not_describe_work(payload, quote):
    c = rag_evidence_context(QUESTION, payload)
    answer = description()
    answer["evidence_quotes"][0]["quote"] = quote
    assert "metadata_quote_does_not_describe_work" in rag_answer_issues(answer, c)


def test_metadata_claims_alone_are_rejected(payload):
    issues = rag_answer_issues(metadata(), rag_evidence_context(QUESTION, payload))
    assert "missing_content_evidence" in issues and "unrequested_structured_claims" in issues


@pytest.mark.parametrize(
    "source_id,quote",
    [
        ("S1", "Práca implementuje prehľadávanie slovenského webu."),
        ("S1", "Projekt využíva databázu Cassandra."),
        ("S2", "Práca skúma klasifikáciu medicínskych obrázkov."),
    ],
)
def test_literal_quote_from_wrong_work_or_person_is_rejected(payload, source_id, quote):
    answer = description()
    answer["source_urls"] += ["https://example.test/boris"]
    answer["evidence_quotes"] = [{"source_id": source_id, "quote": quote}]
    assert "quote_outside_requested_work" in rag_answer_issues(
        answer, rag_evidence_context(QUESTION, payload)
    )


def test_correction_sends_real_description_but_no_unrelated_sections(payload):
    payload = compact_rag_payload(payload)
    outputs = iter([metadata(), description()])
    calls = []

    def complete(messages):
        calls.append(copy.deepcopy(messages))
        return completion(next(outputs))

    result = generate_verified_rag(QUESTION, payload, complete)
    assert result["accepted_for_delivery"] and len(calls) == 2
    sent = json.loads(calls[0][1]["content"])["context"]
    assert "facts" not in sent and "context" not in sent
    text = json.dumps(sent["content_candidates"], ensure_ascii=False)
    assert all(c.get("evidence_id") for c in sent["content_candidates"])
    assert all("text" not in s for s in sent["sources"])
    assert (
        "ohodnotenia dokumentov" in text and "Cassandra" not in text and "medicínskych" not in text
    )
    assert "missing_content_evidence" in result["attempts"][0]["issues"]
    assert "Vypracovanie prehľadu metód ohodnotenia dokumentov" in result["answer"]
    assert (
        "Implementácia systému pre získavanie informácií v slovenskom jazyku." in result["answer"]
    )
    assert result["attempts"][-1]["parsed_answer"]["evidence_quotes"][0]["quote"] == ASSIGNMENT
    assert "Výpis je neúplný." not in result["answer"]
    assert result["complete"] is False and result["answer_coverage"]["status"] == "supported"
    assert "expected_bindings" not in json.dumps(calls)


def test_two_metadata_only_attempts_never_reach_user(payload):
    result = generate_verified_rag(QUESTION, payload, lambda _: completion(metadata()))
    assert not result["accepted_for_delivery"] and len(result["attempts"]) == 2


def test_name_year_and_description_can_be_requested_together(payload):
    question = "Uveď názov, rok a stručne opíš obsah diplomovej práce Jána Holpa."
    c = rag_evidence_context(question, payload)
    assert c["requirements"]["response_mode"] == "mixed"
    good = {**metadata(), "evidence_quotes": description()["evidence_quotes"]}
    assert rag_answer_issues(good, c) == []
    assert "missing_content_evidence" in rag_answer_issues(metadata(), c)
    assert "missing_requested_title" in rag_answer_issues(description(), c)


def test_overview_with_documented_assignment_cannot_be_falsely_refused(payload):
    answer = {"status": "no_answer", "claims": [], "evidence_quotes": [], "source_urls": []}
    assert "rejected_supported_work_description" in rag_answer_issues(
        answer, rag_evidence_context(QUESTION, payload)
    )


def test_no_description_in_delivered_text_does_not_license_an_invented_one(payload):
    payload["sources"] = [payload["sources"][0]]
    payload["sources"][0][
        "text"
    ] = "Sekcia: Diplomová práca 2021\nNázov: Systém získavania informácií v slovenskom jazyku."
    payload["context"] = build_context_text(payload["sources"])
    c = rag_evidence_context(QUESTION, payload)
    assert c["content_passages"] == []
    answer = {"status": "no_answer", "claims": [], "evidence_quotes": [], "source_urls": []}
    assert rag_answer_issues(answer, c) == []


def test_a_request_for_results_is_not_answered_merely_by_a_proposed_assignment(payload):
    c = rag_evidence_context("Zhrň dosiahnuté výsledky diplomovej práce Jána Holpa.", payload)
    answer = {"status": "no_answer", "claims": [], "evidence_quotes": [], "source_urls": []}
    assert rag_answer_issues(answer, c) == []


def test_same_person_other_diploma_year_does_not_satisfy_content_constraint(payload):
    payload["sources"][0][
        "text"
    ] += "\n## Diplomová práca 2022\nNázov: Iná diplomová téma.\nPráca skúma automatické preklady."
    payload["context"] = build_context_text(payload["sources"])
    question = "Uveď názov a rok a opíš diplomovú prácu Jána Holpa v roku 2021."
    c = rag_evidence_context(question, payload)
    answer = metadata()
    answer["evidence_quotes"] = [{"source_id": "S1", "quote": "Práca skúma automatické preklady."}]
    assert "quote_outside_requested_work" in rag_answer_issues(answer, c)
    assert "missing_bound_content_evidence" in rag_answer_issues(answer, c)


def test_mixed_answer_cannot_attach_another_years_description_to_its_claim(payload):
    payload["sources"][0][
        "text"
    ] += "\n## Diplomová práca 2022\nNázov: Iná diplomová téma.\nPráca skúma automatické preklady."
    payload["context"] = build_context_text(payload["sources"])
    c = rag_evidence_context("Uveď názov, rok a opíš diplomovú prácu Jána Holpa.", payload)
    answer = metadata()
    answer["evidence_quotes"] = [{"source_id": "S1", "quote": "Práca skúma automatické preklady."}]
    assert "missing_bound_content_evidence" in rag_answer_issues(answer, c)


def test_comparing_contents_requires_evidence_for_each_requested_person(payload):
    c = rag_evidence_context("Porovnaj obsah diplomových prác osôb Ján Holp a Boris Nový.", payload)
    answer = description()
    assert "missing_bound_content_evidence" in rag_answer_issues(answer, c)
    answer["evidence_quotes"].append(
        {"source_id": "S2", "quote": "Práca skúma klasifikáciu medicínskych obrázkov."}
    )
    answer["source_urls"].append("https://example.test/boris")
    assert rag_answer_issues(answer, c) == []