dp-zp-agent/tools/graphrag/tests/test_evaluation_v5.py

import json
from pathlib import Path

import pytest

from tools.graphrag.graphrag import find_work_heading, extract_work_title
from tools.graphrag.evaluation.source_oracle import academic_type
from tools.graphrag.evaluation.source_validation import validate_returned_facts, claim_issues
from tools.graphrag.evaluation.graphrag_evaluator import evaluate_result
from tools.graphrag.evaluation.reproducibility import content_fingerprints
from tools.answers.answer_format import parse_answer


@pytest.mark.parametrize(
    "heading", ["Príprava na Diplomový projekt 2 2021", "Príprava k bakalárskej práci 2024"]
)
def test_preparation_is_not_academic_work(heading):
    assert find_work_heading(heading) is None and academic_type(heading) is None


def test_assignment_proposal_does_not_establish_an_explicit_title():
    assert (
        extract_work_title(
            ["Návrh na zadanie BP:\n\nDialógový systém pre otázky\n\n1. Vypracujte prehľad."]
        )
        is None
    )
    assert (
        extract_work_title(["Názov práce:\n\nDialógový systém pre otázky"])
        == "Dialógový systém pre otázky"
    )


def test_consultation_sentence_at_chunk_boundary_is_not_a_project_title():
    assert (
        extract_work_title(
            ["Odporúčaný čas konzultácie je piatok.\n\nMožné backendy:\n\n- OpenNMT"]
        )
        is None
    )


@pytest.fixture
def evidence():
    oracle = {
        "people": {
            "Alena Nová": {
                "start_year": 2020,
                "documents": ["a.md"],
                "academics": [
                    {
                        "year": 2024,
                        "title": "Práca A",
                        "academic_kind": "Thesis",
                        "academic_type": "bachelor_thesis",
                        "document_path": "a.md",
                    }
                ],
            }
        },
        "documents": {
            "a.md": {
                "source_url": "https://example.test/a",
                "tags": ["RAG"],
                "categories": ["bp2024"],
                "author": "Autor",
            }
        },
    }
    fact = {
        "fact_type": "person_thesis",
        "person": "Alena Nová",
        "year": 2024,
        "title": "Práca A",
        "thesis_id": "thesis-a",
        "thesis_type": "bachelor_thesis",
        "document_path": "a.md",
        "source_url": "https://example.test/a",
    }
    question = {
        "graph": {
            "schema_version": 3,
            "expected_bindings": [{"person": "Alena Nová", "year": 2024}],
        }
    }
    return oracle, fact, question


def test_explicit_incomplete_result_fails_even_without_truncation(evidence):
    oracle, fact, q = evidence
    result = {"status": "ok", "facts": [fact], "complete": False, "truncated": False}
    assert not evaluate_result(q, result, latency_seconds=0, source_oracle=oracle)["strict"]


def test_wrong_extra_year_fails_even_when_required_fact_is_present(evidence):
    oracle, fact, q = evidence
    result = {"status": "ok", "facts": [fact], "complete": True}
    assert evaluate_result(q, result, latency_seconds=0, source_oracle=oracle)["strict"]
    result["facts"].append({**fact, "year": 1999, "thesis_id": "unrelated-wrong-extra"})
    grade = evaluate_result(q, result, latency_seconds=0, source_oracle=oracle)
    assert grade["bindings_ok"] and not grade["strict"]
    assert grade["source_checked"] and not grade["all_facts_supported"]


@pytest.mark.parametrize(
    "change",
    [
        {"title": "Vymyslený názov"},
        {"thesis_type": "doctoral_thesis"},
        {"document_path": "other.md"},
        {"source_url": "https://wrong.test"},
    ],
)
def test_every_extra_claim_has_correct_source_and_properties(evidence, change):
    oracle, fact, _ = evidence
    assert not validate_returned_facts({"facts": [{**fact, **change}]}, oracle)[
        "all_facts_supported"
    ]


def test_shared_metadata_validated_for_both_people(evidence):
    oracle, _, _ = evidence
    row = {
        "fact_type": "shared_keyword",
        "left_person": "Alena Nová",
        "right_person": "Unknown",
        "keyword": "RAG",
        "left_document_path": "a.md",
        "right_document_path": "a.md",
    }
    assert not validate_returned_facts({"facts": [row]}, oracle)["all_facts_supported"]


def test_source_checker_rejects_unknown_fact_type(evidence):
    oracle, _, _ = evidence
    assert not validate_returned_facts({"facts": [{"year": 1999}]}, oracle)["all_facts_supported"]


def test_metadata_from_different_documents_cannot_form_one_document_claim(evidence):
    oracle, _, _ = evidence
    oracle["people"]["Alena Nová"]["documents"].append("child.md")
    oracle["documents"]["child.md"] = {
        "categories": [],
        "tags": [],
        "author": "Iný autor",
        "source_url": "https://example.test/child",
    }
    assert claim_issues(
        {"person": "Alena Nová", "category": "bp2024", "document_author": "Iný autor"}, oracle
    )


def test_shared_comparison_years_cannot_contain_unbacked_extra_year(evidence):
    oracle, _, _ = evidence
    row = {
        "fact_type": "academic_year_comparison",
        "people": ["Alena Nová"],
        "shared_years": [2024, 1999],
        "same_academic_year": True,
    }
    assert not validate_returned_facts({"facts": [row]}, oracle)["all_facts_supported"]


def test_comparison_checks_individual_years_as_well_as_shared_year(evidence):
    oracle, _, _ = evidence
    row = {
        "fact_type": "start_year_comparison",
        "people": ["Alena Nová"],
        "shared_start_year": 2020,
        "same_start_year": True,
        "start_years": {"Alena Nová": 1999},
    }
    assert not validate_returned_facts({"facts": [row]}, oracle)["all_facts_supported"]
    row["start_years"]["Alena Nová"] = 2020
    assert validate_returned_facts({"facts": [row]}, oracle)["all_facts_supported"]


def test_freeze_includes_shared_modules_and_effective_publication_policy():
    questions = Path("experiments/graphrag/graphrag_questions_v4.json")
    fp = content_fingerprints(questions)
    assert {
        "tools/retrieval/publication.py",
        "tools/retrieval/retrieval_contract.py",
        "tools/sync/index_state.py",
    } <= fp["implementation"].keys()
    assert "deployment/requirements.txt" in fp["implementation"]
    assert fp["retrieval_policy"]["published_only"] is True
    assert fp["retrieval_policy"]["academic_preparation_is_work"] is False


def test_only_one_complete_json_fence_is_unwrapped():
    answer = {
        "answer": "Správny text",
        "status": "ok",
        "claims": [],
        "source_urls": [],
        "count": None,
    }
    text = "Sprievodný text\n```json\n" + json.dumps(answer) + "\n```"
    parsed, normalized = parse_answer(text)
    assert parsed["answer"] == "Správny text" and normalized
    with pytest.raises(ValueError):
        parse_answer(text + "\n```json\n{}\n```")


@pytest.mark.parametrize(
    "text",
    [
        '{"answer":"zlá "citácia""}',
        '{"answer":"test","status":"ok","claims":[],"source_urls":[],"count":"2"}',
    ],
)
def test_malformed_or_wrongly_typed_json_is_not_silently_repaired(text):
    with pytest.raises(ValueError):
        parse_answer(text)