dp-zp-agent/tools/graphrag/tests/test_evaluation_v5.py

import json
from pathlib import Path

import pytest

from tools.graphrag.graphrag import find_work_heading, extract_work_title
from tools.graphrag.evaluation.source_oracle import academic_type
from tools.graphrag.evaluation.source_validation import validate_returned_facts, claim_issues
from tools.graphrag.evaluation.graphrag_evaluator import evaluate_result
from tools.graphrag.evaluation.reproducibility import content_fingerprints
from tools.answer_evaluation.answer_format import parse_answer


@pytest.mark.parametrize('heading',['Príprava na Diplomový projekt 2 2021','Príprava k bakalárskej práci 2024'])
def test_preparation_is_not_academic_work(heading):
    assert find_work_heading(heading) is None and academic_type(heading) is None


def test_assignment_proposal_does_not_establish_an_explicit_title():
    assert extract_work_title(['Návrh na zadanie BP:\n\nDialógový systém pre otázky\n\n1. Vypracujte prehľad.']) is None
    assert extract_work_title(['Názov práce:\n\nDialógový systém pre otázky']) == 'Dialógový systém pre otázky'


def test_consultation_sentence_at_chunk_boundary_is_not_a_project_title():
    assert extract_work_title(['Odporúčaný čas konzultácie je piatok.\n\nMožné backendy:\n\n- OpenNMT']) is None


@pytest.fixture
def evidence():
    oracle={'people':{'Alena Nová':{'start_year':2020,'documents':['a.md'],
              'academics':[{'year':2024,'title':'Práca A','academic_kind':'Thesis',
                            'academic_type':'bachelor_thesis','document_path':'a.md'}]}},
            'documents':{'a.md':{'source_url':'https://example.test/a','tags':['RAG'],
                                 'categories':['bp2024'],'author':'Autor'}}}
    fact={'fact_type':'person_thesis','person':'Alena Nová','year':2024,'title':'Práca A',
          'thesis_id':'thesis-a','thesis_type':'bachelor_thesis',
          'document_path':'a.md','source_url':'https://example.test/a'}
    question={'graph':{'schema_version':3,'expected_bindings':[{'person':'Alena Nová','year':2024}]}}
    return oracle,fact,question


def test_explicit_incomplete_result_fails_even_without_truncation(evidence):
    oracle,fact,q=evidence
    result={'status':'ok','facts':[fact],'complete':False,'truncated':False}
    assert not evaluate_result(q,result,latency_seconds=0,source_oracle=oracle)['strict']


def test_wrong_extra_year_fails_even_when_required_fact_is_present(evidence):
    oracle,fact,q=evidence
    result={'status':'ok','facts':[fact],'complete':True}
    assert evaluate_result(q,result,latency_seconds=0,source_oracle=oracle)['strict']
    result['facts'].append({**fact,'year':1999,'thesis_id':'unrelated-wrong-extra'})
    grade=evaluate_result(q,result,latency_seconds=0,source_oracle=oracle)
    assert grade['bindings_ok'] and not grade['strict']
    assert grade['source_checked'] and not grade['all_facts_supported']


@pytest.mark.parametrize('change',[{'title':'Vymyslený názov'},{'thesis_type':'doctoral_thesis'},
                                  {'document_path':'other.md'},{'source_url':'https://wrong.test'}])
def test_every_extra_claim_has_correct_source_and_properties(evidence,change):
    oracle,fact,_=evidence
    assert not validate_returned_facts({'facts':[{**fact,**change}]},oracle)['all_facts_supported']


def test_shared_metadata_validated_for_both_people(evidence):
    oracle,_,_=evidence
    row={'fact_type':'shared_keyword','left_person':'Alena Nová','right_person':'Unknown',
         'keyword':'RAG','left_document_path':'a.md','right_document_path':'a.md'}
    assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported']


def test_source_checker_rejects_unknown_fact_type(evidence):
    oracle,_,_=evidence
    assert not validate_returned_facts({'facts':[{'year':1999}]},oracle)['all_facts_supported']


def test_metadata_from_different_documents_cannot_form_one_document_claim(evidence):
    oracle,_,_=evidence
    oracle['people']['Alena Nová']['documents'].append('child.md')
    oracle['documents']['child.md']={'categories':[],'tags':[],'author':'Iný autor','source_url':'https://example.test/child'}
    assert claim_issues({'person':'Alena Nová','category':'bp2024','document_author':'Iný autor'},oracle)


def test_shared_comparison_years_cannot_contain_unbacked_extra_year(evidence):
    oracle,_,_=evidence
    row={'fact_type':'academic_year_comparison','people':['Alena Nová'],
         'shared_years':[2024,1999],'same_academic_year':True}
    assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported']


def test_comparison_checks_individual_years_as_well_as_shared_year(evidence):
    oracle,_,_=evidence
    row={'fact_type':'start_year_comparison','people':['Alena Nová'],'shared_start_year':2020,
         'same_start_year':True,'start_years':{'Alena Nová':1999}}
    assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported']
    row['start_years']['Alena Nová']=2020
    assert validate_returned_facts({'facts':[row]},oracle)['all_facts_supported']


def test_freeze_includes_shared_modules_and_effective_publication_policy():
    questions=Path('tools/graphrag/evaluation/graphrag_questions_v4.json')
    fp=content_fingerprints(questions)
    assert {'tools/publication.py','tools/retrieval_contract.py','tools/index_state.py'} <= fp['implementation'].keys()
    assert fp['retrieval_policy']['published_only'] is True
    assert fp['retrieval_policy']['academic_preparation_is_work'] is False


def test_only_one_complete_json_fence_is_unwrapped():
    answer={'answer':'Správny text','status':'ok','claims':[], 'source_urls':[],'count':None}
    text='Sprievodný text\n```json\n'+json.dumps(answer)+'\n```'
    parsed,normalized=parse_answer(text)
    assert parsed['answer']=='Správny text' and normalized
    with pytest.raises(ValueError):
        parse_answer(text+'\n```json\n{}\n```')


@pytest.mark.parametrize('text',['{"answer":"zlá "citácia""}',
    '{"answer":"test","status":"ok","claims":[],"source_urls":[],"count":"2"}'])
def test_malformed_or_wrongly_typed_json_is_not_silently_repaired(text):
    with pytest.raises(ValueError):
        parse_answer(text)