import json from pathlib import Path import pytest from tools.graphrag.graphrag import find_work_heading, extract_work_title from tools.graphrag.evaluation.source_oracle import academic_type from tools.graphrag.evaluation.source_validation import validate_returned_facts, claim_issues from tools.graphrag.evaluation.graphrag_evaluator import evaluate_result from tools.graphrag.evaluation.reproducibility import content_fingerprints from tools.answer_evaluation.answer_format import parse_answer @pytest.mark.parametrize('heading',['Príprava na Diplomový projekt 2 2021','Príprava k bakalárskej práci 2024']) def test_preparation_is_not_academic_work(heading): assert find_work_heading(heading) is None and academic_type(heading) is None def test_assignment_proposal_does_not_establish_an_explicit_title(): assert extract_work_title(['Návrh na zadanie BP:\n\nDialógový systém pre otázky\n\n1. Vypracujte prehľad.']) is None assert extract_work_title(['Názov práce:\n\nDialógový systém pre otázky']) == 'Dialógový systém pre otázky' def test_consultation_sentence_at_chunk_boundary_is_not_a_project_title(): assert extract_work_title(['Odporúčaný čas konzultácie je piatok.\n\nMožné backendy:\n\n- OpenNMT']) is None @pytest.fixture def evidence(): oracle={'people':{'Alena Nová':{'start_year':2020,'documents':['a.md'], 'academics':[{'year':2024,'title':'Práca A','academic_kind':'Thesis', 'academic_type':'bachelor_thesis','document_path':'a.md'}]}}, 'documents':{'a.md':{'source_url':'https://example.test/a','tags':['RAG'], 'categories':['bp2024'],'author':'Autor'}}} fact={'fact_type':'person_thesis','person':'Alena Nová','year':2024,'title':'Práca A', 'thesis_id':'thesis-a','thesis_type':'bachelor_thesis', 'document_path':'a.md','source_url':'https://example.test/a'} question={'graph':{'schema_version':3,'expected_bindings':[{'person':'Alena Nová','year':2024}]}} return oracle,fact,question def test_explicit_incomplete_result_fails_even_without_truncation(evidence): oracle,fact,q=evidence result={'status':'ok','facts':[fact],'complete':False,'truncated':False} assert not evaluate_result(q,result,latency_seconds=0,source_oracle=oracle)['strict'] def test_wrong_extra_year_fails_even_when_required_fact_is_present(evidence): oracle,fact,q=evidence result={'status':'ok','facts':[fact],'complete':True} assert evaluate_result(q,result,latency_seconds=0,source_oracle=oracle)['strict'] result['facts'].append({**fact,'year':1999,'thesis_id':'unrelated-wrong-extra'}) grade=evaluate_result(q,result,latency_seconds=0,source_oracle=oracle) assert grade['bindings_ok'] and not grade['strict'] assert grade['source_checked'] and not grade['all_facts_supported'] @pytest.mark.parametrize('change',[{'title':'Vymyslený názov'},{'thesis_type':'doctoral_thesis'}, {'document_path':'other.md'},{'source_url':'https://wrong.test'}]) def test_every_extra_claim_has_correct_source_and_properties(evidence,change): oracle,fact,_=evidence assert not validate_returned_facts({'facts':[{**fact,**change}]},oracle)['all_facts_supported'] def test_shared_metadata_validated_for_both_people(evidence): oracle,_,_=evidence row={'fact_type':'shared_keyword','left_person':'Alena Nová','right_person':'Unknown', 'keyword':'RAG','left_document_path':'a.md','right_document_path':'a.md'} assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported'] def test_source_checker_rejects_unknown_fact_type(evidence): oracle,_,_=evidence assert not validate_returned_facts({'facts':[{'year':1999}]},oracle)['all_facts_supported'] def test_metadata_from_different_documents_cannot_form_one_document_claim(evidence): oracle,_,_=evidence oracle['people']['Alena Nová']['documents'].append('child.md') oracle['documents']['child.md']={'categories':[],'tags':[],'author':'Iný autor','source_url':'https://example.test/child'} assert claim_issues({'person':'Alena Nová','category':'bp2024','document_author':'Iný autor'},oracle) def test_shared_comparison_years_cannot_contain_unbacked_extra_year(evidence): oracle,_,_=evidence row={'fact_type':'academic_year_comparison','people':['Alena Nová'], 'shared_years':[2024,1999],'same_academic_year':True} assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported'] def test_comparison_checks_individual_years_as_well_as_shared_year(evidence): oracle,_,_=evidence row={'fact_type':'start_year_comparison','people':['Alena Nová'],'shared_start_year':2020, 'same_start_year':True,'start_years':{'Alena Nová':1999}} assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported'] row['start_years']['Alena Nová']=2020 assert validate_returned_facts({'facts':[row]},oracle)['all_facts_supported'] def test_freeze_includes_shared_modules_and_effective_publication_policy(): questions=Path('tools/graphrag/evaluation/graphrag_questions_v4.json') fp=content_fingerprints(questions) assert {'tools/publication.py','tools/retrieval_contract.py','tools/index_state.py'} <= fp['implementation'].keys() assert fp['retrieval_policy']['published_only'] is True assert fp['retrieval_policy']['academic_preparation_is_work'] is False def test_only_one_complete_json_fence_is_unwrapped(): answer={'answer':'Správny text','status':'ok','claims':[], 'source_urls':[],'count':None} text='Sprievodný text\n```json\n'+json.dumps(answer)+'\n```' parsed,normalized=parse_answer(text) assert parsed['answer']=='Správny text' and normalized with pytest.raises(ValueError): parse_answer(text+'\n```json\n{}\n```') @pytest.mark.parametrize('text',['{"answer":"zlá "citácia""}', '{"answer":"test","status":"ok","claims":[],"source_urls":[],"count":"2"}']) def test_malformed_or_wrongly_typed_json_is_not_silently_repaired(text): with pytest.raises(ValueError): parse_answer(text)