122 lines
6.1 KiB
Python
122 lines
6.1 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tools.graphrag.graphrag import find_work_heading, extract_work_title
|
|
from tools.graphrag.evaluation.source_oracle import academic_type
|
|
from tools.graphrag.evaluation.source_validation import validate_returned_facts, claim_issues
|
|
from tools.graphrag.evaluation.graphrag_evaluator import evaluate_result
|
|
from tools.graphrag.evaluation.reproducibility import content_fingerprints
|
|
from tools.answer_evaluation.answer_format import parse_answer
|
|
|
|
|
|
@pytest.mark.parametrize('heading',['Príprava na Diplomový projekt 2 2021','Príprava k bakalárskej práci 2024'])
|
|
def test_preparation_is_not_academic_work(heading):
|
|
assert find_work_heading(heading) is None and academic_type(heading) is None
|
|
|
|
|
|
def test_assignment_proposal_does_not_establish_an_explicit_title():
|
|
assert extract_work_title(['Návrh na zadanie BP:\n\nDialógový systém pre otázky\n\n1. Vypracujte prehľad.']) is None
|
|
assert extract_work_title(['Názov práce:\n\nDialógový systém pre otázky']) == 'Dialógový systém pre otázky'
|
|
|
|
|
|
def test_consultation_sentence_at_chunk_boundary_is_not_a_project_title():
|
|
assert extract_work_title(['Odporúčaný čas konzultácie je piatok.\n\nMožné backendy:\n\n- OpenNMT']) is None
|
|
|
|
|
|
@pytest.fixture
|
|
def evidence():
|
|
oracle={'people':{'Alena Nová':{'start_year':2020,'documents':['a.md'],
|
|
'academics':[{'year':2024,'title':'Práca A','academic_kind':'Thesis',
|
|
'academic_type':'bachelor_thesis','document_path':'a.md'}]}},
|
|
'documents':{'a.md':{'source_url':'https://example.test/a','tags':['RAG'],
|
|
'categories':['bp2024'],'author':'Autor'}}}
|
|
fact={'fact_type':'person_thesis','person':'Alena Nová','year':2024,'title':'Práca A',
|
|
'thesis_id':'thesis-a','thesis_type':'bachelor_thesis',
|
|
'document_path':'a.md','source_url':'https://example.test/a'}
|
|
question={'graph':{'schema_version':3,'expected_bindings':[{'person':'Alena Nová','year':2024}]}}
|
|
return oracle,fact,question
|
|
|
|
|
|
def test_explicit_incomplete_result_fails_even_without_truncation(evidence):
|
|
oracle,fact,q=evidence
|
|
result={'status':'ok','facts':[fact],'complete':False,'truncated':False}
|
|
assert not evaluate_result(q,result,latency_seconds=0,source_oracle=oracle)['strict']
|
|
|
|
|
|
def test_wrong_extra_year_fails_even_when_required_fact_is_present(evidence):
|
|
oracle,fact,q=evidence
|
|
result={'status':'ok','facts':[fact],'complete':True}
|
|
assert evaluate_result(q,result,latency_seconds=0,source_oracle=oracle)['strict']
|
|
result['facts'].append({**fact,'year':1999,'thesis_id':'unrelated-wrong-extra'})
|
|
grade=evaluate_result(q,result,latency_seconds=0,source_oracle=oracle)
|
|
assert grade['bindings_ok'] and not grade['strict']
|
|
assert grade['source_checked'] and not grade['all_facts_supported']
|
|
|
|
|
|
@pytest.mark.parametrize('change',[{'title':'Vymyslený názov'},{'thesis_type':'doctoral_thesis'},
|
|
{'document_path':'other.md'},{'source_url':'https://wrong.test'}])
|
|
def test_every_extra_claim_has_correct_source_and_properties(evidence,change):
|
|
oracle,fact,_=evidence
|
|
assert not validate_returned_facts({'facts':[{**fact,**change}]},oracle)['all_facts_supported']
|
|
|
|
|
|
def test_shared_metadata_validated_for_both_people(evidence):
|
|
oracle,_,_=evidence
|
|
row={'fact_type':'shared_keyword','left_person':'Alena Nová','right_person':'Unknown',
|
|
'keyword':'RAG','left_document_path':'a.md','right_document_path':'a.md'}
|
|
assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported']
|
|
|
|
|
|
def test_source_checker_rejects_unknown_fact_type(evidence):
|
|
oracle,_,_=evidence
|
|
assert not validate_returned_facts({'facts':[{'year':1999}]},oracle)['all_facts_supported']
|
|
|
|
|
|
def test_metadata_from_different_documents_cannot_form_one_document_claim(evidence):
|
|
oracle,_,_=evidence
|
|
oracle['people']['Alena Nová']['documents'].append('child.md')
|
|
oracle['documents']['child.md']={'categories':[],'tags':[],'author':'Iný autor','source_url':'https://example.test/child'}
|
|
assert claim_issues({'person':'Alena Nová','category':'bp2024','document_author':'Iný autor'},oracle)
|
|
|
|
|
|
def test_shared_comparison_years_cannot_contain_unbacked_extra_year(evidence):
|
|
oracle,_,_=evidence
|
|
row={'fact_type':'academic_year_comparison','people':['Alena Nová'],
|
|
'shared_years':[2024,1999],'same_academic_year':True}
|
|
assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported']
|
|
|
|
|
|
def test_comparison_checks_individual_years_as_well_as_shared_year(evidence):
|
|
oracle,_,_=evidence
|
|
row={'fact_type':'start_year_comparison','people':['Alena Nová'],'shared_start_year':2020,
|
|
'same_start_year':True,'start_years':{'Alena Nová':1999}}
|
|
assert not validate_returned_facts({'facts':[row]},oracle)['all_facts_supported']
|
|
row['start_years']['Alena Nová']=2020
|
|
assert validate_returned_facts({'facts':[row]},oracle)['all_facts_supported']
|
|
|
|
|
|
def test_freeze_includes_shared_modules_and_effective_publication_policy():
|
|
questions=Path('tools/graphrag/evaluation/graphrag_questions_v4.json')
|
|
fp=content_fingerprints(questions)
|
|
assert {'tools/retrieval/publication.py','tools/retrieval/retrieval_contract.py','tools/sync/index_state.py'} <= fp['implementation'].keys()
|
|
assert fp['retrieval_policy']['published_only'] is True
|
|
assert fp['retrieval_policy']['academic_preparation_is_work'] is False
|
|
|
|
|
|
def test_only_one_complete_json_fence_is_unwrapped():
|
|
answer={'answer':'Správny text','status':'ok','claims':[], 'source_urls':[],'count':None}
|
|
text='Sprievodný text\n```json\n'+json.dumps(answer)+'\n```'
|
|
parsed,normalized=parse_answer(text)
|
|
assert parsed['answer']=='Správny text' and normalized
|
|
with pytest.raises(ValueError):
|
|
parse_answer(text+'\n```json\n{}\n```')
|
|
|
|
|
|
@pytest.mark.parametrize('text',['{"answer":"zlá "citácia""}',
|
|
'{"answer":"test","status":"ok","claims":[],"source_urls":[],"count":"2"}'])
|
|
def test_malformed_or_wrongly_typed_json_is_not_silently_repaired(text):
|
|
with pytest.raises(ValueError):
|
|
parse_answer(text)
|