114 lines
5.9 KiB
Python
114 lines
5.9 KiB
Python
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools.answer_evaluation.context_policy import answer_context_issues, citation_check, coverage_issues, prose_issues, text_contains_value
|
|
from tools.answer_evaluation.checked_generation import generate_checked_answer
|
|
from tools.answer_evaluation.render_answer import render_graph_answer
|
|
from tools.graphrag.graph_search import extract_query_constraints
|
|
from tools.graphrag.graphrag import extract_start_year
|
|
|
|
|
|
@pytest.fixture
|
|
def context():
|
|
return {'status': 'ok', 'complete': True, 'truncated': False, 'aggregation': None,
|
|
'sources': [{'source_url': 'https://example.test/students/2018/eva'}],
|
|
'facts': [{'person': 'Eva Nová', 'year': 2026, 'title': 'Práca o 3 modeloch',
|
|
'source_url': 'https://example.test/students/2018/eva'}], 'answer_candidates': []}
|
|
|
|
|
|
@pytest.fixture
|
|
def answer(context):
|
|
return {'answer': '', 'status': 'ok', 'claims': [{'person': 'Eva Nová', 'year': 2026, 'title': 'Práca o 3 modeloch'}],
|
|
'source_urls': [context['sources'][0]['source_url']], 'count': None}
|
|
|
|
|
|
@pytest.mark.parametrize('year', [1899, 1999, 2099, 2100, 9999])
|
|
def test_year_constraint_is_never_silently_dropped(year):
|
|
assert extract_query_constraints(f'Aká práca je v roku {year}?')['years'] == [year]
|
|
|
|
|
|
@pytest.mark.parametrize('text', ['Toto je úplný zoznam. Výpis je neúplný.',
|
|
'Uvádzam všetkých študentov. Výpis je neúplný.',
|
|
'Úplný zoznam, ale dôkazy sú orezané.'])
|
|
def test_caveat_does_not_license_contradictory_exhaustive_claim(text):
|
|
assert 'unsupported_exhaustive_statement' in coverage_issues({'answer': text, 'status': 'ok'}, {'complete': False})
|
|
|
|
|
|
@pytest.mark.parametrize('text', ['Výpis je neúplný.', 'Toto nie je úplný zoznam.', 'Nie je to úplný zoznam.'])
|
|
def test_truthful_scope_notice_is_not_rejected(text):
|
|
assert coverage_issues({'answer': text, 'status': 'ok'}, {'complete': False}) == []
|
|
|
|
|
|
@pytest.mark.parametrize('text', ['[zdroj][s]\n\n[s]: /vymysleny', '<a href="/vymysleny">zdroj</a>',
|
|
'<img src="//evil.test/x">', '[s]: javascript:alert(1)',
|
|
'<a href="https://evil.test/x">x</a>'])
|
|
def test_all_citation_syntaxes_are_checked(text):
|
|
assert not citation_check({'answer': text, 'source_urls': []}, {'https://example.test/a'})['citations_ok']
|
|
|
|
|
|
def test_legitimate_balanced_parentheses_and_reference_html_links():
|
|
url = 'https://example.test/a_(b)'
|
|
text = f'[x]({url}) a {url}.\n[s]: <{url}>\n<a href="{url}">x</a>'
|
|
assert citation_check({'answer': text, 'source_urls': [url]}, {url})['citations_ok']
|
|
|
|
|
|
def test_count_requires_explicit_complete_aggregation(context, answer):
|
|
answer['count'] = 98765
|
|
assert 'unsupported_total_count' in answer_context_issues(answer, context, check_text=False)
|
|
|
|
|
|
@pytest.mark.parametrize('field,issue', [('claims', 'missing_required_claims'), ('source_urls', 'missing_required_citations')])
|
|
def test_positive_answer_needs_facts_and_citations(context, answer, field, issue):
|
|
answer[field] = []
|
|
assert issue in answer_context_issues(answer, context, check_text=False)
|
|
|
|
|
|
def test_every_claim_field_needs_its_source(context, answer):
|
|
context['facts'].append({'person': 'Eva Nová', 'keyword': 'python', 'source_url': 'https://example.test/child'})
|
|
context['sources'].append({'source_url': 'https://example.test/child'})
|
|
answer['claims'][0]['keyword'] = 'python'
|
|
assert 'missing_claim_source' in answer_context_issues(answer, context, check_text=False)
|
|
answer['source_urls'].append('https://example.test/child')
|
|
assert answer_context_issues(answer, context, check_text=False) == []
|
|
|
|
|
|
def test_wrong_extra_prose_year_is_rejected_but_citation_path_year_is_not(context, answer):
|
|
answer['answer'] = 'Eva Nová má prácu z roku 2026. ' + answer['source_urls'][0]
|
|
assert prose_issues(answer) == []
|
|
answer['answer'] += ' Obhajoba bola v roku 2099.'
|
|
assert prose_issues(answer) == ['unsupported_prose_number']
|
|
|
|
|
|
def test_renderer_uses_only_validated_fields_and_preserves_model_reply(context, answer):
|
|
original = copy.deepcopy(answer)
|
|
def complete(_): return {'choices': [{'message': {'content': json.dumps(answer)}}]}
|
|
result = generate_checked_answer('Názov a rok?', context, 'Pokyny', complete)
|
|
parsed_original = {key: value for key, value in original.items() if value is not None}
|
|
assert result['accepted_for_delivery'] and result['model_answer'] == parsed_original
|
|
assert result['attempts'][0]['answer'] == parsed_original
|
|
assert result['attempts'][0]['response']['choices'][0]['message']['content'] == json.dumps(original)
|
|
assert '2026' in result['answer']['answer'] and 'Práca o 3 modeloch' in result['answer']['answer']
|
|
assert answer == original
|
|
|
|
|
|
def test_zero_count_is_valid_without_claims_or_sources():
|
|
context = {'status': 'ok', 'facts': [], 'sources': [], 'complete': True,
|
|
'aggregation': {'count': 0, 'complete': True}}
|
|
answer = {'answer': '', 'status': 'ok', 'claims': [], 'source_urls': [], 'count': 0}
|
|
rendered = render_graph_answer(answer, context)
|
|
assert answer_context_issues(rendered, context) == []
|
|
|
|
|
|
def test_directory_cohort_does_not_become_study_start():
|
|
assert extract_start_year('pages/students/2016/eva/README.md', [{'text': 'Diplomová práca 2026'}]) is None
|
|
assert extract_start_year('pages/students/2016/eva/README.md', [{'text': 'Rok začiatku štúdia: 2017'}]) == 2017
|
|
assert extract_start_year('pages/students/2016/eva/README.md', [{'text': 'Beginning of the study: 2018'}]) == 2018
|
|
|
|
|
|
def test_person_inflection_keeps_distinct_names_distinct():
|
|
assert text_contains_value('Adriána Remiáša a Martina NaN1', 'Martin NaN1', person=True)
|
|
assert not text_contains_value('Martin Jančura', 'Martin NaN1', person=True)
|
|
assert not text_contains_value('Martin Malinovský', 'Martin Malý', person=True)
|