152 lines
8.5 KiB
Python
152 lines
8.5 KiB
Python
"""Natural presentation must preserve evidence, modality, binding and guards."""
|
|
import copy
|
|
|
|
import pytest
|
|
|
|
from tools.answer_evaluation.context_policy import answer_context_issues, prose_issues
|
|
from tools.answer_evaluation.graph_context import graph_context
|
|
from tools.answer_evaluation.presentation import content_lines, fact_sentences, goal_text, owner_in_question
|
|
from tools.answer_evaluation.render_answer import render_graph_answer
|
|
from tools.answer_evaluation.verified_rag import generate_verified_rag
|
|
from test_graph_work_listing import retrieved, listing_answer
|
|
from test_rag_work_content import payload, description, completion, ASSIGNMENT, QUESTION, URL
|
|
|
|
|
|
def test_identity_is_a_sentence_but_year_is_not_a_start_or_completion_date():
|
|
claim = {'person': 'Ján Holp', 'work_type': 'diploma_thesis',
|
|
'title': 'Systém získavania informácií.', 'year': 2021}
|
|
before = copy.deepcopy(claim)
|
|
text = fact_sentences(claim, 'Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?')
|
|
assert text == ('Diplomová práca Jána Holpa má názov **Systém získavania informácií** '
|
|
'(rok práce: **2021**).')
|
|
assert not any(word in text for word in ('začal', 'dokonč', 'vypracovaná'))
|
|
assert claim == before
|
|
|
|
|
|
@pytest.mark.parametrize('person,question,expected', [
|
|
('Ján Holp', 'Čomu sa venuje diplomová práca Jána Holpa?', 'Jána Holpa'),
|
|
('Eva Nová', 'Aké je zadanie diplomovej práce Evy Novej?', 'Evy Novej'),
|
|
('Ján Holp', 'Aké práce má Ján Holp?', None),
|
|
('Ján Holp', 'Aké práce má Martin Jančura?', None),
|
|
('Ján Holp', 'Aké je zadanie práce Jána Holpáka?', None),
|
|
])
|
|
def test_display_name_is_bound_to_the_whole_name(person, question, expected):
|
|
assert owner_in_question(person, question) == expected
|
|
|
|
|
|
def test_a_name_without_a_verified_genitive_remains_in_its_original_form():
|
|
text = fact_sentences({'person': 'Alex Kim', 'work_type': 'diploma_thesis',
|
|
'title': 'Vyhľadávanie', 'year': 2020})
|
|
assert text == 'Alex Kim má diplomovú prácu s názvom **Vyhľadávanie** (rok práce: **2020**).'
|
|
|
|
|
|
def test_content_changes_only_grammar_after_literal_evidence_passes(payload):
|
|
before = copy.deepcopy(payload)
|
|
result = generate_verified_rag(QUESTION, payload, lambda _: completion(description()))
|
|
assert result['accepted_for_delivery']
|
|
assert 'Diplomová práca Jána Holpa má podľa dostupných podkladov tieto ciele:' in result['answer']
|
|
assert '1. Vypracovanie prehľadu metód ohodnotenia dokumentov' in result['answer']
|
|
assert '2. Implementácia systému pre získavanie informácií v slovenskom jazyku.' in result['answer']
|
|
assert 'Vypracujte' not in result['answer'] and 'Implementujte' not in result['answer']
|
|
assert result['attempts'][0]['parsed_answer']['evidence_quotes'][0]['quote'] == ASSIGNMENT
|
|
assert result['source_urls'] == [URL] and result['complete'] is False
|
|
assert 'neúplný' not in result['answer'] and payload == before
|
|
|
|
|
|
@pytest.mark.parametrize('original,expected', [
|
|
('1. Vypracujte prehľad metód iba pre slovenské texty.',
|
|
'Vypracovanie prehľadu metód iba pre slovenské texty.'),
|
|
('2. Implementujte jednoduchý systém, v ktorom využijete ohodnotenie dokumentov.',
|
|
'Implementácia jednoduchého systému, v ktorom sa má využiť ohodnotenie dokumentov.'),
|
|
('3. Navrhnite a vypracujte experimenty, v ktorých vyhodnotíte vybrané metódy.',
|
|
'Návrh a vypracovanie experimentov s cieľom vyhodnotiť vybrané metódy.'),
|
|
('4. Navrhnite možné zlepšenia presnosti vyhľadávania.',
|
|
'Návrh možných zlepšení presnosti vyhľadávania.'),
|
|
('Navrhnite a implementujte API bez zmeny databázy.',
|
|
'Navrhnúť a implementovať API bez zmeny databázy.'),
|
|
])
|
|
def test_goals_keep_constraints_and_do_not_claim_completion(original, expected):
|
|
assert goal_text(original) == (expected, True)
|
|
|
|
|
|
@pytest.mark.parametrize('text', [
|
|
'Implementovaný PageRank a indexovanie webových stránok.',
|
|
'Práca neimplementuje neurónové siete.',
|
|
'Ak bude dostupný korpus, otestuje sa presnosť.',
|
|
'Neimplementujte externé API.',
|
|
])
|
|
def test_progress_negations_and_unknown_prose_are_not_reinterpreted(text):
|
|
assert goal_text(text) == (text, False)
|
|
rendered = '\n'.join(content_lines([text], 'Ján Holp', 'diploma_thesis'))
|
|
assert text in rendered and 'tieto ciele' not in rendered
|
|
|
|
|
|
@pytest.mark.parametrize('command',['vyhodnoťte','vyhodnotte','vyhodnnotte'])
|
|
def test_known_coordinated_command_variants_are_natural_without_claiming_completion(command):
|
|
raw='Preštudujte si spôsoby vyhodnotenia RAG - RAGAS, DeepEval a '+command+' systém.'
|
|
text,converted=goal_text(raw)
|
|
assert converted and text=='Preštudovať si spôsoby vyhodnotenia RAG - RAGAS, DeepEval a vyhodnotiť systém.'
|
|
assert 'vyhodnotený' not in text and 'dokončený' not in text
|
|
|
|
|
|
def test_goal_formatting_protects_urls_code_attributes_and_conditions():
|
|
raw=('Vypracujte prehľad iba pre slovenské texty a vyhodnnotte systém podľa '
|
|
'https://example.test/implementujte/vyhodnnotte?x=navrhnite '
|
|
's kódom `implementujte("vyhodnnotte")` a <a href="/vyhodnnotte">odkazom</a>.')
|
|
text,converted=goal_text(raw)
|
|
assert converted and 'Vypracovať prehľad iba pre slovenské texty a vyhodnotiť systém' in text
|
|
assert 'https://example.test/implementujte/vyhodnnotte?x=navrhnite' in text
|
|
assert '`implementujte("vyhodnnotte")`' in text and '<a href="/vyhodnnotte">' in text
|
|
|
|
|
|
@pytest.mark.parametrize('complete',[True,False])
|
|
def test_shared_document_authors_have_a_grammatical_bounded_summary(complete):
|
|
people=['Eva Nová','Boris Nový'];urls=['https://example.test/eva','https://example.test/boris']
|
|
claims=[{'person':person,'document_author':'Daniel Hladek'} for person in people]
|
|
c={'status':'ok','facts':[{**claim,'source_url':url} for claim,url in zip(claims,urls)],
|
|
'sources':[{'source_url':url} for url in urls], 'complete':complete,'truncated':not complete,
|
|
'shared_properties':[{'field':'document_author','value':'Daniel Hladek','people':people}]}
|
|
rendered=render_graph_answer({'status':'ok','claims':claims,'source_urls':urls},c)
|
|
prefix='Spoloční autori dokumentov:' if complete else 'Doložení spoloční autori dokumentov:'
|
|
assert rendered['answer'].startswith(prefix)
|
|
assert answer_context_issues(rendered,c)==[]
|
|
|
|
|
|
def test_assignment_heading_does_not_keep_imperative_style():
|
|
lines = content_lines(['*Zadanie diplomovej práce*:', '1. Vypracujte prehľad metód.',
|
|
'2. Implementujte systém.'], 'Ján Holp', 'diploma_thesis')
|
|
assert lines[-2:] == ['1. Vypracovanie prehľadu metód.', '2. Implementácia systému.']
|
|
|
|
|
|
def test_graph_list_keeps_all_records_and_missing_values_without_technical_labels(retrieved):
|
|
context = graph_context(retrieved); answer = listing_answer(context)
|
|
before = copy.deepcopy(answer)
|
|
rendered = render_graph_answer(answer, context)
|
|
assert answer_context_issues(rendered, context) == []
|
|
text = rendered['answer']
|
|
assert text.startswith('Ján Holp má v dostupných podkladoch tieto práce a projekty:')
|
|
assert all(f'\n{i}. **' in text for i in range(1, 6))
|
|
assert text.count('názov v zdroji nie je uvedený') == 3
|
|
assert 'Diplomový projekt 2 2020' in text
|
|
assert 'typ práce:' not in text and 'záznam:' not in text
|
|
assert answer == before and rendered['claims'] == answer['claims']
|
|
|
|
|
|
def test_ordinal_numbers_do_not_license_fabricated_years_or_totals(retrieved):
|
|
c = graph_context(retrieved); a = render_graph_answer(listing_answer(c), c)
|
|
assert prose_issues(a) == []
|
|
for suffix in ['\nPráca bola dokončená v roku 2099.', '\nPočet: 98765.', '\n2099. Ďalšia práca.']:
|
|
assert 'unsupported_prose_number' in prose_issues({**a, 'answer': a['answer'] + suffix})
|
|
a['count'] = 98765
|
|
assert 'unsupported_total_count' in answer_context_issues(a, c)
|
|
|
|
|
|
def test_negative_is_a_bounded_absence_not_a_claim_about_real_existence():
|
|
c = {'status': 'no_answer', 'requested_people': ['Ján Holp'], 'facts': [], 'sources': [],
|
|
'complete': True, 'question': 'Akú prácu mal Ján Holp v roku 1999?'}
|
|
a = {'status': 'no_answer', 'claims': [], 'source_urls': []}
|
|
text = render_graph_answer(a, c)['answer']
|
|
assert 'Ján Holp' in text and 'požadované údaje sa v dostupných podkladoch nenašli' in text
|
|
assert 'neexistuje' not in text and '2021' not in text
|
|
assert answer_context_issues(render_graph_answer(a, c), c) == []
|