212 lines
9.0 KiB
Python
212 lines
9.0 KiB
Python
"""Natural presentation must preserve evidence, modality, binding and guards."""
|
|
|
|
import copy
|
|
|
|
import pytest
|
|
|
|
from tools.answers.context_policy import answer_context_issues, prose_issues
|
|
from tools.answers.graph_context import graph_context
|
|
from tools.answers.presentation import content_lines, fact_sentences, goal_text, owner_in_question
|
|
from tools.answers.render_answer import render_graph_answer
|
|
from tools.answers.verified_rag import generate_verified_rag
|
|
from test_graph_work_listing import retrieved, listing_answer
|
|
from test_rag_work_content import payload, description, completion, ASSIGNMENT, QUESTION, URL
|
|
|
|
|
|
def test_identity_is_a_sentence_but_year_is_not_a_start_or_completion_date():
|
|
claim = {
|
|
"person": "Ján Holp",
|
|
"work_type": "diploma_thesis",
|
|
"title": "Systém získavania informácií.",
|
|
"year": 2021,
|
|
}
|
|
before = copy.deepcopy(claim)
|
|
text = fact_sentences(claim, "Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?")
|
|
assert text == (
|
|
"Diplomová práca Jána Holpa má názov **Systém získavania informácií** "
|
|
"(rok práce: **2021**)."
|
|
)
|
|
assert not any(word in text for word in ("začal", "dokonč", "vypracovaná"))
|
|
assert claim == before
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"person,question,expected",
|
|
[
|
|
("Ján Holp", "Čomu sa venuje diplomová práca Jána Holpa?", "Jána Holpa"),
|
|
("Eva Nová", "Aké je zadanie diplomovej práce Evy Novej?", "Evy Novej"),
|
|
("Ján Holp", "Aké práce má Ján Holp?", None),
|
|
("Ján Holp", "Aké práce má Martin Jančura?", None),
|
|
("Ján Holp", "Aké je zadanie práce Jána Holpáka?", None),
|
|
],
|
|
)
|
|
def test_display_name_is_bound_to_the_whole_name(person, question, expected):
|
|
assert owner_in_question(person, question) == expected
|
|
|
|
|
|
def test_a_name_without_a_verified_genitive_remains_in_its_original_form():
|
|
text = fact_sentences(
|
|
{"person": "Alex Kim", "work_type": "diploma_thesis", "title": "Vyhľadávanie", "year": 2020}
|
|
)
|
|
assert text == "Alex Kim má diplomovú prácu s názvom **Vyhľadávanie** (rok práce: **2020**)."
|
|
|
|
|
|
def test_content_changes_only_grammar_after_literal_evidence_passes(payload):
|
|
before = copy.deepcopy(payload)
|
|
result = generate_verified_rag(QUESTION, payload, lambda _: completion(description()))
|
|
assert result["accepted_for_delivery"]
|
|
assert (
|
|
"Diplomová práca Jána Holpa má podľa dostupných podkladov tieto ciele:" in result["answer"]
|
|
)
|
|
assert "1. Vypracovanie prehľadu metód ohodnotenia dokumentov" in result["answer"]
|
|
assert (
|
|
"2. Implementácia systému pre získavanie informácií v slovenskom jazyku."
|
|
in result["answer"]
|
|
)
|
|
assert "Vypracujte" not in result["answer"] and "Implementujte" not in result["answer"]
|
|
assert result["attempts"][0]["parsed_answer"]["evidence_quotes"][0]["quote"] == ASSIGNMENT
|
|
assert result["source_urls"] == [URL] and result["complete"] is False
|
|
assert "neúplný" not in result["answer"] and payload == before
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"original,expected",
|
|
[
|
|
(
|
|
"1. Vypracujte prehľad metód iba pre slovenské texty.",
|
|
"Vypracovanie prehľadu metód iba pre slovenské texty.",
|
|
),
|
|
(
|
|
"2. Implementujte jednoduchý systém, v ktorom využijete ohodnotenie dokumentov.",
|
|
"Implementácia jednoduchého systému, v ktorom sa má využiť ohodnotenie dokumentov.",
|
|
),
|
|
(
|
|
"3. Navrhnite a vypracujte experimenty, v ktorých vyhodnotíte vybrané metódy.",
|
|
"Návrh a vypracovanie experimentov s cieľom vyhodnotiť vybrané metódy.",
|
|
),
|
|
(
|
|
"4. Navrhnite možné zlepšenia presnosti vyhľadávania.",
|
|
"Návrh možných zlepšení presnosti vyhľadávania.",
|
|
),
|
|
(
|
|
"Navrhnite a implementujte API bez zmeny databázy.",
|
|
"Navrhnúť a implementovať API bez zmeny databázy.",
|
|
),
|
|
],
|
|
)
|
|
def test_goals_keep_constraints_and_do_not_claim_completion(original, expected):
|
|
assert goal_text(original) == (expected, True)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"text",
|
|
[
|
|
"Implementovaný PageRank a indexovanie webových stránok.",
|
|
"Práca neimplementuje neurónové siete.",
|
|
"Ak bude dostupný korpus, otestuje sa presnosť.",
|
|
"Neimplementujte externé API.",
|
|
],
|
|
)
|
|
def test_progress_negations_and_unknown_prose_are_not_reinterpreted(text):
|
|
assert goal_text(text) == (text, False)
|
|
rendered = "\n".join(content_lines([text], "Ján Holp", "diploma_thesis"))
|
|
assert text in rendered and "tieto ciele" not in rendered
|
|
|
|
|
|
@pytest.mark.parametrize("command", ["vyhodnoťte", "vyhodnotte", "vyhodnnotte"])
|
|
def test_known_coordinated_command_variants_are_natural_without_claiming_completion(command):
|
|
raw = "Preštudujte si spôsoby vyhodnotenia RAG - RAGAS, DeepEval a " + command + " systém."
|
|
text, converted = goal_text(raw)
|
|
assert (
|
|
converted
|
|
and text == "Preštudovať si spôsoby vyhodnotenia RAG - RAGAS, DeepEval a vyhodnotiť systém."
|
|
)
|
|
assert "vyhodnotený" not in text and "dokončený" not in text
|
|
|
|
|
|
def test_goal_formatting_protects_urls_code_attributes_and_conditions():
|
|
raw = (
|
|
"Vypracujte prehľad iba pre slovenské texty a vyhodnnotte systém podľa "
|
|
"https://example.test/implementujte/vyhodnnotte?x=navrhnite "
|
|
's kódom `implementujte("vyhodnnotte")` a <a href="/vyhodnnotte">odkazom</a>.'
|
|
)
|
|
text, converted = goal_text(raw)
|
|
assert converted and "Vypracovať prehľad iba pre slovenské texty a vyhodnotiť systém" in text
|
|
assert "https://example.test/implementujte/vyhodnnotte?x=navrhnite" in text
|
|
assert '`implementujte("vyhodnnotte")`' in text and '<a href="/vyhodnnotte">' in text
|
|
|
|
|
|
@pytest.mark.parametrize("complete", [True, False])
|
|
def test_shared_document_authors_have_a_grammatical_bounded_summary(complete):
|
|
people = ["Eva Nová", "Boris Nový"]
|
|
urls = ["https://example.test/eva", "https://example.test/boris"]
|
|
claims = [{"person": person, "document_author": "Daniel Hladek"} for person in people]
|
|
c = {
|
|
"status": "ok",
|
|
"facts": [{**claim, "source_url": url} for claim, url in zip(claims, urls)],
|
|
"sources": [{"source_url": url} for url in urls],
|
|
"complete": complete,
|
|
"truncated": not complete,
|
|
"shared_properties": [
|
|
{"field": "document_author", "value": "Daniel Hladek", "people": people}
|
|
],
|
|
}
|
|
rendered = render_graph_answer({"status": "ok", "claims": claims, "source_urls": urls}, c)
|
|
prefix = "Spoloční autori dokumentov:" if complete else "Doložení spoloční autori dokumentov:"
|
|
assert rendered["answer"].startswith(prefix)
|
|
assert answer_context_issues(rendered, c) == []
|
|
|
|
|
|
def test_assignment_heading_does_not_keep_imperative_style():
|
|
lines = content_lines(
|
|
["*Zadanie diplomovej práce*:", "1. Vypracujte prehľad metód.", "2. Implementujte systém."],
|
|
"Ján Holp",
|
|
"diploma_thesis",
|
|
)
|
|
assert lines[-2:] == ["1. Vypracovanie prehľadu metód.", "2. Implementácia systému."]
|
|
|
|
|
|
def test_graph_list_keeps_all_records_and_missing_values_without_technical_labels(retrieved):
|
|
context = graph_context(retrieved)
|
|
answer = listing_answer(context)
|
|
before = copy.deepcopy(answer)
|
|
rendered = render_graph_answer(answer, context)
|
|
assert answer_context_issues(rendered, context) == []
|
|
text = rendered["answer"]
|
|
assert text.startswith("Ján Holp má v dostupných podkladoch tieto práce a projekty:")
|
|
assert all(f"\n{i}. **" in text for i in range(1, 6))
|
|
assert text.count("názov v zdroji nie je uvedený") == 3
|
|
assert "Diplomový projekt 2 2020" in text
|
|
assert "typ práce:" not in text and "záznam:" not in text
|
|
assert answer == before and rendered["claims"] == answer["claims"]
|
|
|
|
|
|
def test_ordinal_numbers_do_not_license_fabricated_years_or_totals(retrieved):
|
|
c = graph_context(retrieved)
|
|
a = render_graph_answer(listing_answer(c), c)
|
|
assert prose_issues(a) == []
|
|
for suffix in [
|
|
"\nPráca bola dokončená v roku 2099.",
|
|
"\nPočet: 98765.",
|
|
"\n2099. Ďalšia práca.",
|
|
]:
|
|
assert "unsupported_prose_number" in prose_issues({**a, "answer": a["answer"] + suffix})
|
|
a["count"] = 98765
|
|
assert "unsupported_total_count" in answer_context_issues(a, c)
|
|
|
|
|
|
def test_negative_is_a_bounded_absence_not_a_claim_about_real_existence():
|
|
c = {
|
|
"status": "no_answer",
|
|
"requested_people": ["Ján Holp"],
|
|
"facts": [],
|
|
"sources": [],
|
|
"complete": True,
|
|
"question": "Akú prácu mal Ján Holp v roku 1999?",
|
|
}
|
|
a = {"status": "no_answer", "claims": [], "source_urls": []}
|
|
text = render_graph_answer(a, c)["answer"]
|
|
assert "Ján Holp" in text and "požadované údaje sa v dostupných podkladoch nenašli" in text
|
|
assert "neexistuje" not in text and "2021" not in text
|
|
assert answer_context_issues(render_graph_answer(a, c), c) == []
|