245 lines
10 KiB
Python
245 lines
10 KiB
Python
"""Concise presentation must keep factual bindings, qualifications and gaps."""
|
|
|
|
import copy
|
|
import sqlite3
|
|
|
|
import pytest
|
|
|
|
from test_quality_block import corpus, prepare, complete
|
|
from test_evidence_selection import update_eva, selection
|
|
from tools.answers.context_policy import answer_context_issues
|
|
from tools.answers.content_selection import facet_lines
|
|
from tools.answers.graph_context import graph_context
|
|
from tools.answers.presentation import comparison_table
|
|
from tools.answers.rag_evidence import rag_evidence_context
|
|
from tools.answers.render_answer import render_graph_answer
|
|
from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues
|
|
|
|
|
|
def test_missing_aspects_share_one_specific_notice_without_empty_sections(corpus):
|
|
update_eva(corpus, "Stav:\n- Implementovaný parser.")
|
|
q = "Aké výsledky, problémy a plánované experimenty má diplomová práca Evy Novej?"
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
result = generate_verified_rag(q, p, complete(selection(c)))
|
|
assert result["accepted_for_delivery"] and result["answer_coverage"]["status"] == "partial"
|
|
text = result["answer"]
|
|
assert "Implementovaný parser." in text
|
|
assert text.count("V poskytnutých podkladoch chýbajú údaje") == 1
|
|
assert "riešených problémoch, plánovaných úlohách a plánovaných experimentoch" in text
|
|
assert "**Riešené problémy:**" not in text and "**Plánované experimenty" not in text
|
|
assert "Táto časť odpovede" not in text and "\n\n\n" not in text
|
|
|
|
|
|
def test_short_comparison_aligns_people_and_different_results(corpus):
|
|
q = "Porovnaj výsledky Evy Novej a Borisa Nového."
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
a = selection(c)
|
|
before = copy.deepcopy((a, c))
|
|
lines = facet_lines(a, c)
|
|
text = "\n".join(lines)
|
|
assert text.startswith(
|
|
"| Oblasť | Eva Nová — diplomová práca (2024) | Boris Nový — bakalárska práca (2023) |"
|
|
)
|
|
row = next(line for line in lines if line.startswith("| Zaznamenané výsledky |"))
|
|
assert "BERT" in row.split("|")[2] and "Python" in row.split("|")[3]
|
|
assert "Konzultácia" not in text and "Pripravte experiment" not in text
|
|
assert "lepš" not in text and (a, c) == before
|
|
r = generate_verified_rag(q, p, complete(a))
|
|
assert r["accepted_for_delivery"] and len(r["source_urls"]) == 2
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"literal",
|
|
[
|
|
"Implementovaný parser. Výsledok platí iba pre malý korpus. " + ("Technický opis. " * 70),
|
|
"Implementovaný parser pre `a | b` iba na testovacích dátach.",
|
|
"Implementovaný parser podľa https://example.test/method iba na testovacích dátach.",
|
|
],
|
|
)
|
|
def test_complex_comparisons_fall_back_without_losing_qualifiers_or_literals(corpus, literal):
|
|
update_eva(corpus, "Výsledky:\n- " + literal)
|
|
q = "Porovnaj výsledky Evy Novej a Borisa Nového."
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
a = selection(c)
|
|
text = "\n".join(facet_lines(a, c))
|
|
assert not text.startswith("| Oblasť |") and literal.strip() in text
|
|
assert "Boris Nový" in text and "Python" in text
|
|
assert all("Ďalšie doložené" not in line for line in text.splitlines())
|
|
|
|
|
|
def test_shared_excerpt_is_not_repeated_under_each_matching_facet(corpus):
|
|
update_eva(corpus, "Výsledky:\n- Vyhodnotená presnosť modelu v experimente.")
|
|
q = "Aké experimenty a výsledky má diplomová práca Evy Novej?"
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
result = generate_verified_rag(q, p, complete(selection(c)))
|
|
assert result["accepted_for_delivery"]
|
|
assert result["answer"].count("Vyhodnotená presnosť modelu v experimente.") == 1
|
|
assert "chýbajú údaje" not in result["answer"] and "Doložené vyššie" not in result["answer"]
|
|
assert (
|
|
"Zaznamenaný výsledok:" in result["answer"]
|
|
or "**Zaznamenané výsledky:**" in result["answer"]
|
|
)
|
|
|
|
|
|
def test_multiline_table_is_bounded_and_does_not_rewrite_source_html():
|
|
table = comparison_table(
|
|
["Oblasť", "Eva", "Boris"], [("Výsledky", ["Prvý výsledok.\nDruhý výsledok.", "Parser."])]
|
|
)
|
|
assert table and "Prvý výsledok.<br>Druhý výsledok." in table[-1]
|
|
assert comparison_table(["Oblasť", "Eva"], [("Výsledky", ["<b>Prvý výsledok.</b>"])]) is None
|
|
|
|
|
|
def test_shared_result_precedes_people_and_citations_without_repeating_tags():
|
|
people = ["Ján Pták", "Ján Malinovský"]
|
|
urls = ["https://example.test/ptak", "https://example.test/malinovsky"]
|
|
claims = [{"person": person, "keyword": tag} for person in people for tag in ["NLP", "RAG"]]
|
|
c = {
|
|
"status": "ok",
|
|
"complete": True,
|
|
"truncated": False,
|
|
"facts": [{**claim, "source_url": urls[people.index(claim["person"])]} for claim in claims],
|
|
"sources": [{"source_url": u} for u in urls],
|
|
"requested_people": people,
|
|
"shared_properties": [
|
|
{"field": "keyword", "value": tag, "people": people} for tag in ["NLP", "RAG"]
|
|
],
|
|
}
|
|
a = {"status": "ok", "claims": claims, "source_urls": urls}
|
|
r = render_graph_answer(a, c)
|
|
text = r["answer"]
|
|
assert text.startswith("Spoločné kľúčové slová: **NLP**, **RAG**.")
|
|
assert text.index("RAG") < text.index("Doložené pri:") < text.index("Zdroje:")
|
|
assert all(text.count("**" + tag + "**") == 1 for tag in ["NLP", "RAG"])
|
|
assert all(person in text for person in people)
|
|
assert not answer_context_issues(r, c)
|
|
|
|
|
|
def test_negative_names_failed_year_but_does_not_license_positive_year_claims():
|
|
c = {
|
|
"status": "no_answer",
|
|
"facts": [],
|
|
"sources": [],
|
|
"requested_people": ["Martin Šarišský"],
|
|
"requested_years": [1999],
|
|
"complete": True,
|
|
}
|
|
a = render_graph_answer({"status": "no_answer", "claims": [], "source_urls": []}, c)
|
|
assert "Martin Šarišský" in a["answer"] and "z roku **1999**" in a["answer"]
|
|
assert not answer_context_issues(a, c)
|
|
for year in [1999, 2099]:
|
|
bad = {**a, "answer": a["answer"] + f" Práca bola dokončená v roku {year}."}
|
|
assert "unsupported_prose_number" in answer_context_issues(bad, c)
|
|
|
|
|
|
def test_start_year_is_not_rendered_as_a_missing_academic_work_year():
|
|
c = graph_context(
|
|
{
|
|
"status": "no_answer",
|
|
"reason": "not_found",
|
|
"facts": [],
|
|
"sources": [],
|
|
"aggregation": None,
|
|
"complete": True,
|
|
"constraints": {"years": [1999], "mentions_start_year": True},
|
|
"resolved_entities": [{"kind": "Person", "name": "Martin Šarišský"}],
|
|
}
|
|
)
|
|
a = render_graph_answer({"status": "no_answer", "claims": [], "source_urls": []}, c)
|
|
assert "záznam práce" not in a["answer"] and "1999" not in a["answer"]
|
|
|
|
|
|
def test_partial_overview_retains_supported_person_and_names_missing_description(corpus):
|
|
db, raw = corpus
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"UPDATE chunks SET text='Názov: Ukážkový názov.' WHERE document_path LIKE '%boris_novy%'"
|
|
)
|
|
q = "Porovnaj obsah diplomovej práce Evy Novej a bakalárskej práce Borisa Nového."
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
assert not c["requirements"].get("facets")
|
|
passage = next(p for p in c["content_passages"] if p["person"] == "Eva Nová")
|
|
a = {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [],
|
|
"source_urls": [s["source_url"] for s in c["sources"]],
|
|
"evidence_quotes": [
|
|
{
|
|
"source_id": passage["source_id"],
|
|
"passage_id": passage["passage_id"],
|
|
"quote": "Implementujte systém vyhľadávania.",
|
|
}
|
|
],
|
|
}
|
|
assert not rag_answer_issues(a, c)
|
|
result = generate_verified_rag(q, p, complete(a))
|
|
assert result["accepted_for_delivery"] and result["answer_coverage"]["status"] == "partial"
|
|
assert "Implementácia systému vyhľadávania" in result["answer"]
|
|
assert (
|
|
"Boris Nový: v poskytnutých podkladoch chýba opis požadovanej práce alebo projektu."
|
|
in result["answer"]
|
|
)
|
|
assert "Výpis je neúplný" not in result["answer"]
|
|
|
|
|
|
def test_available_description_cannot_be_silently_omitted(corpus):
|
|
q = "Porovnaj obsah diplomovej práce Evy Novej a bakalárskej práce Borisa Nového."
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
passage = next(p for p in c["content_passages"] if p["person"] == "Eva Nová")
|
|
a = {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [],
|
|
"source_urls": [s["source_url"] for s in c["sources"]],
|
|
"evidence_quotes": [
|
|
{
|
|
"source_id": passage["source_id"],
|
|
"passage_id": passage["passage_id"],
|
|
"quote": "Implementujte systém vyhľadávania.",
|
|
}
|
|
],
|
|
}
|
|
assert "missing_bound_content_evidence" in rag_answer_issues(a, c)
|
|
|
|
|
|
@pytest.mark.parametrize("aspect", ["obsah", "zameranie"])
|
|
def test_overview_comparison_keeps_each_work_in_its_own_column(corpus, aspect):
|
|
q = f"Porovnaj {aspect} diplomovej práce Evy Novej a bakalárskej práce Borisa Nového."
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
a = {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [],
|
|
"source_urls": [s["source_url"] for s in c["sources"]],
|
|
"evidence_quotes": [
|
|
{"source_id": passage["source_id"], "passage_id": passage["passage_id"], "quote": text}
|
|
for person, text in [
|
|
("Eva Nová", "Implementujte systém vyhľadávania."),
|
|
("Boris Nový", "Implementujte parser."),
|
|
]
|
|
for passage in c["content_passages"]
|
|
if passage["person"] == person and text in passage["text"]
|
|
],
|
|
}
|
|
result = generate_verified_rag(q, p, complete(a))
|
|
assert result["accepted_for_delivery"] and result["answer_coverage"]["status"] == "supported"
|
|
assert result["answer"].startswith(
|
|
"| Oblasť | Diplomová práca Evy Novej (2024) | Bakalárska práca Borisa Nového (2023) |"
|
|
)
|
|
row = next(
|
|
line
|
|
for line in result["answer"].splitlines()
|
|
if line.startswith("| Zameranie podľa podkladov |")
|
|
)
|
|
assert "vyhľadávania" in row.split("|")[2] and "parser" in row.split("|")[3]
|
|
assert "Pripravte experiment" not in row and len(result["source_urls"]) == 2
|
|
refused = {**a, "status": "no_answer", "evidence_quotes": [], "source_urls": []}
|
|
assert "rejected_supported_work_description" in rag_answer_issues(refused, c)
|