384 lines
14 KiB
Python
384 lines
14 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from tools.evaluation.evaluate_verified_rag import binding_matches, evaluate_case
|
|
from tools.answers.rag_evidence import rag_evidence_context
|
|
from tools.answers.verified_rag import rag_answer_issues, generate_verified_rag
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
|
|
def payload(text, *, categories=None):
|
|
sources = [
|
|
{
|
|
"source_id": "S1",
|
|
"title": "Alena Nová",
|
|
"document_path": "pages/students/2020/alena_nova/README.md",
|
|
"source_url": "https://example.test/alena",
|
|
"categories": categories or [],
|
|
"text": text,
|
|
}
|
|
]
|
|
return {
|
|
"status": "ok",
|
|
"complete": None,
|
|
"sources": sources,
|
|
"context": build_context_text(sources),
|
|
}
|
|
|
|
|
|
def refusal():
|
|
return {"status": "no_answer", "claims": [], "evidence_quotes": [], "source_urls": []}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question",
|
|
["V ktorom roku začal študovať Alena Nová?", "Aký je rok začiatku štúdia osoby Alena Nová?"],
|
|
)
|
|
def test_explicit_study_start_cannot_be_falsely_refused(question):
|
|
p = payload("Dokument: Alena Nová\n\n*Rok začiatku štúdia:* 2020")
|
|
context = rag_evidence_context(question, p)
|
|
assert context["requirements"]["start_year"]
|
|
assert "rejected_supported_academic_answer" in rag_answer_issues(refusal(), context)
|
|
result = generate_verified_rag(
|
|
question, p, lambda messages: {"choices": [{"message": {"content": json.dumps(refusal())}}]}
|
|
)
|
|
assert not result["accepted_for_delivery"]
|
|
|
|
|
|
def test_study_start_never_comes_from_cohort_or_academic_year():
|
|
c = rag_evidence_context(
|
|
"Kedy začal študovať Alena Nová?",
|
|
payload(
|
|
"Sekcia: Bakalárska práca 2023\nNázov: Overené modelovanie textu", categories=["bp2023"]
|
|
),
|
|
)
|
|
assert not any("start_year" in row for row in c["facts"])
|
|
assert rag_answer_issues(refusal(), c) == []
|
|
|
|
|
|
@pytest.mark.parametrize("year,unsupported", [("3000", False), ("12345", True)])
|
|
def test_explicit_year_is_honored_or_rejected_even_in_pre_rok_form(year, unsupported):
|
|
c = rag_evidence_context(
|
|
"Aký je názov diplomovej práce osoby Alena Nová pre rok " + year + "?",
|
|
payload("Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu"),
|
|
)
|
|
assert c["requirements"]["unsupported_year"] == unsupported
|
|
answer = {
|
|
"status": "ok",
|
|
"claims": [
|
|
{
|
|
"person": "Alena Nová",
|
|
"work_type": "diploma_thesis",
|
|
"year": 2025,
|
|
"title": "Overené modelovanie textu",
|
|
}
|
|
],
|
|
"source_urls": ["https://example.test/alena"],
|
|
}
|
|
assert (
|
|
"unsupported_requested_year" if unsupported else "wrong_requested_academic_year"
|
|
) in rag_answer_issues(answer, c)
|
|
|
|
|
|
def test_multiline_title_and_unlabelled_title_preceding_assignment():
|
|
p = payload(
|
|
"Sekcia: Diplomová práca 2025\nNázov práce:\n\nOverené modelovanie textu\n\nZadanie:\n1. Skúmajte dáta."
|
|
)
|
|
c = rag_evidence_context("Aký je názov diplomovej práce osoby Alena Nová?", p)
|
|
assert any(
|
|
row.get("title") == "Overené modelovanie textu" and row["year"] == 2025
|
|
for row in c["facts"]
|
|
)
|
|
p = payload("Sekcia: Diplomová práca 2025\nOverené modelovanie textu\nCieľ:\nZlepšiť model.")
|
|
assert any(
|
|
row.get("title") == "Overené modelovanie textu"
|
|
for row in rag_evidence_context("Práca?", p)["facts"]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"text",
|
|
[
|
|
"Sekcia: Diplomová práca 2025\nStretnutie po úspešných experimentoch\nStav:\nHotovo.",
|
|
"Sekcia: Diplomová práca 2025\nTéma:\nOverené modelovanie tex\n[OBSAH ZDROJA OREZANÝ]",
|
|
"Sekcia: Diplomová práca 2025\nNázov:\n\nĎALŠÍ DÔKAZ Z ROVNAKEJ ŠTUDENTSKEJ VETVY 2\nSekcia: Diplomová práca 2025\nStretnutie 12.3.",
|
|
],
|
|
)
|
|
def test_notes_clipped_titles_and_frame_labels_never_become_titles(text):
|
|
assert not any(
|
|
row.get("title") for row in rag_evidence_context("Práca?", payload(text))["facts"]
|
|
)
|
|
|
|
|
|
def test_sibling_scope_and_preparation_cannot_inherit_previous_work():
|
|
p = payload(
|
|
"# Diplomová práca 2025\nNázov: Overené modelovanie textu\n"
|
|
"# Poznámky\nNázov: Úplne iný dokument\n"
|
|
"# Príprava na diplomový projekt 2026\nNázov: Ďalší prípravný dokument"
|
|
)
|
|
c = rag_evidence_context("Aký je názov diplomovej práce osoby Alena Nová?", p)
|
|
titles = {row.get("title") for row in c["facts"] if row.get("work_type")}
|
|
assert titles == {"Overené modelovanie textu"}
|
|
|
|
|
|
def test_typed_root_proposal_needs_one_matching_academic_scope():
|
|
p = payload(
|
|
"Dokument: Alena Nová\nNávrh na názov DP:\nOverené modelovanie textu\n"
|
|
"Sekcia: Diplomová práca 2025\nStretnutie 12.3."
|
|
)
|
|
c = rag_evidence_context("Práca?", p)
|
|
assert any(
|
|
r.get("title") == "Overené modelovanie textu" and r["year"] == 2025 for r in c["facts"]
|
|
)
|
|
p["sources"][0]["text"] += "\nSekcia: Diplomová práca 2024\nStretnutie 5.3."
|
|
p["context"] = build_context_text(p["sources"])
|
|
assert not any(r.get("title") for r in rag_evidence_context("Práca?", p)["facts"])
|
|
|
|
|
|
def test_quote_from_another_student_or_only_metadata_is_not_an_answer():
|
|
p = payload("Rok začiatku štúdia: 2020\nPoužíva Python.")
|
|
other = {
|
|
**p["sources"][0],
|
|
"source_id": "S2",
|
|
"title": "Boris Nový",
|
|
"document_path": "pages/students/2020/boris_novy/README.md",
|
|
"source_url": "https://example.test/boris",
|
|
"text": "Používa Rust.",
|
|
}
|
|
p["sources"].append(other)
|
|
p["context"] = build_context_text(p["sources"])
|
|
c = rag_evidence_context("Aký jazyk používa Alena Nová?", p)
|
|
a = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"source_urls": ["https://example.test/boris"],
|
|
"evidence_quotes": [{"source_id": "S2", "quote": "Používa Rust."}],
|
|
}
|
|
assert "unrequested_quote_person" in rag_answer_issues(a, c)
|
|
a.update(
|
|
source_urls=["https://example.test/alena"],
|
|
evidence_quotes=[{"source_id": "S1", "quote": "Názov dokumentu: Alena Nová"}],
|
|
)
|
|
assert "metadata_quote_does_not_answer_fact" in rag_answer_issues(a, c)
|
|
|
|
|
|
def test_evaluator_rejects_cross_person_join_wrong_tuple_and_false_refusal():
|
|
expected = {
|
|
"person": "Alena Nová",
|
|
"title": "Overené modelovanie textu",
|
|
"year": 2025,
|
|
"work_type": "diploma_thesis",
|
|
}
|
|
assert not binding_matches(
|
|
expected,
|
|
[
|
|
{"person": "Alena Nová", "title": expected["title"]},
|
|
{"person": "Boris Nový", "year": 2025, "work_type": "diploma_thesis"},
|
|
],
|
|
)
|
|
p = payload("Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu")
|
|
case = {
|
|
"id": "fixture",
|
|
"category": "fact_title",
|
|
"question": "Aký je názov diplomovej práce osoby Alena Nová?",
|
|
"should_answer": True,
|
|
"expected_bindings": [expected],
|
|
"expected_source_urls": ["https://example.test/alena"],
|
|
"expected_answer_contains": ["Overené modelovanie textu"],
|
|
}
|
|
refused = {
|
|
"status": "no_answer",
|
|
"accepted_for_delivery": True,
|
|
"answer": "Nenájdené.",
|
|
"claims": [],
|
|
"source_urls": [],
|
|
"attempts": [],
|
|
}
|
|
assert "false_no_answer" in evaluate_case(case, p, refused)["issues"]
|
|
bad = {
|
|
**refused,
|
|
"status": "ok",
|
|
"answer": "Overené modelovanie textu",
|
|
"source_urls": case["expected_source_urls"],
|
|
"claims": [{**expected, "year": 2024}],
|
|
}
|
|
assert "unsupported_person_work_type_year_binding" in evaluate_case(case, p, bad)["issues"]
|
|
|
|
|
|
def test_evaluator_accepts_extra_supported_project_but_rejects_wrong_rendered_year():
|
|
from tools.answers.verified_rag import render_rag_answer
|
|
|
|
p = payload(
|
|
"Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu\n"
|
|
"Sekcia: Diplomový projekt 2024\nStretnutie 12.3."
|
|
)
|
|
question = "Aký je rok práce osoby Alena Nová?"
|
|
context = rag_evidence_context(question, p)
|
|
claims = [
|
|
{"person": "Alena Nová", "work_type": "diploma_thesis", "year": 2025},
|
|
{"person": "Alena Nová", "work_type": "diploma_project", "year": 2024},
|
|
]
|
|
parsed = {
|
|
"status": "ok",
|
|
"claims": claims,
|
|
"source_urls": ["https://example.test/alena"],
|
|
"evidence_quotes": [],
|
|
}
|
|
delivered = {
|
|
**parsed,
|
|
"accepted_for_delivery": True,
|
|
"answer": render_rag_answer(parsed, context),
|
|
"attempts": [{"parsed_answer": parsed, "issues": []}],
|
|
}
|
|
case = {
|
|
"id": "fixture",
|
|
"category": "fact_work_year",
|
|
"question": question,
|
|
"should_answer": True,
|
|
"expected_source_urls": parsed["source_urls"],
|
|
"expected_answer_contains": ["2025"],
|
|
"expected_bindings": [
|
|
{"person": "Alena Nová", "work_type": "diploma_thesis", "year": 2025}
|
|
],
|
|
}
|
|
assert evaluate_case(case, p, delivered)["passed"]
|
|
delivered["answer"] += "\nPráca bola ukončená v roku 2099."
|
|
assert (
|
|
"delivered_text_differs_from_verified_renderer"
|
|
in evaluate_case(case, p, delivered)["issues"]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"reference",
|
|
[
|
|
"https://library.example/docs",
|
|
"/topics/library",
|
|
"../manual",
|
|
"#installation",
|
|
"[guide]: /guide",
|
|
],
|
|
)
|
|
def test_literal_source_references_are_allowed_without_replacing_wiki_citation(reference):
|
|
from tools.answers.verified_rag import quoted_reference_urls, render_rag_answer
|
|
from tools.answers.context_policy import citation_check, allowed_urls
|
|
|
|
quote = "Dokumentácia: " + reference if not reference.startswith("[guide]") else reference
|
|
if reference.startswith(("/", ".", "#")):
|
|
quote = "[Dokumentácia](" + reference + ")"
|
|
c = rag_evidence_context("Aká dokumentácia je uvedená?", payload(quote))
|
|
a = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"source_urls": ["https://example.test/alena"],
|
|
"evidence_quotes": [{"source_id": "S1", "quote": quote}],
|
|
}
|
|
assert rag_answer_issues(a, c) == []
|
|
rendered = {**a, "answer": render_rag_answer(a, c)}
|
|
assert citation_check(rendered, allowed_urls(c) | quoted_reference_urls(a, c))["citations_ok"]
|
|
refs = quoted_reference_urls(a, c)
|
|
assert refs
|
|
a["source_urls"].extend(refs)
|
|
assert "unsupported_citation" in rag_answer_issues(a, c)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"quote",
|
|
["Pozri https://unknown.example/doc", "[manuál](/unknown)", '<a href="/unknown">manuál</a>'],
|
|
)
|
|
def test_quote_cannot_borrow_an_unknown_or_other_source_reference(quote):
|
|
p = payload("Presný obsah bez odkazu.")
|
|
p["sources"].append(
|
|
{
|
|
**p["sources"][0],
|
|
"source_id": "S2",
|
|
"title": "Iný dokument",
|
|
"source_url": "https://example.test/other",
|
|
"text": quote,
|
|
}
|
|
)
|
|
p["context"] = build_context_text(p["sources"])
|
|
c = rag_evidence_context("Aký je obsah dokumentu?", p)
|
|
a = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"source_urls": ["https://example.test/alena"],
|
|
"evidence_quotes": [{"source_id": "S1", "quote": quote}],
|
|
}
|
|
assert "unsupported_quote_citation" in rag_answer_issues(a, c)
|
|
|
|
|
|
def test_active_url_schemes_and_noncontiguous_quotes_are_rejected():
|
|
c = rag_evidence_context(
|
|
"Aký je obsah dokumentu?",
|
|
payload("[odkaz](javascript:alert(1))\nPrvý fakt.\nMedziľahlý text.\nDruhý fakt."),
|
|
)
|
|
a = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"source_urls": ["https://example.test/alena"],
|
|
"evidence_quotes": [{"source_id": "S1", "quote": "[odkaz](javascript:alert(1))"}],
|
|
}
|
|
assert "unsupported_quote_citation" in rag_answer_issues(a, c)
|
|
a["evidence_quotes"] = [{"source_id": "S1", "quote": "Prvý fakt.\nDruhý fakt."}]
|
|
assert "unsupported_evidence_quote" in rag_answer_issues(a, c)
|
|
|
|
|
|
def test_general_question_repairs_unrequested_claims_and_sends_only_quote_evidence():
|
|
p = payload("Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu\nPoužíva Python.")
|
|
bad = {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [{"person": "Alena Nová", "work_type": "diploma_thesis", "year": 2025}],
|
|
"evidence_quotes": [{"source_id": "S1", "quote": "Používa Python."}],
|
|
"source_urls": ["https://example.test/alena"],
|
|
}
|
|
good = {**bad, "claims": []}
|
|
outputs = [bad, good]
|
|
calls = []
|
|
|
|
def complete(messages):
|
|
calls.append(json.loads(messages[1]["content"]))
|
|
return {"choices": [{"message": {"content": json.dumps(outputs[len(calls) - 1])}}]}
|
|
|
|
delivered = generate_verified_rag("Aký jazyk používa Alena Nová?", p, complete)
|
|
assert delivered["accepted_for_delivery"] and len(calls) == 2
|
|
assert "unrequested_structured_claims" in delivered["attempts"][0]["issues"]
|
|
assert all("facts" not in call["context"] for call in calls)
|
|
assert delivered["claims"] == [] and "Používa Python." in delivered["answer"]
|
|
|
|
|
|
def test_evaluator_accepts_exact_quoted_reference_but_rejects_invented_rendered_link():
|
|
from tools.answers.verified_rag import render_rag_answer
|
|
|
|
p = payload("Knižnica Library: https://library.example/docs")
|
|
question = "Aká knižnica je uvedená?"
|
|
c = rag_evidence_context(question, p)
|
|
parsed = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"source_urls": ["https://example.test/alena"],
|
|
"evidence_quotes": [
|
|
{"source_id": "S1", "quote": "Knižnica Library: https://library.example/docs"}
|
|
],
|
|
}
|
|
delivered = {
|
|
**parsed,
|
|
"accepted_for_delivery": True,
|
|
"answer": render_rag_answer(parsed, c),
|
|
"attempts": [{"parsed_answer": parsed, "issues": []}],
|
|
}
|
|
case = {
|
|
"id": "reference",
|
|
"category": "specific_fact",
|
|
"question": question,
|
|
"should_answer": True,
|
|
"expected_bindings": [],
|
|
"expected_source_urls": parsed["source_urls"],
|
|
"expected_answer_contains": ["Library"],
|
|
}
|
|
assert evaluate_case(case, p, delivered)["passed"]
|
|
delivered["answer"] += " https://invented.example/"
|
|
assert "delivered_unsupported_citation" in evaluate_case(case, p, delivered)["issues"]
|