292 lines
11 KiB
Python
292 lines
11 KiB
Python
"""Academic joins: roles, record dates and outcomes must belong to one work."""
|
|
|
|
import pytest
|
|
|
|
from tools.knowledge.sources import profile_units, advisor_name
|
|
from tools.knowledge.academic import answer_academic, completion_event, academic_intent
|
|
from tools.knowledge.query import answer_from_index, intent
|
|
|
|
|
|
def index(*profiles):
|
|
rows = []
|
|
for number, (name, text) in enumerate(profiles):
|
|
rows.extend(
|
|
profile_units(
|
|
{
|
|
"title": name,
|
|
"path": f"pages/students/2022/student_{number}/README.md",
|
|
"content": text,
|
|
},
|
|
include_records=True,
|
|
)
|
|
)
|
|
return {
|
|
"version": "fixture",
|
|
"index_version": {"version": "fixture"},
|
|
"academic_schema": 1,
|
|
"units": rows,
|
|
}
|
|
|
|
|
|
@pytest.fixture(params=["rag", "graphrag"])
|
|
def engine(request):
|
|
return request.param
|
|
|
|
|
|
def test_supervisor_and_consultant_are_distinct_and_missing_role_is_not_invented(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Bakalárska práca 2026\nkonzultant Matúš Čavojský\n"
|
|
"Predbežný názov:\nOrchestrácia modelov pomocou MCP\n",
|
|
)
|
|
)
|
|
r = answer_from_index(
|
|
"Kto je vedúci a konzultant bakalárskej práce Aleny Novej?", data, engine=engine
|
|
)
|
|
assert r["status"] == "ok"
|
|
assert "Konzultant: Matúš Čavojský" in r["answer"]
|
|
assert "Vedúci práce: v dostupných podkladoch nie je uvedený" in r["answer"]
|
|
assert "Orchestrácia modelov" in r["answer"] and r["source_urls"]
|
|
|
|
|
|
def test_profile_level_supervisor_is_not_assigned_to_every_work(engine):
|
|
data = index(
|
|
("Alena Nová", "Vedúci: Daniel Hládek\n# Diplomová práca 2024\nNázov: Overené grafy\n")
|
|
)
|
|
r = answer_academic(
|
|
"Kto je vedúci diplomovej práce Aleny Novej z roku 2024?", data, engine=engine
|
|
)
|
|
assert "Vedúci práce: Daniel" not in r["answer"]
|
|
assert "nie je uvedený" in r["answer"]
|
|
|
|
|
|
def test_latest_meeting_is_selected_by_date_even_when_profile_order_is_reversed(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2026\nStretnutie 5.5.2026\nStav:\n"
|
|
"- Implementovaný systém RAG.\nÚlohy:\n- Vyhodnoťte model.\n"
|
|
"Stretnutie 1.2.2026\nStav:\n- Vytvorený prvý návrh systému.\n",
|
|
)
|
|
)
|
|
r = answer_academic(
|
|
"Kedy mala Alena Nová posledné stretnutie a o čom bolo?", data, engine=engine
|
|
)
|
|
assert r["status"] == "ok" and "2026-05-05" in r["answer"]
|
|
assert "Implementovaný" in r["answer"] and "Plán:" in r["answer"]
|
|
assert "prvý návrh" not in r["answer"]
|
|
assert all(e.get("date") == "2026-05-05" for e in r["evidence"])
|
|
|
|
|
|
def test_empty_latest_meeting_does_not_borrow_older_meeting_contents(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2026\nStretnutie 1.2.2026\n"
|
|
"Stav:\n- Implementovaný systém RAG.\nStretnutie 5.5.2026\n",
|
|
)
|
|
)
|
|
r = answer_academic("Kedy mala Alena Nová posledné stretnutie?", data, engine=engine)
|
|
assert "2026-05-05" in r["answer"] and "Obsah tohto stretnutia nie je" in r["answer"]
|
|
assert "Implementovaný" not in r["answer"]
|
|
|
|
|
|
def test_partial_date_does_not_inherit_year_or_establish_absolute_latest_meeting(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2027\nStretnutie 5.5.2026\nStav:\n"
|
|
"- Implementovaný systém RAG.\nStretnutie 7.11.\nStav:\n- Urobená nová verzia systému.\n",
|
|
)
|
|
)
|
|
r = answer_academic("Kedy mala Alena Nová posledné stretnutie?", data, engine=engine)
|
|
assert "2026-05-05" in r["answer"] and "či sú novšie" in r["answer"]
|
|
assert "2027-11-07" not in r["answer"] and "2026-11-07" not in r["answer"]
|
|
|
|
|
|
def test_premature_work_note_does_not_prove_failed_defense_or_ended_studies(engine):
|
|
data = index(("Samuel Vasiľ", "# Bakalárska práca 2026\nUkončil predčasne\n"))
|
|
r = answer_academic(
|
|
"Ukončil Samuel Vasiľ štúdium úspešne alebo predčasne?", data, engine=engine
|
|
)
|
|
assert "Ukončil predčasne" in r["answer"]
|
|
assert r["reason"] == "completion_not_documented"
|
|
assert "rozsahom" in r["answer"] and "Nemožno z toho usúdiť, že stále prebieha" in r["answer"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"sentence",
|
|
[
|
|
"Plán ukončiť štúdium v roku 2024.",
|
|
"Mal by úspešne ukončiť diplomovú prácu v roku 2024.",
|
|
"Dokončil text práce v roku 2024.",
|
|
"Dokončil prácu na experimente v roku 2024.",
|
|
"Implementovaný systém RAG.",
|
|
"Nie je potvrdené, že úspešne obhájil diplomovú prácu.",
|
|
"Kolega úspešne obhájil diplomovú prácu v roku 2024.",
|
|
"Úspešne obhájil diplomovú prácu, ale neukončil štúdium.",
|
|
],
|
|
)
|
|
def test_subtasks_plans_negations_and_other_people_do_not_prove_graduation(sentence):
|
|
row = {"text": sentence, "phase": "progress", "work_type": "diplomova", "work_year": 2024}
|
|
assert completion_event(row) is None
|
|
|
|
|
|
def test_work_heading_year_is_not_a_completion_year(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2024\nVedúci: Daniel Hládek\n"
|
|
"Stretnutie 1.6.2024\nStav:\n- Úspešne obhájila diplomovú prácu.\n",
|
|
)
|
|
)
|
|
r = answer_academic(
|
|
"Ktorí študenti v roku 2024 úspešne ukončili práce pod vedením Daniela Hladeka?",
|
|
data,
|
|
engine=engine,
|
|
)
|
|
assert r["status"] == "no_answer" and r["reason"] == "academic_conditions_not_documented"
|
|
|
|
|
|
def test_completion_supervisor_and_year_join_to_same_person_and_work(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2024\nVedúci: Daniel Hládek\nNázov: Presné grafy\n"
|
|
"Stav:\n- Úspešne obhájila diplomovú prácu v roku 2024.\n",
|
|
),
|
|
(
|
|
"Boris Starý",
|
|
"# Diplomová práca 2024\nVedúci: Ján Staš\nStav:\n"
|
|
"- Úspešne obhájil diplomovú prácu v roku 2024.\n"
|
|
"# Bakalárska práca 2022\nVedúci: Daniel Hládek\n",
|
|
),
|
|
(
|
|
"Cyril Malý",
|
|
"# Diplomová práca 2024\nVedúci: Daniel Hládek\n"
|
|
"Stav:\n- Úspešne obhájil diplomovú prácu v roku 2025.\n",
|
|
),
|
|
)
|
|
r = answer_academic(
|
|
"Ktorí študenti v roku 2024 úspešne ukončili práce pod vedením Daniela Hladeka?",
|
|
data,
|
|
engine=engine,
|
|
)
|
|
assert r["status"] == "ok" and "Alena Nová" in r["answer"] and "Presné grafy" in r["answer"]
|
|
assert "Boris Starý" not in r["answer"] and "Cyril Malý" not in r["answer"]
|
|
assert {e["person"] for e in r["evidence"]} == {"Alena Nová"}
|
|
assert {e["work_label"] for e in r["evidence"]} == {"Diplomová práca 2024"}
|
|
|
|
|
|
def test_submission_is_not_summarized_as_success(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2024\nVedúci: Daniel Hládek\n"
|
|
"Stav:\n- Odovzdala diplomovú prácu v roku 2024.\n",
|
|
)
|
|
)
|
|
r = answer_academic(
|
|
"Ktorí študenti v roku 2024 úspešne ukončili práce pod vedením Daniela Hladeka?",
|
|
data,
|
|
engine=engine,
|
|
)
|
|
assert r["status"] == "no_answer"
|
|
|
|
|
|
def test_actual_conflicting_outcomes_are_reported_instead_of_cherry_picking_success(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2024\nVedúci: Daniel Hládek\nStav:\n"
|
|
"- Úspešne obhájila diplomovú prácu v roku 2024.\n"
|
|
"- Neobhájila diplomovú prácu v roku 2024.\n",
|
|
)
|
|
)
|
|
r = answer_academic(
|
|
"Ktorí študenti v roku 2024 úspešne ukončili práce pod vedením Daniela Hladeka?",
|
|
data,
|
|
engine=engine,
|
|
)
|
|
assert r["status"] == "no_answer" and r["reason"] == "conflicting_completion_evidence"
|
|
assert "rozporné záznamy" in r["answer"]
|
|
|
|
|
|
def test_exact_completion_date_needs_an_explicit_event_date_not_only_event_year(engine):
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2024\nStav:\n" "- Úspešne obhájila diplomovú prácu v roku 2024.\n",
|
|
)
|
|
)
|
|
r = answer_academic("Obhájila Alena Nová diplomovú prácu 5.6.2024?", data, engine=engine)
|
|
assert r["status"] == "no_answer"
|
|
data = index(
|
|
(
|
|
"Alena Nová",
|
|
"# Diplomová práca 2024\nStav:\n" "- Úspešne obhájila diplomovú prácu dňa 5.6.2024.\n",
|
|
)
|
|
)
|
|
r = answer_academic("Obhájila Alena Nová diplomovú prácu 5.6.2024?", data, engine=engine)
|
|
assert r["status"] == "ok" and "5.6.2024" in r["answer"]
|
|
|
|
|
|
def test_limit_is_reported_without_claiming_an_exhaustive_list(engine):
|
|
data = index(
|
|
*[
|
|
(
|
|
name,
|
|
"# Diplomová práca 2024\nVedúci: Daniel Hládek\n"
|
|
"Stav:\n- Úspešne obhájil diplomovú prácu v roku 2024.\n",
|
|
)
|
|
for name in ["Alena Nová", "Boris Starý"]
|
|
]
|
|
)
|
|
r = answer_academic(
|
|
"Ktorí študenti v roku 2024 úspešne ukončili práce pod vedením Daniela Hladeka?",
|
|
data,
|
|
engine=engine,
|
|
limit=1,
|
|
)
|
|
assert r["truncated"] and "obmedzený výber" in r["answer"]
|
|
assert len({e["person"] for e in r["evidence"]}) == 1
|
|
|
|
|
|
def test_old_checkpoint_does_not_silently_answer_new_intents(engine):
|
|
data = index(("Alena Nová", "# Diplomová práca 2024\nVedúci: Daniel Hládek\n"))
|
|
data.pop("academic_schema")
|
|
r = answer_academic("Kto je vedúci práce Aleny Novej?", data, engine=engine)
|
|
assert r["status"] == "error" and r["reason"] == "content_index_unavailable"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question,mode",
|
|
[
|
|
("Kto je vedúci práce Jána Holpa?", "advisors"),
|
|
("Kto vedie diplomovú prácu Jána Holpa?", "advisors"),
|
|
("Kedy mal Ján Holp naposledy stretnutie?", "meeting"),
|
|
("Ukončil Samuel Vasiľ štúdium úspešne?", "completion"),
|
|
],
|
|
)
|
|
def test_academic_intents_use_the_shared_backend(question, mode):
|
|
assert intent(question) == academic_intent(question) == mode
|
|
|
|
|
|
def test_completed_subtasks_are_not_routed_to_degree_completion():
|
|
assert academic_intent("Aké experimenty dokončil Ján Holp?") is None
|
|
assert academic_intent("Ktoré úlohy dokončil Ján Holp?") is None
|
|
|
|
|
|
def test_colloquial_thesis_name_keeps_the_work_type_instead_of_listing_all_works(engine):
|
|
data = index(
|
|
(
|
|
"Ján Holp",
|
|
"# Diplomová práca 2021\nVedúci: Daniel Hládek\n"
|
|
"# Bakalárska práca 2019\nVedúci: Ján Staš\n",
|
|
)
|
|
)
|
|
r = answer_academic("Kto vedie diplomovku Jána Holpa?", data, engine=engine)
|
|
assert r["status"] == "ok" and "Vedúci práce: Daniel Hládek" in r["answer"]
|
|
assert "Bakalárska práca" not in r["answer"] and "Ján Staš" not in r["answer"]
|