244 lines
9.7 KiB
Python
244 lines
9.7 KiB
Python
"""Exact/both/negated work selection and proof of actual use, not future tasks."""
|
|
|
|
import json
|
|
import sqlite3
|
|
|
|
import pytest
|
|
|
|
from tools.answers.rag_preparation import prepare_named_content
|
|
from tools.answers.rag_evidence import rag_evidence_context, project_number, content_sections
|
|
from tools.answers.verified_rag import generate_verified_rag
|
|
from tools.answers.content_selection import facets_for
|
|
from tools.retrieval.work_selector import parse_work_selector
|
|
from tools.retrieval.question_constraints import requested_years, unsupported_constraint
|
|
from tools.answers import answer_delivery
|
|
from tools.rag.llm_context import compact_rag_payload
|
|
|
|
|
|
@pytest.fixture
|
|
def projects(tmp_path):
|
|
db = tmp_path / "index.sqlite"
|
|
path = "pages/students/2016/eva_nova/README.md"
|
|
with sqlite3.connect(db) as c:
|
|
c.execute("CREATE TABLE documents(path,title,author,published)")
|
|
c.execute(
|
|
"CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)"
|
|
)
|
|
c.execute("INSERT INTO documents VALUES(?,?,?,?)", (path, "Eva Nová", "Autor dokumentu", 1))
|
|
for i, (label, text) in enumerate(
|
|
[
|
|
(
|
|
"Diplomový projekt 2020",
|
|
"Hotové:\n- Funguje databáza SQLite.\nÚlohy na ďalšie stretnutie:\n- Naštudujte si ako sa používa Redis.",
|
|
),
|
|
(
|
|
"Diplomový projekt 2 2020",
|
|
"Stav:\n- Implementovaný parser pomocou Python.\nÚlohy na ďalšie stretnutie:\n- urobte návrh metódy PageRank.\nDo ďalšieho stretnutia:\n- Naštudujte si ako sa používa Promise.",
|
|
),
|
|
]
|
|
):
|
|
c.execute(
|
|
"INSERT INTO chunks VALUES(?,?,?,?,?)",
|
|
(path, text, json.dumps([["Eva Nová", label]]), 1, i),
|
|
)
|
|
return db
|
|
|
|
|
|
def prepare(db, q):
|
|
raw = {
|
|
"status": "ok",
|
|
"reason": None,
|
|
"sources": [],
|
|
"context": "",
|
|
"complete": False,
|
|
"truncated": True,
|
|
"truncation_reasons": ["result_limit"],
|
|
}
|
|
return compact_rag_payload(prepare_named_content(db, q, raw))
|
|
|
|
|
|
def select(messages):
|
|
context = json.loads(messages[1]["content"])["context"]
|
|
a = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"source_urls": [],
|
|
"evidence_ids": [u["evidence_id"] for u in context.get("content_candidates", [])],
|
|
}
|
|
return {"choices": [{"message": {"content": json.dumps(a)}}]}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"phrase",
|
|
[
|
|
'pri práci "Diplomový projekt 2020" (nie Diplomový projekt 2)',
|
|
"pri nečíslovanom diplomovom projekte v roku 2020",
|
|
"pri diplomovom projekte bez čísla v roku 2020",
|
|
"pri diplomovom projekte v roku 2020 okrem projektu 2",
|
|
],
|
|
)
|
|
def test_exact_unnumbered_and_excluded_number_select_the_actual_work(projects, phrase):
|
|
q = "Aké metódy používala Eva Nová " + phrase + "?"
|
|
assert not unsupported_constraint(q) and requested_years(q) == [2020]
|
|
p = prepare(projects, q)
|
|
assert p["status"] == "ok"
|
|
assert [s["work_label"] for s in p["work_scopes"]] == ["Diplomový projekt 2020"]
|
|
r = generate_verified_rag(q, p, select)
|
|
assert r["accepted_for_delivery"] and "SQLite" in r["answer"]
|
|
assert all(x not in r["answer"] for x in ["Python", "PageRank", "Promise", "Redis"])
|
|
|
|
|
|
def test_numbered_work_keeps_actual_methods_and_rejects_future_task_evidence(projects):
|
|
q = "Aké metódy používala Eva Nová pri diplomovom projekte 2 v roku 2020?"
|
|
p = prepare(projects, q)
|
|
c = rag_evidence_context(q, p)
|
|
assert all(u["phase"] not in {"planned", "assignment"} for u in c["content_candidates"])
|
|
r = generate_verified_rag(q, p, select)
|
|
assert r["accepted_for_delivery"] and "Python" in r["answer"]
|
|
assert (
|
|
"PageRank" not in r["answer"]
|
|
and "Promise" not in r["answer"]
|
|
and "SQLite" not in r["answer"]
|
|
)
|
|
bad = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"source_urls": [s["source_url"] for s in c["sources"]],
|
|
"evidence_quotes": [
|
|
{"source_id": c["sources"][0]["source_id"], "quote": "- urobte návrh metódy PageRank."}
|
|
],
|
|
}
|
|
r = generate_verified_rag(
|
|
q, p, lambda _: {"choices": [{"message": {"content": json.dumps(bad)}}]}
|
|
)
|
|
assert (
|
|
not r["accepted_for_delivery"]
|
|
and "quote_outside_delivered_candidates" in r["attempts"][-1]["issues"]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("word", ["oboch", "obidvoch", "všetkých"])
|
|
def test_each_requested_work_has_its_own_methods_and_heading(projects, word):
|
|
q = f"Aké metódy používala Eva Nová pri {word} diplomových projektoch v roku 2020?"
|
|
p = prepare(projects, q)
|
|
c = rag_evidence_context(q, p)
|
|
assert c["requirements"]["separate_works"] and len(p["work_scopes"]) == 2
|
|
r = generate_verified_rag(q, p, select)
|
|
assert r["accepted_for_delivery"] and r["answer"].count("**Eva Nová — Diplomový projekt") == 2
|
|
first, second = r["answer"].split("**Eva Nová — Diplomový projekt 2 2020")
|
|
assert "SQLite" in first and "Python" not in first and "Python" in second
|
|
assert "PageRank" not in r["answer"] and "Promise" not in r["answer"]
|
|
|
|
|
|
def test_omitting_a_second_work_with_evidence_is_rejected_even_for_the_same_person_and_year(
|
|
projects,
|
|
):
|
|
q = "Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?"
|
|
p = prepare(projects, q)
|
|
|
|
def incomplete(messages):
|
|
c = json.loads(messages[1]["content"])["context"]
|
|
a = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"source_urls": [],
|
|
"evidence_ids": [
|
|
u["evidence_id"]
|
|
for u in c["content_candidates"]
|
|
if u["work_label"] == "Diplomový projekt 2020"
|
|
],
|
|
}
|
|
return {"choices": [{"message": {"content": json.dumps(a)}}]}
|
|
|
|
r = generate_verified_rag(q, p, incomplete)
|
|
assert (
|
|
not r["accepted_for_delivery"] and "missing_methods_evidence" in r["attempts"][-1]["issues"]
|
|
)
|
|
|
|
|
|
def test_missing_methods_are_reported_for_the_right_work_without_stealing_another_works_evidence(
|
|
projects,
|
|
):
|
|
with sqlite3.connect(projects) as c:
|
|
c.execute(
|
|
"UPDATE chunks SET text='Úlohy na ďalšie stretnutie:\n- Použite metódu BERT.' WHERE chunk_index=1"
|
|
)
|
|
q = "Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?"
|
|
r = generate_verified_rag(q, prepare(projects, q), select)
|
|
assert r["accepted_for_delivery"] and r["answer_coverage"]["status"] == "partial"
|
|
assert "SQLite" in r["answer"] and "BERT" not in r["answer"]
|
|
assert (
|
|
"Diplomový projekt 2 2020" in r["answer"] and "skutočne použitých metódach" in r["answer"]
|
|
)
|
|
|
|
|
|
def test_plans_remain_available_as_plans_with_natural_verbs(projects):
|
|
q = "Aké plánované úlohy mala Eva Nová pri oboch diplomových projektoch v roku 2020?"
|
|
r = generate_verified_rag(q, prepare(projects, q), select)
|
|
assert r["accepted_for_delivery"]
|
|
assert (
|
|
"Urobiť návrh metódy PageRank." in r["answer"]
|
|
and "Naštudovať si ako sa používa Promise." in r["answer"]
|
|
)
|
|
assert "urobte" not in r["answer"] and "Naštudujte" not in r["answer"]
|
|
|
|
|
|
def test_unknown_requested_work_is_not_silently_dropped(projects):
|
|
q = 'Porovnaj metódy Evy Novej pri "Diplomový projekt 2020" a "Diplomový projekt 3 2020".'
|
|
p = prepare(projects, q)
|
|
assert p["status"] == "ambiguous" and p["reason"] == "unresolved_work_selection"
|
|
|
|
|
|
def test_short_followup_needs_context_instead_of_global_retrieval(monkeypatch):
|
|
def forbidden(*a, **kw):
|
|
pytest.fail("A bare follow-up must not search or call an LLM")
|
|
|
|
monkeypatch.setattr(answer_delivery, "build_rag_context", forbidden)
|
|
r = answer_delivery.deliver_answer("daj obe", engine="rag", complete=forbidden)
|
|
assert r["status"] == "ambiguous" and r["reason"] == "missing_followup_context"
|
|
|
|
|
|
def test_negative_selector_is_not_a_positive_number_and_literal_title_years_are_not_filters():
|
|
q = 'Aké metódy používal Ján Holp pri "Diplomový projekt 2020" (nie Diplomový projekt 2)?'
|
|
selector = parse_work_selector(q)
|
|
assert (
|
|
project_number(q) is None
|
|
and selector.excluded_numbers == (2,)
|
|
and requested_years(q) == [2020]
|
|
)
|
|
assert selector.matches("Diplomový projekt 2020") and not selector.matches(
|
|
"Diplomový projekt 2 2020"
|
|
)
|
|
assert requested_years('Kto má názov "Projekt 2 o svete 2099"?') == []
|
|
assert not parse_work_selector('Kto má názov "Projekt 2 o svete 2099"?').specific
|
|
assert unsupported_constraint("Aké metódy používal Ján Holp, nie v roku 2020?") == "exclusion"
|
|
|
|
|
|
def test_unimplemented_method_is_not_proof_of_actual_use():
|
|
assert "methods" not in facets_for(
|
|
"Metóda BERT ešte nie je implementovaná.",
|
|
"progress",
|
|
"Aké metódy skutočne použila Eva Nová?",
|
|
)
|
|
|
|
|
|
def test_scalar_question_also_preserves_exact_work_and_exclusion(projects):
|
|
q = 'Aký názov má "Diplomový projekt 2020" Evy Novej (nie Diplomový projekt 2)?'
|
|
p = prepare(projects, q)
|
|
assert (
|
|
len(p["work_scopes"]) == 1 and p["work_scopes"][0]["work_label"] == "Diplomový projekt 2020"
|
|
)
|
|
assert "Python" not in p["context"] and "Promise" not in p["context"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"heading",
|
|
["Úlohy na ďalšie stretnutie:", "Do ďalšieho stretnutia:", "Do ďalšiehio stretnutia:"],
|
|
)
|
|
def test_source_task_headings_cannot_inherit_a_completed_state(heading):
|
|
sections = content_sections(
|
|
"Stav:\n- Implementovaný parser.\n" + heading + "\n- urobte návrh metódy PageRank."
|
|
)
|
|
assert [s["kind"] for s in sections] == ["progress", "task"]
|
|
assert sections[-1]["phase"] == "planned"
|