341 lines
14 KiB
Python
341 lines
14 KiB
Python
"""Bound excerpts to works, preserve their phase and share comparison space."""
|
|
|
|
import copy
|
|
import json
|
|
import sqlite3
|
|
import pytest
|
|
|
|
from test_quality_block import corpus, prepare, complete
|
|
from tools.answers.content_selection import administrative_only, evidence_units
|
|
from tools.answers.rag_evidence import rag_evidence_context, content_sections
|
|
from tools.answers.rag_preparation import prepare_named_content
|
|
from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues
|
|
from tools.answers.verified_rag import overview_excerpts
|
|
from tools.rag.llm_context import compact_rag_payload
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
|
|
def update_eva(corpus, text, headings=None):
|
|
db, raw = corpus
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"UPDATE chunks SET text=?,heading_paths_json=? WHERE document_path=?",
|
|
(
|
|
text,
|
|
json.dumps(headings or [["Eva Nová", "Diplomová práca 2024"]]),
|
|
raw["sources"][0]["document_path"],
|
|
),
|
|
)
|
|
|
|
|
|
def selection(context):
|
|
return {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [],
|
|
"evidence_quotes": [
|
|
{key: c[key] for key in ["source_id", "passage_id"] if c.get(key)}
|
|
| {"quote": c["text"]}
|
|
for c in context.get("content_candidates", [])
|
|
],
|
|
"source_urls": [s["source_url"] for s in context["sources"]],
|
|
}
|
|
|
|
|
|
def test_assignment_plan_progress_and_recorded_result_are_separate(corpus):
|
|
update_eva(
|
|
corpus,
|
|
"Zadanie:\n- Implementujte vyhľadávač.\nPriebežný stav:\n"
|
|
"- Rozpracovaná integrácia vyhľadávania.\n- Bude implementovaný klasifikátor.\n"
|
|
"Výsledky:\n- Natrénovaný model BERT.\nPlánované úlohy:\n- Pripravte experiment s presnosťou.",
|
|
)
|
|
q = "Zhrň zadanie, plánované úlohy, priebežný stav a výsledky diplomovej práce Evy Novej."
|
|
payload = prepare(corpus, q)
|
|
context = rag_evidence_context(q, payload)
|
|
units = evidence_units(context)
|
|
assert {u["phase"] for u in units} >= {"assignment", "planned", "progress", "result"}
|
|
future = next(u for u in units if "Bude implementovaný" in u["text"])
|
|
assert future["phase"] == "planned" and "results" not in future["facets"]
|
|
result = generate_verified_rag(q, payload, complete(selection(context)))
|
|
assert result["accepted_for_delivery"]
|
|
assert all(
|
|
label in result["answer"]
|
|
for label in [
|
|
"Zadanie a ciele",
|
|
"Plánované úlohy",
|
|
"Priebežný stav",
|
|
"Zaznamenané výsledky",
|
|
]
|
|
)
|
|
assert "Bude implementovaný" not in result["answer"].split("**Zaznamenané výsledky:**")[1]
|
|
|
|
|
|
@pytest.mark.parametrize("heading", ["Zadanie", "Zadanie diplomovej práce"])
|
|
def test_phase_from_chunk_heading_ancestry_is_not_lost(corpus, heading):
|
|
update_eva(
|
|
corpus, "- Pripravte vyhodnotenie modelu.", [["Eva Nová", "Diplomová práca 2024", heading]]
|
|
)
|
|
q = "Aké je zadanie a plánované úlohy diplomovej práce Evy Novej?"
|
|
c = rag_evidence_context(q, prepare(corpus, q))
|
|
assert any(u["kind"] == "assignment" and u["phase"] == "assignment" for u in evidence_units(c))
|
|
assert not any(u["kind"] == "task" for u in evidence_units(c))
|
|
assert all(
|
|
u["year"] == 2024 and u["work_label"] == "Diplomová práca 2024" for u in evidence_units(c)
|
|
)
|
|
|
|
|
|
def test_negated_completion_is_progress_not_a_completed_result(corpus):
|
|
update_eva(corpus, "Stav:\n- Parser zatiaľ nie je implementovaný.")
|
|
q = "Aký je priebežný stav diplomovej práce Evy Novej?"
|
|
c = rag_evidence_context(q, prepare(corpus, q))
|
|
row = next(u for u in evidence_units(c) if "Parser" in u["text"])
|
|
assert row["phase"] == "progress" and "results" not in row["facets"]
|
|
q = "Aké výsledky dosiahla Eva Nová pri diplomovej práci?"
|
|
r = generate_verified_rag(
|
|
q,
|
|
prepare(corpus, q),
|
|
lambda _: (_ for _ in ()).throw(AssertionError("No results model call")),
|
|
)
|
|
assert r["status"] == "no_answer" and r["accepted_for_delivery"]
|
|
|
|
|
|
def test_administration_alone_is_not_a_technical_result(corpus):
|
|
update_eva(corpus, "Stav:\n- Konzultácia štruktúry práce.\n- Dohodnutý termín odovzdania.")
|
|
q = "Aké výsledky dosiahla Eva Nová pri diplomovej práci?"
|
|
c = rag_evidence_context(q, prepare(corpus, q))
|
|
assert not c["content_candidates"]
|
|
assert administrative_only("Konzultácia štruktúry práce.")
|
|
assert not administrative_only("Vyriešený problém s GPU po konzultácii.")
|
|
assert not administrative_only("Odovzdaný zdrojový kód indexovania do SQLite.")
|
|
|
|
|
|
def test_recorded_working_system_is_a_result_not_just_administration(corpus):
|
|
update_eva(corpus, "Stav:\n- Funguje RAG systém, zostáva overiť presnosť.")
|
|
q = "Aké výsledky má diplomová práca Evy Novej?"
|
|
c = rag_evidence_context(q, prepare(corpus, q))
|
|
assert any(u["phase"] == "result" and "zostáva" in u["text"] for u in c["content_candidates"])
|
|
|
|
|
|
def test_completed_experiments_are_not_confused_with_planned_tasks(corpus):
|
|
update_eva(
|
|
corpus,
|
|
"Výsledky:\n- Vyhodnotená presnosť modelu v experimente.\n"
|
|
"Úlohy:\n- Pripravte experiment s novým tokenizérom.",
|
|
)
|
|
q = "Aké vykonané experimenty má Eva Nová a aké plánované úlohy jej zostali?"
|
|
c = rag_evidence_context(q, prepare(corpus, q))
|
|
planned = next(u for u in evidence_units(c) if "Pripravte" in u["text"])
|
|
assert "plans" in planned["facets"] and "experiments" not in planned["facets"]
|
|
|
|
|
|
def test_a_planned_method_does_not_prove_an_actually_used_method(corpus):
|
|
update_eva(corpus, "Úlohy:\n- Použiť metódu BERT.")
|
|
q = "Aké použité metódy má diplomová práca Evy Novej?"
|
|
c = rag_evidence_context(q, prepare(corpus, q))
|
|
assert not c["content_candidates"]
|
|
|
|
|
|
def test_explicit_negative_finding_remains_a_result(corpus):
|
|
update_eva(corpus, "Výsledky:\n- Model nebol úspešný pri teste presnosti.")
|
|
q = "Aké výsledky má diplomová práca Evy Novej?"
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
assert c["content_candidates"][0]["phase"] == "result"
|
|
r = generate_verified_rag(q, p, complete(selection(c)))
|
|
assert r["accepted_for_delivery"] and "nebol úspešný" in r["answer"]
|
|
|
|
|
|
def test_identical_text_in_two_work_years_needs_passage_identity(corpus):
|
|
_, raw = corpus
|
|
source = raw["sources"][0]
|
|
source["text"] = "Sekcia: Diplomová práca 2024\nStav:\nImplementovaný parser.\n\n"
|
|
source["text"] += "Sekcia: Diplomová práca 2025\nStav:\nImplementovaný parser."
|
|
raw["sources"] = [source]
|
|
raw["context"] = build_context_text(raw["sources"])
|
|
q = "Aké výsledky má Eva Nová pri diplomovej práci?"
|
|
c = rag_evidence_context(q, raw)
|
|
a = {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [],
|
|
"source_urls": [source["source_url"]],
|
|
"evidence_quotes": [{"source_id": "S1", "quote": "Implementovaný parser."}],
|
|
}
|
|
assert "ambiguous_evidence_scope" in rag_answer_issues(a, c)
|
|
p = c["content_passages"][0]
|
|
a["evidence_quotes"][0]["passage_id"] = p["passage_id"]
|
|
assert "ambiguous_evidence_scope" not in rag_answer_issues(a, c)
|
|
assert len({p["passage_id"] for p in c["content_passages"]}) == 2
|
|
|
|
|
|
def test_identical_text_cannot_reuse_another_persons_passage_id(corpus):
|
|
for person_name in ["eva_nova", "boris_novy"]:
|
|
db, _ = corpus
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"UPDATE chunks SET text=? WHERE document_path LIKE ?",
|
|
("Stav:\nImplementovaný parser.", f"%{person_name}%"),
|
|
)
|
|
q = "Porovnaj výsledky Evy Novej a Borisa Nového."
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
a = selection(c)
|
|
assert not rag_answer_issues(a, c)
|
|
wrong = copy.deepcopy(a)
|
|
wrong["evidence_quotes"][0]["passage_id"] = wrong["evidence_quotes"][1]["passage_id"]
|
|
assert "unsupported_evidence_passage" in rag_answer_issues(wrong, c)
|
|
|
|
|
|
def test_overlapping_excerpt_preserves_qualification_without_repetition(corpus):
|
|
update_eva(
|
|
corpus,
|
|
"Stav:\n- Natrénovaný model BERT.\n"
|
|
"- Natrénovaný model BERT. Výsledok platí iba pre malý korpus.",
|
|
)
|
|
q = "Aké výsledky dosiahla Eva Nová?"
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
r = generate_verified_rag(q, p, complete(selection(c)))
|
|
assert r["accepted_for_delivery"]
|
|
assert r["answer"].count("Natrénovaný model BERT") == 1 and "iba pre malý korpus" in r["answer"]
|
|
|
|
|
|
def add_same_year_projects(corpus):
|
|
db, raw = corpus
|
|
with sqlite3.connect(db) as conn:
|
|
for i, method in [(1, "BERT"), (2, "Cassandra")]:
|
|
conn.execute(
|
|
"INSERT INTO chunks VALUES(?,?,?,?,?)",
|
|
(
|
|
raw["sources"][0]["document_path"],
|
|
f"Stav:\nImplementovaná metóda {method}.",
|
|
json.dumps([["Eva Nová", f"Diplomový projekt {i} 2022"]]),
|
|
1,
|
|
i,
|
|
),
|
|
)
|
|
|
|
|
|
def test_same_year_project_records_are_not_merged(corpus):
|
|
add_same_year_projects(corpus)
|
|
p = prepare(corpus, "Aké metódy používala Eva Nová pri diplomovom projekte v roku 2022?")
|
|
assert (
|
|
p["status"] == "ambiguous"
|
|
and "projekt 1" in p["clarification"]
|
|
and "projekt 2" in p["clarification"]
|
|
)
|
|
|
|
|
|
def test_explicit_project_number_binds_the_excerpt_to_that_project(corpus):
|
|
add_same_year_projects(corpus)
|
|
q = "Aké metódy používala Eva Nová pri diplomovom projekte 2 v roku 2022?"
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
assert p["status"] == "ok" and "Cassandra" in p["context"] and "BERT" not in p["context"]
|
|
assert all("projekt 2" in u["work_label"] for u in evidence_units(c))
|
|
a = selection(c)
|
|
result = generate_verified_rag(q, p, complete(a))
|
|
assert result["accepted_for_delivery"] and "projekt 2" in result["answer"]
|
|
|
|
|
|
def test_large_first_profile_cannot_starve_the_second_person(corpus):
|
|
db, raw = corpus
|
|
q = "Porovnaj výsledky Evy Novej a Borisa Nového."
|
|
p = prepare_named_content(db, q, raw)
|
|
p["sources"][0]["text"] += "\n" + ("Dlhé administratívne poznámky. " * 3000)
|
|
compact = compact_rag_payload(p, max_tokens=2200)
|
|
assert compact["source_count"] == 2 and compact["context_budget"]["used_tokens"] <= 2200
|
|
assert compact["truncated"] and "llm_context_budget" in compact["truncation_reasons"]
|
|
c = rag_evidence_context(q, compact)
|
|
assert {row["person"] for row in c["content_candidates"]} == {"Eva Nová", "Boris Nový"}
|
|
assert all(row["passage_id"] and row["work_label"] for row in c["content_candidates"])
|
|
|
|
|
|
def test_clipping_cannot_drop_a_wrapped_qualification_from_an_excerpt(corpus):
|
|
db, raw = corpus
|
|
q = "Porovnaj výsledky Evy Novej a Borisa Nového."
|
|
p = prepare_named_content(db, q, raw)
|
|
p["sources"][0]["text"] = "Sekcia: Diplomová práca 2024\nStav:\n- Natrénovaný model BERT.\n "
|
|
p["sources"][0]["text"] += "Výsledok platí iba pre malý korpus. " * 2000
|
|
compact = compact_rag_payload(p, max_tokens=1700)
|
|
assert compact["truncated"] and "BERT" not in compact["context"]
|
|
c = rag_evidence_context(q, compact)
|
|
assert any(row["person"] == "Boris Nový" for row in c["content_candidates"])
|
|
assert not any(row["person"] == "Eva Nová" for row in c["content_candidates"])
|
|
|
|
|
|
def test_single_profile_clipping_preserves_the_same_qualification_rule(corpus):
|
|
_, raw = corpus
|
|
raw["sources"] = raw["sources"][:1]
|
|
raw["sources"][0]["text"] = "Sekcia: Diplomová práca 2024\nStav:\n- Natrénovaný model BERT.\n "
|
|
raw["sources"][0]["text"] += "Výsledok platí iba pre malý testovací korpus. " * 2000
|
|
compact = compact_rag_payload(raw, max_tokens=1700)
|
|
assert compact["truncated"] and "BERT" not in compact["context"]
|
|
|
|
|
|
@pytest.mark.parametrize("with_title", [False, True])
|
|
def test_relevant_results_are_prioritized_before_context_is_clipped(corpus, with_title):
|
|
update_eva(
|
|
corpus,
|
|
("Názov: Skutočný názov práce.\n" if with_title else "")
|
|
+ "Zadanie:\n- Implementujte parser.",
|
|
)
|
|
db, raw = corpus
|
|
path = raw["sources"][0]["document_path"]
|
|
with sqlite3.connect(db) as conn:
|
|
for i in range(1, 12):
|
|
conn.execute(
|
|
"INSERT INTO chunks VALUES(?,?,?,?,?)",
|
|
(
|
|
path,
|
|
"Stav:\n- Konzultácia štruktúry práce. " + ("Administratívne poznámky. " * 100),
|
|
json.dumps([["Eva Nová", "Diplomová práca 2024"]]),
|
|
1,
|
|
i,
|
|
),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO chunks VALUES(?,?,?,?,?)",
|
|
(
|
|
path,
|
|
"Výsledky:\n- Vyhodnotená presnosť parsera.",
|
|
json.dumps([["Eva Nová", "Diplomová práca 2024"]]),
|
|
1,
|
|
12,
|
|
),
|
|
)
|
|
q = "Porovnaj " + ("názvy a " if with_title else "") + "výsledky Evy Novej a Borisa Nového."
|
|
compact = compact_rag_payload(prepare_named_content(db, q, raw), max_tokens=1700)
|
|
assert compact["truncated"] and "Vyhodnotená presnosť parsera." in compact["context"]
|
|
c = rag_evidence_context(q, compact)
|
|
assert {u["person"] for u in c["content_candidates"]} == {"Eva Nová", "Boris Nový"}
|
|
if with_title:
|
|
assert (
|
|
"Skutočný názov práce" in compact["context"]
|
|
and "title" in c["requirements"]["available_fields"]["Eva Nová"]
|
|
)
|
|
|
|
|
|
def test_overview_cannot_cut_off_a_qualification_for_brevity():
|
|
text = (
|
|
"Systém vyhľadáva dokumenty. "
|
|
+ ("Opis implementácie a dát. " * 30)
|
|
+ "Platí iba pre malý testovací korpus."
|
|
)
|
|
quote = {"source_id": "S1", "quote": text}
|
|
c = {"content_passages": [{"source_id": "S1", "person": "Eva Nová", "text": text}], "facts": []}
|
|
assert overview_excerpts({"evidence_quotes": [quote]}, c) == [text]
|
|
|
|
|
|
def test_markdown_role_headings_preserve_explicit_source_phase():
|
|
sections = content_sections(
|
|
"## Zadanie\nImplementujte systém.\n## Plánované úlohy\nPripravte dáta.\n"
|
|
"## Priebežný stav\nRozpracovaná integrácia.\n## Výsledky\nVyhodnotená presnosť."
|
|
)
|
|
assert [(s["kind"], s["phase"]) for s in sections] == [
|
|
("assignment", "assignment"),
|
|
("task", "planned"),
|
|
("progress", "progress"),
|
|
("progress", "result"),
|
|
]
|