476 lines
17 KiB
Python
476 lines
17 KiB
Python
"""Real multi-person query regressions, including deliberately wrong evidence."""
|
|
|
|
import copy
|
|
import json
|
|
import sqlite3
|
|
|
|
import pytest
|
|
|
|
from tools.answers.context_policy import answer_context_issues, text_contains_value
|
|
from tools.answers.graph_context import graph_context
|
|
from tools.answers.presentation import content_lines, goal_text
|
|
from tools.answers.rag_evidence import (
|
|
content_sections,
|
|
expand_comparison_sources,
|
|
rag_evidence_context,
|
|
requested_types,
|
|
)
|
|
from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues
|
|
from tools.answers.render_answer import render_graph_answer
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
|
from tools.rag.llm_context import compact_rag_payload
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
COMPARE = (
|
|
"Porovnaj zameranie bakalárskej práce Ondreja Megelu a diplomovej práce Jána Holpa. "
|
|
"Rozlišuj zadanie od dosiahnutých výsledkov a uveď zdroje."
|
|
)
|
|
SHARED = "V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt?"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"canonical,wording",
|
|
[
|
|
("Ondrej Megela", "Ondreja Megelu"),
|
|
("Martin Šarišský", "Martina Šarišského"),
|
|
("Ján Malinovský", "Jána Malinovského"),
|
|
("Eva Nová", "Evy Novej"),
|
|
],
|
|
)
|
|
def test_complete_case_endings(canonical, wording):
|
|
assert text_contains_value(wording, canonical, person=True)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"canonical,other",
|
|
[
|
|
("Martin Šarišský", "Martin Jančura"),
|
|
("Ján Holp", "Ján Holpák"),
|
|
("Ondrej Megela", "Ondrej Megelovec"),
|
|
("Martin NaN1", "Martin NaN2"),
|
|
],
|
|
)
|
|
def test_no_short_prefix_matching(canonical, other):
|
|
assert not text_contains_value(other, canonical, person=True)
|
|
|
|
|
|
def test_each_person_gets_the_requested_work_type():
|
|
assert requested_types(COMPARE, ["Ján Holp", "Ondrej Megela"]) == {
|
|
"Ján Holp": "diploma_thesis",
|
|
"Ondrej Megela": "bachelor_thesis",
|
|
}
|
|
|
|
|
|
def test_goal_conversion_is_not_all_or_nothing():
|
|
text = "\n".join(
|
|
content_lines(
|
|
[
|
|
"- Zistite, čo je to RAG.",
|
|
"- Naučte sa čo je to GraphRAG.",
|
|
"- Zostavte malý znalostný graf.",
|
|
"- Využite graf pri vyhľadávaní.",
|
|
"- Použite databázu Neo4j.",
|
|
"Práca neimplementuje vlastný model.",
|
|
],
|
|
"Ján Malinovský",
|
|
"diploma_project",
|
|
"Aké sú ciele diplomového projektu Jána Malinovského?",
|
|
)
|
|
)
|
|
assert "Diplomový projekt Jána Malinovského" in text
|
|
assert "Zistiť, čo je to RAG." in text and "Použiť databázu Neo4j." in text
|
|
assert "Ďalšie podklady:" not in text and "Práca neimplementuje vlastný model." in text
|
|
assert not any(word in text for word in ["Zistite", "Naučte", "Zostavte", "Využite", "Použite"])
|
|
|
|
|
|
def test_prefixed_goal_keeps_other_converted_items():
|
|
text = "\n".join(
|
|
content_lines(
|
|
[
|
|
"1. Vypracujte prehľad systémov generovania prirodzeného jazyka.",
|
|
"2. Na základe tohto prehľadu navrhnite a implementujte agentový systém.",
|
|
],
|
|
"Martin Šarišský",
|
|
"diploma_thesis",
|
|
"Čomu sa venuje diplomová práca Martina Šarišského?",
|
|
)
|
|
)
|
|
assert "Diplomová práca Martina Šarišského" in text
|
|
assert "Vypracovanie prehľadu" in text and "navrhnúť a implementovať" in text
|
|
assert "Vypracujte" not in text and "navrhnite" not in text
|
|
|
|
|
|
def test_coordinated_commands_and_objects_keep_grammatical_infinitives():
|
|
text = "\n".join(
|
|
content_lines(
|
|
[
|
|
"Vypracujte prehľad aplikácií BERT a spôsoby ich vyhodnotenia.",
|
|
"Vypracujte prehľad metód a implementujte systém.",
|
|
],
|
|
"Ondrej Megela",
|
|
"bachelor_thesis",
|
|
)
|
|
)
|
|
assert "Vypracovať prehľad aplikácií BERT a spôsoby ich vyhodnotenia." in text
|
|
assert "Vypracovať prehľad metód a implementovať systém." in text
|
|
|
|
|
|
def test_goal_headings_are_not_extra_evidence_and_sentences_keep_capitals():
|
|
text = "\n".join(
|
|
content_lines(
|
|
["Ciele na semester.", "Zlepšite proces. Zlepšite extrakciu."],
|
|
"Ján Malinovský",
|
|
"diploma_project",
|
|
)
|
|
)
|
|
assert "Ďalšie podklady" not in text and "Zlepšiť proces. Zlepšiť extrakciu." in text
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"literal,expected",
|
|
[
|
|
(
|
|
"Použite https://example.test/pouzite/vypracujte.",
|
|
"Použiť https://example.test/pouzite/vypracujte.",
|
|
),
|
|
(
|
|
"Implementujte parser pre `navrhnite a implementujte`.",
|
|
"Implementovať parser pre `navrhnite a implementujte`.",
|
|
),
|
|
],
|
|
)
|
|
def test_grammar_does_not_change_urls_or_code_literals(literal, expected):
|
|
assert goal_text(literal) == (expected, True)
|
|
|
|
|
|
@pytest.fixture
|
|
def comparison_payload(tmp_path):
|
|
sources = []
|
|
rows = []
|
|
for i, (person, slug, kind, year, goal, progress) in enumerate(
|
|
[
|
|
(
|
|
"Ján Holp",
|
|
"jan_holp",
|
|
"Diplomová práca",
|
|
2021,
|
|
"Implementujte systém vyhľadávania.",
|
|
"Implementovaný PageRank a indexovanie webových stránok.",
|
|
),
|
|
(
|
|
"Ondrej Megela",
|
|
"ondrej_megela",
|
|
"Bakalárska práca",
|
|
2020,
|
|
"Natrénujte jazykový model metódou BERT.",
|
|
"Natrénovaný model wiki103 a vykonané vyhodnotenie.",
|
|
),
|
|
],
|
|
1,
|
|
):
|
|
path = f"pages/students/2018/{slug}/README.md"
|
|
sources.append(
|
|
{
|
|
"source_id": f"S{i}",
|
|
"title": person,
|
|
"document_path": path,
|
|
"source_url": f"https://example.test/{slug}",
|
|
"text": f"Sekcia: {kind} {year}\nNázov: Ukážkový názov práce.",
|
|
}
|
|
)
|
|
rows.append(
|
|
(
|
|
path,
|
|
json.dumps([[person, f"{kind} {year}"]]),
|
|
f"Zadanie:\n- {goal}\nStav:\n- {progress}",
|
|
1,
|
|
0,
|
|
)
|
|
)
|
|
rows.append(
|
|
(
|
|
path,
|
|
json.dumps([[person, "Diplomový projekt 2022"]]),
|
|
"Stav:\nCudzí projekt využíva Cassandra.",
|
|
1,
|
|
1,
|
|
)
|
|
)
|
|
db = tmp_path / "index.sqlite"
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE chunks(document_path,heading_paths_json,text,published,chunk_index)"
|
|
)
|
|
conn.executemany("INSERT INTO chunks VALUES(?,?,?,?,?)", rows)
|
|
raw = {
|
|
"status": "ok",
|
|
"reason": None,
|
|
"sources": sources,
|
|
"context": build_context_text(sources),
|
|
"complete": False,
|
|
"truncated": True,
|
|
"truncation_reasons": ["result_limit"],
|
|
"index_version": {"version": "fixture"},
|
|
}
|
|
return db, raw
|
|
|
|
|
|
def selected_comparison(context):
|
|
quotes = (
|
|
[{"source_id": c["source_id"], "quote": c["text"]} for c in context["content_candidates"]]
|
|
if "content_candidates" in context
|
|
else [
|
|
{"source_id": p["source_id"], "quote": s["text"]}
|
|
for p in context["content_passages"]
|
|
for s in content_sections(p["text"])
|
|
if s["kind"] in {"assignment", "progress"}
|
|
]
|
|
)
|
|
return {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [],
|
|
"evidence_quotes": quotes,
|
|
"source_urls": [s["source_url"] for s in context["sources"]],
|
|
}
|
|
|
|
|
|
def test_expansion_preserves_binding_versions_and_budget(comparison_payload):
|
|
db, raw = comparison_payload
|
|
before = copy.deepcopy(raw)
|
|
payload = compact_rag_payload(expand_comparison_sources(db, COMPARE, raw))
|
|
c = rag_evidence_context(COMPARE, payload)
|
|
assert c["requirements"]["work_types"] == {
|
|
"Ján Holp": "diploma_thesis",
|
|
"Ondrej Megela": "bachelor_thesis",
|
|
}
|
|
assert c["requirements"]["content_roles"] == {
|
|
p: ["assignment", "progress"] for p in c["requirements"]["people"]
|
|
}
|
|
assert "Cassandra" not in payload["context"]
|
|
assert payload["index_version"] == raw["index_version"] and payload["complete"] is False
|
|
assert payload["context_budget"]["used_tokens"] <= payload["context_budget"]["max_tokens"]
|
|
assert raw == before
|
|
|
|
|
|
def test_hidden_chunks_are_not_added_to_comparison_evidence(comparison_payload):
|
|
db, raw = comparison_payload
|
|
path = raw["sources"][0]["document_path"]
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"INSERT INTO chunks VALUES(?,?,?,?,?)",
|
|
(
|
|
path,
|
|
json.dumps([["Ján Holp", "Diplomová práca 2021"]]),
|
|
"Stav:\nNezverejnený tajný výsledok.",
|
|
0,
|
|
2,
|
|
),
|
|
)
|
|
expanded = expand_comparison_sources(db, COMPARE, raw)
|
|
assert "tajný výsledok" not in expanded["context"]
|
|
|
|
|
|
def test_relevant_child_document_is_preserved(comparison_payload):
|
|
db, raw = comparison_payload
|
|
child = {
|
|
**raw["sources"][0],
|
|
"source_id": "S3",
|
|
"title": "Meranie projektu",
|
|
"document_path": raw["sources"][0]["document_path"].replace(
|
|
"README.md", "meranie/README.md"
|
|
),
|
|
"text": "Sekcia: Diplomová práca 2021\nStav:\nVyhodnotená presnosť vyhľadávania.",
|
|
}
|
|
raw["sources"].append(child)
|
|
raw["context"] = build_context_text(raw["sources"])
|
|
expanded = expand_comparison_sources(db, COMPARE, raw)
|
|
assert "Vyhodnotená presnosť vyhľadávania." in expanded["context"]
|
|
|
|
|
|
def test_comparison_separates_goals_and_results(comparison_payload):
|
|
db, raw = comparison_payload
|
|
payload = compact_rag_payload(expand_comparison_sources(db, COMPARE, raw))
|
|
c = rag_evidence_context(COMPARE, payload)
|
|
selected = selected_comparison(c)
|
|
assert rag_answer_issues(selected, c) == []
|
|
result = generate_verified_rag(
|
|
COMPARE, payload, lambda _: {"choices": [{"message": {"content": json.dumps(selected)}}]}
|
|
)
|
|
assert result["accepted_for_delivery"]
|
|
text = result["answer"]
|
|
assert text.startswith("| Oblasť |")
|
|
assert text.count("| Zadanie a ciele |") == text.count("| Zaznamenané výsledky |") == 1
|
|
assert (
|
|
"Diplomová práca Jána Holpa (2021)" in text
|
|
and "Bakalárska práca Ondreja Megelu (2020)" in text
|
|
)
|
|
assert "Natrénovať jazykový model metódou BERT." in text
|
|
assert "Natrénovaný model wiki103" in text and "Implementovaný PageRank" in text
|
|
assert "Cassandra" not in text and len(result["source_urls"]) == 2
|
|
|
|
|
|
def test_goals_cannot_replace_available_results(comparison_payload):
|
|
db, raw = comparison_payload
|
|
c = rag_evidence_context(COMPARE, expand_comparison_sources(db, COMPARE, raw))
|
|
a = selected_comparison(c)
|
|
a["evidence_quotes"] = [
|
|
q
|
|
for q in a["evidence_quotes"]
|
|
if not any(word in q["quote"] for word in ["Implementovaný", "Natrénovaný"])
|
|
]
|
|
assert "missing_progress_evidence" in rag_answer_issues(a, c)
|
|
|
|
|
|
def test_missing_results_allow_an_explicit_partial_comparison(comparison_payload):
|
|
db, raw = comparison_payload
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"UPDATE chunks SET text='Zadanie:\n- Implementujte systém vyhľadávania.' WHERE document_path LIKE '%jan_holp%' AND chunk_index=0"
|
|
)
|
|
payload = expand_comparison_sources(db, COMPARE, raw)
|
|
c = rag_evidence_context(COMPARE, payload)
|
|
selected = selected_comparison(c)
|
|
result = generate_verified_rag(
|
|
COMPARE, payload, lambda _: {"choices": [{"message": {"content": json.dumps(selected)}}]}
|
|
)
|
|
assert result["accepted_for_delivery"]
|
|
assert "| Zaznamenané výsledky | V podkladoch chýbajú údaje. |" in result["answer"]
|
|
assert "Implementácia systému vyhľadávania" in result["answer"]
|
|
|
|
|
|
def test_future_tasks_are_not_presented_as_achieved_results(comparison_payload):
|
|
db, raw = comparison_payload
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"UPDATE chunks SET text=text || '\nÚlohy:\n- Pripravte experiment s PageRank.' WHERE document_path LIKE '%jan_holp%' AND chunk_index=0"
|
|
)
|
|
payload = expand_comparison_sources(db, COMPARE, raw)
|
|
c = rag_evidence_context(COMPARE, payload)
|
|
selected = selected_comparison(c)
|
|
p = next(p for p in c["content_passages"] if p["person"] == "Ján Holp")
|
|
selected["evidence_quotes"].append(
|
|
{"source_id": p["source_id"], "quote": "- Pripravte experiment s PageRank."}
|
|
)
|
|
result = generate_verified_rag(
|
|
COMPARE, payload, lambda _: {"choices": [{"message": {"content": json.dumps(selected)}}]}
|
|
)
|
|
# The model may not quote a future task omitted from its actual candidates.
|
|
assert not result["accepted_for_delivery"]
|
|
assert "quote_outside_delivered_candidates" in result["attempts"][-1]["issues"]
|
|
|
|
|
|
@pytest.fixture
|
|
def shared_session():
|
|
catalog = []
|
|
theses = []
|
|
works = []
|
|
for pid, person, slug in [("p1", "Ján Holp", "jan"), ("p2", "Ondrej Megela", "ondrej")]:
|
|
path = f"pages/students/2018/{slug}/README.md"
|
|
url = f"https://example.test/{slug}"
|
|
catalog.append(
|
|
{"element_id": pid, "kind": "Person", "name": person, "source_document": path}
|
|
)
|
|
base = {
|
|
"person_element_id": pid,
|
|
"person": person,
|
|
"document_path": path,
|
|
"source_url": url,
|
|
}
|
|
theses.append(
|
|
{
|
|
**base,
|
|
"thesis_element_id": pid + "t",
|
|
"thesis_id": pid + "t",
|
|
"title": "Ukážkový názov",
|
|
"year": 2019 if pid == "p1" else 2020,
|
|
"heading": "Bakalárska práca",
|
|
"thesis_type": "bachelor_thesis",
|
|
}
|
|
)
|
|
works.append(
|
|
{
|
|
**base,
|
|
"work_element_id": pid + "w",
|
|
"work_id": pid + "w",
|
|
"year": 2020,
|
|
"heading": "Diplomový projekt",
|
|
"work_type": "diploma_project",
|
|
}
|
|
)
|
|
return FakeSession(
|
|
{gs.CATALOG_QUERY: catalog, gs.THESIS_FACTS_QUERY: theses, gs.WORK_FACTS_QUERY: works}
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("word", ["obaja", "obidvaja"])
|
|
def test_both_means_shared_years(word):
|
|
c = gs.extract_query_constraints(SHARED.replace("obaja", word))
|
|
assert c["mentions_shared"] and c["mentions_academic_year"]
|
|
|
|
|
|
def test_shared_year_has_only_intersection_and_bound_work_details(shared_session):
|
|
result = gs.search_graph_with_session(shared_session, SHARED)
|
|
assert result["academic_year_comparison"]["shared_years"] == [2020]
|
|
c = graph_context(result)
|
|
records = c["required_academic_records"]
|
|
assert len(records) == 3 and all(r["year"] == 2020 for r in records)
|
|
a = {"status": "ok", "claims": records, "source_urls": [s["source_url"] for s in c["sources"]]}
|
|
rendered = render_graph_answer(a, c)
|
|
assert answer_context_issues(rendered, c) == []
|
|
assert "Spoločný evidovaný rok je **2020**." in rendered["answer"]
|
|
assert "2019" not in rendered["answer"] and "Ukážkový názov" in rendered["answer"]
|
|
|
|
|
|
def test_shared_year_with_per_person_presentation_instructions(shared_session):
|
|
question = (
|
|
"V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt? "
|
|
"Najprv uveď spoločné roky, potom pri každej osobe príslušné práce s typom a názvom, "
|
|
"ak je uvedený. Pridaj zdroje."
|
|
)
|
|
result = gs.search_graph_with_session(shared_session, question)
|
|
assert result["status"] == "ok"
|
|
assert result["academic_year_comparison"]["shared_years"] == [2020]
|
|
context = graph_context(result)
|
|
records = context["required_academic_records"]
|
|
assert len(records) == 3 and {r["person"] for r in records} == {"Ján Holp", "Ondrej Megela"}
|
|
answer = {
|
|
"status": "ok",
|
|
"claims": records,
|
|
"source_urls": [s["source_url"] for s in context["sources"]],
|
|
}
|
|
rendered = render_graph_answer(answer, context)
|
|
assert answer_context_issues(rendered, context) == []
|
|
assert "Spoločný evidovaný rok je **2020**." in rendered["answer"]
|
|
assert "2019" not in rendered["answer"]
|
|
|
|
|
|
def test_no_common_year_is_not_a_union(shared_session):
|
|
for row in shared_session.responses[gs.WORK_FACTS_QUERY]:
|
|
if row["person_element_id"] == "p1":
|
|
row["year"] = 2021
|
|
result = gs.search_graph_with_session(shared_session, SHARED)
|
|
assert (
|
|
result["status"] == "no_answer" and result["academic_year_comparison"]["shared_years"] == []
|
|
)
|
|
c = graph_context(result)
|
|
assert (
|
|
"spoločný rok nenašiel"
|
|
in render_graph_answer({"status": "no_answer", "claims": [], "source_urls": []}, c)[
|
|
"answer"
|
|
]
|
|
)
|
|
|
|
|
|
def test_partial_records_do_not_claim_an_unwitnessed_common_year(shared_session):
|
|
result = gs.search_graph_with_session(shared_session, SHARED, limit=1)
|
|
c = graph_context(result)
|
|
a = {
|
|
"status": "ok",
|
|
"claims": c["required_academic_records"],
|
|
"source_urls": [s["source_url"] for s in c["sources"]],
|
|
}
|
|
text = render_graph_answer(a, c)["answer"]
|
|
assert "Výpis je neúplný." in text and "Spoločný evidovaný rok je" not in text
|