331 lines
12 KiB
Python
331 lines
12 KiB
Python
"""Regression for work listings losing titles/types and merging same-year work."""
|
|
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tools.answers.answer_format import parse_answer
|
|
from tools.answers.checked_generation import generate_checked_answer
|
|
from tools.answers.context_policy import answer_context_issues
|
|
from tools.answers.graph_context import graph_context
|
|
from tools.answers.render_answer import render_graph_answer
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession, sample_catalog
|
|
|
|
QUESTION = "Aké práce má Ján Holp a v ktorých rokoch?"
|
|
URL = "https://example.test/jan"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question",
|
|
[
|
|
QUESTION,
|
|
"Aké práce má Ján Holp?",
|
|
"Ktoré práce má Ján Holp?",
|
|
"Vypíš práce Jána Holpa.",
|
|
"Uveď všetky práce Jána Holpa.",
|
|
"Zoznam prác Jána Holpa",
|
|
"Prehľad projektov Jána Holpa",
|
|
"Aký diplomový projekt mal Ján Holp v roku 2020?",
|
|
"Aké má Ján Holp práce a v ktorých rokoch?",
|
|
],
|
|
)
|
|
def test_work_listing_intent_without_explicit_title_word(question):
|
|
assert gs.extract_query_constraints(question)["mentions_work_listing"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question",
|
|
[
|
|
"V ktorých rokoch má Ján Holp práce?",
|
|
"Aký je rok práce Jána Holpa?",
|
|
"Aký je názov diplomovej práce Jána Holpa?",
|
|
"Koľko prác má Ján Holp?",
|
|
"Ktoré osoby majú práce z roku 2020?",
|
|
"Aké kľúčové slová má Ján Holp?",
|
|
],
|
|
)
|
|
def test_other_projections_are_not_changed_into_work_listings(question):
|
|
assert not gs.extract_query_constraints(question)["mentions_work_listing"]
|
|
|
|
|
|
@pytest.fixture
|
|
def retrieved():
|
|
common = {"person": "Ján Holp", "document_path": "p.md", "source_url": URL}
|
|
theses = [
|
|
{
|
|
"thesis_id": "thesis:bp",
|
|
"thesis_type": "bachelor_thesis",
|
|
"year": 2019,
|
|
"heading": "Bakalárska práca 2019",
|
|
"title": "Vyhľadávač na slovenskom internete",
|
|
},
|
|
{
|
|
"thesis_id": "thesis:dp",
|
|
"thesis_type": "diploma_thesis",
|
|
"year": 2021,
|
|
"heading": "Diplomová práca 2021",
|
|
"title": "Systém získavania informácií v slovenskom jazyku.",
|
|
},
|
|
]
|
|
projects = [
|
|
{
|
|
"work_id": "work:tp",
|
|
"work_type": "team_project",
|
|
"year": 2019,
|
|
"heading": "Tímový projekt 2019",
|
|
"title": None,
|
|
},
|
|
{
|
|
"work_id": "work:dp",
|
|
"work_type": "diploma_project",
|
|
"year": 2020,
|
|
"heading": "Diplomový projekt 2020",
|
|
"title": None,
|
|
},
|
|
{
|
|
"work_id": "work:dp2",
|
|
"work_type": "diploma_project",
|
|
"year": 2020,
|
|
"heading": "Diplomový projekt 2 2020",
|
|
"title": None,
|
|
},
|
|
]
|
|
return {
|
|
"status": "ok",
|
|
"reason": None,
|
|
"complete": True,
|
|
"truncated": False,
|
|
"truncation_reasons": [],
|
|
"index_version": {"version": "fixture"},
|
|
"aggregation": None,
|
|
"constraints": gs.extract_query_constraints(QUESTION),
|
|
"sources": [{"document_path": "p.md", "source_url": URL}],
|
|
"facts": [{**common, **row, "fact_type": "person_thesis"} for row in theses]
|
|
+ [{**common, **row, "fact_type": "person_work"} for row in projects],
|
|
}
|
|
|
|
|
|
def listing_answer(context):
|
|
keys = ["person", "record_id", "title", "year", "work_type", "work_label"]
|
|
return {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"source_urls": [URL],
|
|
"count": None,
|
|
"claims": [
|
|
{key: row[key] for key in keys if row.get(key) is not None}
|
|
for row in context["required_academic_records"]
|
|
],
|
|
}
|
|
|
|
|
|
def completion(answer):
|
|
return {"choices": [{"message": {"content": json.dumps(answer, ensure_ascii=False)}}]}
|
|
|
|
|
|
def test_search_without_year_or_title_fetches_both_academic_kinds():
|
|
fact = {
|
|
"person": "Jakub Maruniak",
|
|
"person_element_id": "p2",
|
|
"work_id": "work:dp2",
|
|
"work_element_id": "w2",
|
|
"heading": "Diplomový projekt 2 2020",
|
|
"year": 2020,
|
|
"work_type": "diploma_project",
|
|
"title": None,
|
|
}
|
|
session = FakeSession({gs.CATALOG_QUERY: sample_catalog(), gs.WORK_FACTS_QUERY: [fact]})
|
|
result = gs.search_graph_with_session(session, "Aké práce má Jakub Maruniak?")
|
|
calls = {query for query, _ in session.calls}
|
|
assert {gs.THESIS_FACTS_QUERY, gs.WORK_FACTS_QUERY} <= calls
|
|
assert result["facts"][0]["work_id"] == "work:dp2"
|
|
assert result["facts"][0]["title"] is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question,expected,excluded",
|
|
[
|
|
("Aké projekty má Jakub Maruniak?", gs.WORK_FACTS_QUERY, gs.THESIS_FACTS_QUERY),
|
|
("Aké záverečné práce má Jakub Maruniak?", gs.THESIS_FACTS_QUERY, gs.WORK_FACTS_QUERY),
|
|
],
|
|
)
|
|
def test_listing_preserves_requested_domain(question, expected, excluded):
|
|
session = FakeSession({gs.CATALOG_QUERY: sample_catalog()})
|
|
gs.search_graph_with_session(session, question)
|
|
calls = {query for query, _ in session.calls}
|
|
assert expected in calls and excluded not in calls
|
|
|
|
|
|
def test_context_preserves_five_records_instead_of_three_years(retrieved):
|
|
context = graph_context(retrieved)
|
|
assert len(context["required_academic_records"]) == 5
|
|
assert len(context["facts"]) == 5
|
|
assert {r["record_id"] for r in context["facts"]} == {
|
|
"thesis:bp",
|
|
"thesis:dp",
|
|
"work:tp",
|
|
"work:dp",
|
|
"work:dp2",
|
|
}
|
|
assert [r["title"] for r in context["facts"]].count(None) == 3
|
|
assert context["complete"] is True
|
|
|
|
|
|
@pytest.mark.parametrize("change", ["years_only", "no_types", "missing_project", "no_record_ids"])
|
|
def test_guard_rejects_incomplete_response_even_when_retrieval_is_complete(retrieved, change):
|
|
context = graph_context(retrieved)
|
|
answer = listing_answer(context)
|
|
if change == "years_only":
|
|
answer["claims"] = [{"person": "Ján Holp", "year": year} for year in (2019, 2020, 2021)]
|
|
elif change == "no_types":
|
|
for claim in answer["claims"]:
|
|
claim.pop("work_type")
|
|
elif change == "missing_project":
|
|
answer["claims"].pop()
|
|
else:
|
|
for claim in answer["claims"]:
|
|
claim.pop("record_id")
|
|
assert "missing_academic_record_details" in answer_context_issues(
|
|
answer, context, check_text=False
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"field,value",
|
|
[
|
|
("year", 2020),
|
|
("work_type", "team_project"),
|
|
("record_id", "work:dp2"),
|
|
("record_id", "thesis/dp"),
|
|
("work_label", "Diplomový projekt 2020"),
|
|
],
|
|
)
|
|
def test_record_fields_cannot_be_mixed_between_works(retrieved, field, value):
|
|
context = graph_context(retrieved)
|
|
answer = listing_answer(context)
|
|
answer["claims"][1][field] = value
|
|
assert "unsupported_academic_binding" in answer_context_issues(
|
|
answer, context, check_text=False
|
|
)
|
|
|
|
|
|
def test_citation_must_support_the_whole_record_not_separate_field_occurrences(retrieved):
|
|
context = graph_context(retrieved)
|
|
claim = {"person": "Ján Holp", "title": "A", "year": 2021}
|
|
context.pop("required_academic_records")
|
|
other = "https://example.test/other"
|
|
context["sources"].append({"source_url": other})
|
|
context["facts"] = [
|
|
{"person": "Ján Holp", "title": "A", "year": 2021, "source_url": URL},
|
|
{"person": "Ján Holp", "title": "A", "year": 2020, "source_url": other},
|
|
{"person": "Ján Holp", "title": "B", "year": 2021, "source_url": other},
|
|
]
|
|
answer = {"status": "ok", "answer": "", "claims": [claim], "source_urls": [other]}
|
|
assert "missing_academic_record_source" in answer_context_issues(
|
|
answer, context, check_text=False
|
|
)
|
|
answer["source_urls"].append(URL)
|
|
assert answer_context_issues(answer, context, check_text=False) == []
|
|
|
|
|
|
def test_missing_title_is_reported_without_invention_and_ids_stay_private(retrieved):
|
|
context = graph_context(retrieved)
|
|
answer, _ = parse_answer(json.dumps(listing_answer(context)))
|
|
assert answer_context_issues(answer, context, check_text=False) == []
|
|
rendered = render_graph_answer(answer, context)
|
|
assert answer_context_issues(rendered, context) == []
|
|
assert rendered["answer"].count("názov v zdroji nie je uvedený") == 3
|
|
assert "Diplomový projekt 2 2020" in rendered["answer"]
|
|
assert "Systém získavania informácií" in rendered["answer"]
|
|
assert "**Bakalárska práca 2019**" in rendered["answer"]
|
|
assert "typ práce:" not in rendered["answer"]
|
|
assert all(claim["record_id"] not in rendered["answer"] for claim in answer["claims"])
|
|
answer["claims"][2]["title"] = "Vymyslený názov projektu"
|
|
assert "unsupported_academic_binding" in answer_context_issues(
|
|
answer, context, check_text=False
|
|
)
|
|
|
|
|
|
def test_unknown_year_is_not_filled_from_heading_or_another_record(retrieved):
|
|
retrieved["facts"][2]["year"] = None
|
|
retrieved["facts"][2]["heading"] = "Tímový projekt"
|
|
context = graph_context(retrieved)
|
|
answer = listing_answer(context)
|
|
rendered = render_graph_answer(answer, context)
|
|
assert "rok v zdroji nie je uvedený" in rendered["answer"]
|
|
assert answer_context_issues(rendered, context) == []
|
|
answer["claims"][2]["year"] = 2019
|
|
assert "unsupported_academic_binding" in answer_context_issues(
|
|
answer, context, check_text=False
|
|
)
|
|
|
|
|
|
def test_known_record_cannot_be_refused(retrieved):
|
|
context = graph_context(retrieved)
|
|
answer = {"status": "no_answer", "answer": "", "claims": [], "source_urls": []}
|
|
assert "rejected_supported_academic_listing" in answer_context_issues(
|
|
answer, context, check_text=False
|
|
)
|
|
|
|
|
|
def test_listing_does_not_require_records_outside_truncated_context(retrieved):
|
|
retrieved.update(complete=False, truncated=True, truncation_reasons=["result_limit"])
|
|
retrieved["facts"] = retrieved["facts"][:2]
|
|
context = graph_context(retrieved)
|
|
rendered = render_graph_answer(listing_answer(context), context)
|
|
assert answer_context_issues(rendered, context) == []
|
|
assert "Výpis je neúplný." in rendered["answer"]
|
|
|
|
|
|
def test_year_only_question_still_returns_only_years(retrieved):
|
|
retrieved["constraints"] = gs.extract_query_constraints("V ktorých rokoch má Ján Holp práce?")
|
|
context = graph_context(retrieved)
|
|
assert "required_academic_records" not in context
|
|
assert len(context["facts"]) == 3
|
|
assert all(
|
|
set(row) == {"person", "year", "document_path", "source_url"} for row in context["facts"]
|
|
)
|
|
|
|
|
|
def test_production_delivery_repairs_year_only_answer_and_renders_all_records(
|
|
monkeypatch, retrieved
|
|
):
|
|
from tools.answers import answer_delivery as delivery
|
|
|
|
from contextlib import nullcontext
|
|
from tools.answers import question_plan
|
|
monkeypatch.setattr(question_plan,'planning_catalog',lambda db:[{'name':'Ján Holp','kind':'Person','source_document':'pages/students/2016/jan_holp/README.md'}])
|
|
monkeypatch.setattr(delivery,'index_snapshot',lambda *a,**kw:nullcontext())
|
|
monkeypatch.setattr(delivery, "search_graph", lambda query, **kwargs: retrieved)
|
|
good = listing_answer(graph_context(retrieved))
|
|
bad = {**good, "claims": [{"person": "Ján Holp", "year": year} for year in (2019, 2020, 2021)]}
|
|
replies = iter([bad, good])
|
|
messages = []
|
|
|
|
def complete(rows):
|
|
messages.append(copy.deepcopy(rows))
|
|
return completion(next(replies))
|
|
|
|
result = delivery.deliver_answer(QUESTION, engine="graphrag", complete=complete)
|
|
assert result["accepted_for_delivery"]
|
|
assert len(result["claims"]) == 5 and len(messages) == 2
|
|
assert "missing_academic_record_details" in result["attempts"][0]["context_issues"]
|
|
assert "Diplomový projekt 2 2020" in result["answer"]
|
|
assert "expected_bindings" not in json.dumps(messages)
|
|
|
|
|
|
def test_two_year_only_attempts_are_never_delivered(retrieved):
|
|
context = graph_context(retrieved)
|
|
answer = {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"source_urls": [URL],
|
|
"claims": [{"person": "Ján Holp", "year": 2021}],
|
|
}
|
|
prompt = Path("tools/answers/graph_answer_prompt.txt").read_text()
|
|
result = generate_checked_answer(QUESTION, context, prompt, lambda _: completion(answer))
|
|
assert not result["accepted_for_delivery"] and len(result["attempts"]) == 2
|