"""Real multi-person query regressions, including deliberately wrong evidence.""" import copy import json import sqlite3 import pytest from tools.answers.context_policy import answer_context_issues, text_contains_value from tools.answers.graph_context import graph_context from tools.answers.presentation import content_lines, goal_text from tools.answers.rag_evidence import ( content_sections, expand_comparison_sources, rag_evidence_context, requested_types, ) from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues from tools.answers.render_answer import render_graph_answer from tools.graphrag import graph_search as gs from tools.graphrag.tests.test_graph_search import FakeSession from tools.rag.llm_context import compact_rag_payload from tools.rag.rag_utils import build_context_text COMPARE = ( "Porovnaj zameranie bakalárskej práce Ondreja Megelu a diplomovej práce Jána Holpa. " "Rozlišuj zadanie od dosiahnutých výsledkov a uveď zdroje." ) SHARED = "V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt?" @pytest.mark.parametrize( "canonical,wording", [ ("Ondrej Megela", "Ondreja Megelu"), ("Martin Šarišský", "Martina Šarišského"), ("Ján Malinovský", "Jána Malinovského"), ("Eva Nová", "Evy Novej"), ], ) def test_complete_case_endings(canonical, wording): assert text_contains_value(wording, canonical, person=True) @pytest.mark.parametrize( "canonical,other", [ ("Martin Šarišský", "Martin Jančura"), ("Ján Holp", "Ján Holpák"), ("Ondrej Megela", "Ondrej Megelovec"), ("Martin NaN1", "Martin NaN2"), ], ) def test_no_short_prefix_matching(canonical, other): assert not text_contains_value(other, canonical, person=True) def test_each_person_gets_the_requested_work_type(): assert requested_types(COMPARE, ["Ján Holp", "Ondrej Megela"]) == { "Ján Holp": "diploma_thesis", "Ondrej Megela": "bachelor_thesis", } def test_goal_conversion_is_not_all_or_nothing(): text = "\n".join( content_lines( [ "- Zistite, čo je to RAG.", "- Naučte sa čo je to GraphRAG.", "- Zostavte malý znalostný graf.", "- Využite graf pri vyhľadávaní.", "- Použite databázu Neo4j.", "Práca neimplementuje vlastný model.", ], "Ján Malinovský", "diploma_project", "Aké sú ciele diplomového projektu Jána Malinovského?", ) ) assert "Diplomový projekt Jána Malinovského" in text assert "Zistiť, čo je to RAG." in text and "Použiť databázu Neo4j." in text assert "Ďalšie podklady:" not in text and "Práca neimplementuje vlastný model." in text assert not any(word in text for word in ["Zistite", "Naučte", "Zostavte", "Využite", "Použite"]) def test_prefixed_goal_keeps_other_converted_items(): text = "\n".join( content_lines( [ "1. Vypracujte prehľad systémov generovania prirodzeného jazyka.", "2. Na základe tohto prehľadu navrhnite a implementujte agentový systém.", ], "Martin Šarišský", "diploma_thesis", "Čomu sa venuje diplomová práca Martina Šarišského?", ) ) assert "Diplomová práca Martina Šarišského" in text assert "Vypracovanie prehľadu" in text and "navrhnúť a implementovať" in text assert "Vypracujte" not in text and "navrhnite" not in text def test_coordinated_commands_and_objects_keep_grammatical_infinitives(): text = "\n".join( content_lines( [ "Vypracujte prehľad aplikácií BERT a spôsoby ich vyhodnotenia.", "Vypracujte prehľad metód a implementujte systém.", ], "Ondrej Megela", "bachelor_thesis", ) ) assert "Vypracovať prehľad aplikácií BERT a spôsoby ich vyhodnotenia." in text assert "Vypracovať prehľad metód a implementovať systém." in text def test_goal_headings_are_not_extra_evidence_and_sentences_keep_capitals(): text = "\n".join( content_lines( ["Ciele na semester.", "Zlepšite proces. Zlepšite extrakciu."], "Ján Malinovský", "diploma_project", ) ) assert "Ďalšie podklady" not in text and "Zlepšiť proces. Zlepšiť extrakciu." in text @pytest.mark.parametrize( "literal,expected", [ ( "Použite https://example.test/pouzite/vypracujte.", "Použiť https://example.test/pouzite/vypracujte.", ), ( "Implementujte parser pre `navrhnite a implementujte`.", "Implementovať parser pre `navrhnite a implementujte`.", ), ], ) def test_grammar_does_not_change_urls_or_code_literals(literal, expected): assert goal_text(literal) == (expected, True) @pytest.fixture def comparison_payload(tmp_path): sources = [] rows = [] for i, (person, slug, kind, year, goal, progress) in enumerate( [ ( "Ján Holp", "jan_holp", "Diplomová práca", 2021, "Implementujte systém vyhľadávania.", "Implementovaný PageRank a indexovanie webových stránok.", ), ( "Ondrej Megela", "ondrej_megela", "Bakalárska práca", 2020, "Natrénujte jazykový model metódou BERT.", "Natrénovaný model wiki103 a vykonané vyhodnotenie.", ), ], 1, ): path = f"pages/students/2018/{slug}/README.md" sources.append( { "source_id": f"S{i}", "title": person, "document_path": path, "source_url": f"https://example.test/{slug}", "text": f"Sekcia: {kind} {year}\nNázov: Ukážkový názov práce.", } ) rows.append( ( path, json.dumps([[person, f"{kind} {year}"]]), f"Zadanie:\n- {goal}\nStav:\n- {progress}", 1, 0, ) ) rows.append( ( path, json.dumps([[person, "Diplomový projekt 2022"]]), "Stav:\nCudzí projekt využíva Cassandra.", 1, 1, ) ) db = tmp_path / "index.sqlite" with sqlite3.connect(db) as conn: conn.execute( "CREATE TABLE chunks(document_path,heading_paths_json,text,published,chunk_index)" ) conn.executemany("INSERT INTO chunks VALUES(?,?,?,?,?)", rows) raw = { "status": "ok", "reason": None, "sources": sources, "context": build_context_text(sources), "complete": False, "truncated": True, "truncation_reasons": ["result_limit"], "index_version": {"version": "fixture"}, } return db, raw def selected_comparison(context): quotes = ( [{"source_id": c["source_id"], "quote": c["text"]} for c in context["content_candidates"]] if "content_candidates" in context else [ {"source_id": p["source_id"], "quote": s["text"]} for p in context["content_passages"] for s in content_sections(p["text"]) if s["kind"] in {"assignment", "progress"} ] ) return { "status": "ok", "answer": "", "claims": [], "evidence_quotes": quotes, "source_urls": [s["source_url"] for s in context["sources"]], } def test_expansion_preserves_binding_versions_and_budget(comparison_payload): db, raw = comparison_payload before = copy.deepcopy(raw) payload = compact_rag_payload(expand_comparison_sources(db, COMPARE, raw)) c = rag_evidence_context(COMPARE, payload) assert c["requirements"]["work_types"] == { "Ján Holp": "diploma_thesis", "Ondrej Megela": "bachelor_thesis", } assert c["requirements"]["content_roles"] == { p: ["assignment", "progress"] for p in c["requirements"]["people"] } assert "Cassandra" not in payload["context"] assert payload["index_version"] == raw["index_version"] and payload["complete"] is False assert payload["context_budget"]["used_tokens"] <= payload["context_budget"]["max_tokens"] assert raw == before def test_hidden_chunks_are_not_added_to_comparison_evidence(comparison_payload): db, raw = comparison_payload path = raw["sources"][0]["document_path"] with sqlite3.connect(db) as conn: conn.execute( "INSERT INTO chunks VALUES(?,?,?,?,?)", ( path, json.dumps([["Ján Holp", "Diplomová práca 2021"]]), "Stav:\nNezverejnený tajný výsledok.", 0, 2, ), ) expanded = expand_comparison_sources(db, COMPARE, raw) assert "tajný výsledok" not in expanded["context"] def test_relevant_child_document_is_preserved(comparison_payload): db, raw = comparison_payload child = { **raw["sources"][0], "source_id": "S3", "title": "Meranie projektu", "document_path": raw["sources"][0]["document_path"].replace( "README.md", "meranie/README.md" ), "text": "Sekcia: Diplomová práca 2021\nStav:\nVyhodnotená presnosť vyhľadávania.", } raw["sources"].append(child) raw["context"] = build_context_text(raw["sources"]) expanded = expand_comparison_sources(db, COMPARE, raw) assert "Vyhodnotená presnosť vyhľadávania." in expanded["context"] def test_comparison_separates_goals_and_results(comparison_payload): db, raw = comparison_payload payload = compact_rag_payload(expand_comparison_sources(db, COMPARE, raw)) c = rag_evidence_context(COMPARE, payload) selected = selected_comparison(c) assert rag_answer_issues(selected, c) == [] result = generate_verified_rag( COMPARE, payload, lambda _: {"choices": [{"message": {"content": json.dumps(selected)}}]} ) assert result["accepted_for_delivery"] text = result["answer"] assert text.startswith("| Oblasť |") assert text.count("| Zadanie a ciele |") == text.count("| Zaznamenané výsledky |") == 1 assert ( "Diplomová práca Jána Holpa (2021)" in text and "Bakalárska práca Ondreja Megelu (2020)" in text ) assert "Natrénovať jazykový model metódou BERT." in text assert "Natrénovaný model wiki103" in text and "Implementovaný PageRank" in text assert "Cassandra" not in text and len(result["source_urls"]) == 2 def test_goals_cannot_replace_available_results(comparison_payload): db, raw = comparison_payload c = rag_evidence_context(COMPARE, expand_comparison_sources(db, COMPARE, raw)) a = selected_comparison(c) a["evidence_quotes"] = [ q for q in a["evidence_quotes"] if not any(word in q["quote"] for word in ["Implementovaný", "Natrénovaný"]) ] assert "missing_progress_evidence" in rag_answer_issues(a, c) def test_missing_results_allow_an_explicit_partial_comparison(comparison_payload): db, raw = comparison_payload with sqlite3.connect(db) as conn: conn.execute( "UPDATE chunks SET text='Zadanie:\n- Implementujte systém vyhľadávania.' WHERE document_path LIKE '%jan_holp%' AND chunk_index=0" ) payload = expand_comparison_sources(db, COMPARE, raw) c = rag_evidence_context(COMPARE, payload) selected = selected_comparison(c) result = generate_verified_rag( COMPARE, payload, lambda _: {"choices": [{"message": {"content": json.dumps(selected)}}]} ) assert result["accepted_for_delivery"] assert "| Zaznamenané výsledky | V podkladoch chýbajú údaje. |" in result["answer"] assert "Implementácia systému vyhľadávania" in result["answer"] def test_future_tasks_are_not_presented_as_achieved_results(comparison_payload): db, raw = comparison_payload with sqlite3.connect(db) as conn: conn.execute( "UPDATE chunks SET text=text || '\nÚlohy:\n- Pripravte experiment s PageRank.' WHERE document_path LIKE '%jan_holp%' AND chunk_index=0" ) payload = expand_comparison_sources(db, COMPARE, raw) c = rag_evidence_context(COMPARE, payload) selected = selected_comparison(c) p = next(p for p in c["content_passages"] if p["person"] == "Ján Holp") selected["evidence_quotes"].append( {"source_id": p["source_id"], "quote": "- Pripravte experiment s PageRank."} ) result = generate_verified_rag( COMPARE, payload, lambda _: {"choices": [{"message": {"content": json.dumps(selected)}}]} ) # The model may not quote a future task omitted from its actual candidates. assert not result["accepted_for_delivery"] assert "quote_outside_delivered_candidates" in result["attempts"][-1]["issues"] @pytest.fixture def shared_session(): catalog = [] theses = [] works = [] for pid, person, slug in [("p1", "Ján Holp", "jan"), ("p2", "Ondrej Megela", "ondrej")]: path = f"pages/students/2018/{slug}/README.md" url = f"https://example.test/{slug}" catalog.append( {"element_id": pid, "kind": "Person", "name": person, "source_document": path} ) base = { "person_element_id": pid, "person": person, "document_path": path, "source_url": url, } theses.append( { **base, "thesis_element_id": pid + "t", "thesis_id": pid + "t", "title": "Ukážkový názov", "year": 2019 if pid == "p1" else 2020, "heading": "Bakalárska práca", "thesis_type": "bachelor_thesis", } ) works.append( { **base, "work_element_id": pid + "w", "work_id": pid + "w", "year": 2020, "heading": "Diplomový projekt", "work_type": "diploma_project", } ) return FakeSession( {gs.CATALOG_QUERY: catalog, gs.THESIS_FACTS_QUERY: theses, gs.WORK_FACTS_QUERY: works} ) @pytest.mark.parametrize("word", ["obaja", "obidvaja"]) def test_both_means_shared_years(word): c = gs.extract_query_constraints(SHARED.replace("obaja", word)) assert c["mentions_shared"] and c["mentions_academic_year"] def test_shared_year_has_only_intersection_and_bound_work_details(shared_session): result = gs.search_graph_with_session(shared_session, SHARED) assert result["academic_year_comparison"]["shared_years"] == [2020] c = graph_context(result) records = c["required_academic_records"] assert len(records) == 3 and all(r["year"] == 2020 for r in records) a = {"status": "ok", "claims": records, "source_urls": [s["source_url"] for s in c["sources"]]} rendered = render_graph_answer(a, c) assert answer_context_issues(rendered, c) == [] assert "Spoločný evidovaný rok je **2020**." in rendered["answer"] assert "2019" not in rendered["answer"] and "Ukážkový názov" in rendered["answer"] def test_shared_year_with_per_person_presentation_instructions(shared_session): question = ( "V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt? " "Najprv uveď spoločné roky, potom pri každej osobe príslušné práce s typom a názvom, " "ak je uvedený. Pridaj zdroje." ) result = gs.search_graph_with_session(shared_session, question) assert result["status"] == "ok" assert result["academic_year_comparison"]["shared_years"] == [2020] context = graph_context(result) records = context["required_academic_records"] assert len(records) == 3 and {r["person"] for r in records} == {"Ján Holp", "Ondrej Megela"} answer = { "status": "ok", "claims": records, "source_urls": [s["source_url"] for s in context["sources"]], } rendered = render_graph_answer(answer, context) assert answer_context_issues(rendered, context) == [] assert "Spoločný evidovaný rok je **2020**." in rendered["answer"] assert "2019" not in rendered["answer"] def test_no_common_year_is_not_a_union(shared_session): for row in shared_session.responses[gs.WORK_FACTS_QUERY]: if row["person_element_id"] == "p1": row["year"] = 2021 result = gs.search_graph_with_session(shared_session, SHARED) assert ( result["status"] == "no_answer" and result["academic_year_comparison"]["shared_years"] == [] ) c = graph_context(result) assert ( "spoločný rok nenašiel" in render_graph_answer({"status": "no_answer", "claims": [], "source_urls": []}, c)[ "answer" ] ) def test_partial_records_do_not_claim_an_unwitnessed_common_year(shared_session): result = gs.search_graph_with_session(shared_session, SHARED, limit=1) c = graph_context(result) a = { "status": "ok", "claims": c["required_academic_records"], "source_urls": [s["source_url"] for s in c["sources"]], } text = render_graph_answer(a, c)["answer"] assert "Výpis je neúplný." in text and "Spoločný evidovaný rok je" not in text