"""Exact/both/negated work selection and proof of actual use, not future tasks.""" import json import sqlite3 import pytest from tools.answers.rag_preparation import prepare_named_content from tools.answers.rag_evidence import rag_evidence_context, project_number, content_sections from tools.answers.verified_rag import generate_verified_rag from tools.answers.content_selection import facets_for from tools.retrieval.work_selector import parse_work_selector from tools.retrieval.question_constraints import requested_years, unsupported_constraint from tools.answers import answer_delivery from tools.rag.llm_context import compact_rag_payload @pytest.fixture def projects(tmp_path): db = tmp_path / "index.sqlite" path = "pages/students/2016/eva_nova/README.md" with sqlite3.connect(db) as c: c.execute("CREATE TABLE documents(path,title,author,published)") c.execute( "CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)" ) c.execute("INSERT INTO documents VALUES(?,?,?,?)", (path, "Eva Nová", "Autor dokumentu", 1)) for i, (label, text) in enumerate( [ ( "Diplomový projekt 2020", "Hotové:\n- Funguje databáza SQLite.\nÚlohy na ďalšie stretnutie:\n- Naštudujte si ako sa používa Redis.", ), ( "Diplomový projekt 2 2020", "Stav:\n- Implementovaný parser pomocou Python.\nÚlohy na ďalšie stretnutie:\n- urobte návrh metódy PageRank.\nDo ďalšieho stretnutia:\n- Naštudujte si ako sa používa Promise.", ), ] ): c.execute( "INSERT INTO chunks VALUES(?,?,?,?,?)", (path, text, json.dumps([["Eva Nová", label]]), 1, i), ) return db def prepare(db, q): raw = { "status": "ok", "reason": None, "sources": [], "context": "", "complete": False, "truncated": True, "truncation_reasons": ["result_limit"], } return compact_rag_payload(prepare_named_content(db, q, raw)) def select(messages): context = json.loads(messages[1]["content"])["context"] a = { "status": "ok", "claims": [], "source_urls": [], "evidence_ids": [u["evidence_id"] for u in context.get("content_candidates", [])], } return {"choices": [{"message": {"content": json.dumps(a)}}]} @pytest.mark.parametrize( "phrase", [ 'pri práci "Diplomový projekt 2020" (nie Diplomový projekt 2)', "pri nečíslovanom diplomovom projekte v roku 2020", "pri diplomovom projekte bez čísla v roku 2020", "pri diplomovom projekte v roku 2020 okrem projektu 2", ], ) def test_exact_unnumbered_and_excluded_number_select_the_actual_work(projects, phrase): q = "Aké metódy používala Eva Nová " + phrase + "?" assert not unsupported_constraint(q) and requested_years(q) == [2020] p = prepare(projects, q) assert p["status"] == "ok" assert [s["work_label"] for s in p["work_scopes"]] == ["Diplomový projekt 2020"] r = generate_verified_rag(q, p, select) assert r["accepted_for_delivery"] and "SQLite" in r["answer"] assert all(x not in r["answer"] for x in ["Python", "PageRank", "Promise", "Redis"]) def test_numbered_work_keeps_actual_methods_and_rejects_future_task_evidence(projects): q = "Aké metódy používala Eva Nová pri diplomovom projekte 2 v roku 2020?" p = prepare(projects, q) c = rag_evidence_context(q, p) assert all(u["phase"] not in {"planned", "assignment"} for u in c["content_candidates"]) r = generate_verified_rag(q, p, select) assert r["accepted_for_delivery"] and "Python" in r["answer"] assert ( "PageRank" not in r["answer"] and "Promise" not in r["answer"] and "SQLite" not in r["answer"] ) bad = { "status": "ok", "claims": [], "source_urls": [s["source_url"] for s in c["sources"]], "evidence_quotes": [ {"source_id": c["sources"][0]["source_id"], "quote": "- urobte návrh metódy PageRank."} ], } r = generate_verified_rag( q, p, lambda _: {"choices": [{"message": {"content": json.dumps(bad)}}]} ) assert ( not r["accepted_for_delivery"] and "quote_outside_delivered_candidates" in r["attempts"][-1]["issues"] ) @pytest.mark.parametrize("word", ["oboch", "obidvoch", "všetkých"]) def test_each_requested_work_has_its_own_methods_and_heading(projects, word): q = f"Aké metódy používala Eva Nová pri {word} diplomových projektoch v roku 2020?" p = prepare(projects, q) c = rag_evidence_context(q, p) assert c["requirements"]["separate_works"] and len(p["work_scopes"]) == 2 r = generate_verified_rag(q, p, select) assert r["accepted_for_delivery"] and r["answer"].count("**Eva Nová — Diplomový projekt") == 2 first, second = r["answer"].split("**Eva Nová — Diplomový projekt 2 2020") assert "SQLite" in first and "Python" not in first and "Python" in second assert "PageRank" not in r["answer"] and "Promise" not in r["answer"] def test_omitting_a_second_work_with_evidence_is_rejected_even_for_the_same_person_and_year( projects, ): q = "Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?" p = prepare(projects, q) def incomplete(messages): c = json.loads(messages[1]["content"])["context"] a = { "status": "ok", "claims": [], "source_urls": [], "evidence_ids": [ u["evidence_id"] for u in c["content_candidates"] if u["work_label"] == "Diplomový projekt 2020" ], } return {"choices": [{"message": {"content": json.dumps(a)}}]} r = generate_verified_rag(q, p, incomplete) assert ( not r["accepted_for_delivery"] and "missing_methods_evidence" in r["attempts"][-1]["issues"] ) def test_missing_methods_are_reported_for_the_right_work_without_stealing_another_works_evidence( projects, ): with sqlite3.connect(projects) as c: c.execute( "UPDATE chunks SET text='Úlohy na ďalšie stretnutie:\n- Použite metódu BERT.' WHERE chunk_index=1" ) q = "Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?" r = generate_verified_rag(q, prepare(projects, q), select) assert r["accepted_for_delivery"] and r["answer_coverage"]["status"] == "partial" assert "SQLite" in r["answer"] and "BERT" not in r["answer"] assert ( "Diplomový projekt 2 2020" in r["answer"] and "skutočne použitých metódach" in r["answer"] ) def test_plans_remain_available_as_plans_with_natural_verbs(projects): q = "Aké plánované úlohy mala Eva Nová pri oboch diplomových projektoch v roku 2020?" r = generate_verified_rag(q, prepare(projects, q), select) assert r["accepted_for_delivery"] assert ( "Urobiť návrh metódy PageRank." in r["answer"] and "Naštudovať si ako sa používa Promise." in r["answer"] ) assert "urobte" not in r["answer"] and "Naštudujte" not in r["answer"] def test_unknown_requested_work_is_not_silently_dropped(projects): q = 'Porovnaj metódy Evy Novej pri "Diplomový projekt 2020" a "Diplomový projekt 3 2020".' p = prepare(projects, q) assert p["status"] == "ambiguous" and p["reason"] == "unresolved_work_selection" def test_short_followup_needs_context_instead_of_global_retrieval(monkeypatch): def forbidden(*a, **kw): pytest.fail("A bare follow-up must not search or call an LLM") monkeypatch.setattr(answer_delivery, "build_rag_context", forbidden) r = answer_delivery.deliver_answer("daj obe", engine="rag", complete=forbidden) assert r["status"] == "ambiguous" and r["reason"] == "missing_followup_context" def test_negative_selector_is_not_a_positive_number_and_literal_title_years_are_not_filters(): q = 'Aké metódy používal Ján Holp pri "Diplomový projekt 2020" (nie Diplomový projekt 2)?' selector = parse_work_selector(q) assert ( project_number(q) is None and selector.excluded_numbers == (2,) and requested_years(q) == [2020] ) assert selector.matches("Diplomový projekt 2020") and not selector.matches( "Diplomový projekt 2 2020" ) assert requested_years('Kto má názov "Projekt 2 o svete 2099"?') == [] assert not parse_work_selector('Kto má názov "Projekt 2 o svete 2099"?').specific assert unsupported_constraint("Aké metódy používal Ján Holp, nie v roku 2020?") == "exclusion" def test_unimplemented_method_is_not_proof_of_actual_use(): assert "methods" not in facets_for( "Metóda BERT ešte nie je implementovaná.", "progress", "Aké metódy skutočne použila Eva Nová?", ) def test_scalar_question_also_preserves_exact_work_and_exclusion(projects): q = 'Aký názov má "Diplomový projekt 2020" Evy Novej (nie Diplomový projekt 2)?' p = prepare(projects, q) assert ( len(p["work_scopes"]) == 1 and p["work_scopes"][0]["work_label"] == "Diplomový projekt 2020" ) assert "Python" not in p["context"] and "Promise" not in p["context"] @pytest.mark.parametrize( "heading", ["Úlohy na ďalšie stretnutie:", "Do ďalšieho stretnutia:", "Do ďalšiehio stretnutia:"], ) def test_source_task_headings_cannot_inherit_a_completed_state(heading): sections = content_sections( "Stav:\n- Implementovaný parser.\n" + heading + "\n- urobte návrh metódy PageRank." ) assert [s["kind"] for s in sections] == ["progress", "task"] assert sections[-1]["phase"] == "planned"