diff --git a/experiments/user_questions_v1/repair_verification.json b/experiments/user_questions_v1/repair_verification.json new file mode 100644 index 0000000..81d5608 --- /dev/null +++ b/experiments/user_questions_v1/repair_verification.json @@ -0,0 +1,800 @@ +{ + "checked_at_utc": "2026-11-09T20:18:46.810794+00:00", + "state": "IMPLEMENTED_REGRESSION_VERIFIED_AWAITING_CLAUDE_UI_AND_JOINT_CLOSURE", + "scope": "Repair verification of known cases and operational components; NOT a new held-out accuracy measurement.", + "base_commit": "c2e516c1e538acf8fa5eabeebd48cdc039fb1a77", + "index_version": { + "version": "904884c1a86a45f39bbeccaec0e5e823", + "git_revision": "c3ec9f611e645a2fe14fe66d518eb0924bf9bb40", + "created_at_utc": "2026-11-08T20:49:39.997566+00:00", + "source_sha256": "201798bf276106986e7ef01b4d6357960cb495d09c9855b7df922c1df4fb35a7", + "documents_sha256": "8b7eea54f6233cdc9d16610296d32b9336ab0cbaa4b881872a088c05464960be", + "chunks_sha256": "8357d1e0641206990d26827b94f37916661595261f368e30f828bccd03171c95", + "embedding_model": "intfloat/multilingual-e5-small", + "publication_policy": "explicit-false-hidden-v1", + "embedding_fingerprint": "d066fb9a266a1a0c77c65da75dcf538cfab5c87d5c605ff26b1fd2da8a12aa92" + }, + "content_version": "33c3062af9f1aee887e8950f3ec51b8f", + "checks": { + "full_pytest": { + "command": ".venv/bin/python -m pytest -q test tools -o faulthandler_timeout=45", + "passed": 1543, + "skipped": 90, + "note": "Skipped checks are not counted as passed. A separate live Neo4j scope test previously passed 5 cases." + }, + "client_contract_regression": { + "passed": 104, + "note": "Exact metadata result exercised through actual packages.claude.client.query_backend, including coverage enum validation." + }, + "known_live_question_regressions": { + "cases": 44, + "passed": 44, + "per_engine": 22, + "method": "Expected status, accepted delivery, mandatory anchor terms and selected forbidden substitutions. Followed by source-binding audit and AI inspection of teacher-style answers; not a fully independent semantic judge." + }, + "post_contract_fix_metadata_replay": { + "cases": 14, + "passed": 14 + }, + "source_checkpoint_audit": { + "units": 6293, + "source_files": 72, + "whitespace_only_normalizations": 806, + "literal_errors": [], + "extraction_schema": 2, + "coverage": { + "decisions": { + "irrelevant": 463, + "uncertain": 419, + "structural": 877, + "knowledge": 4534 + }, + "units": 6293 + } + }, + "assertion_and_relation_invariants": { + "units": 6293, + "activated_relations": 2, + "errors": [] + }, + "returned_evidence_audit": { + "cases": 44, + "evidence_occurrences_checked": 85, + "evidence_source_files": 18, + "source_binding_errors": [], + "metadata_versions": [ + "904884c1a86a45f39bbeccaec0e5e823" + ], + "metadata_proofs": [ + { + "person": "Ronald Zalacko", + "document_author": "Daniel Hladek", + "source_path": "pages/students/2023/ronald_zalacko/README.md", + "source_url": "https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko", + "metadata_field": "author" + }, + { + "person": "Ronald Zalacko", + "document_author": "Daniel Hladek", + "source_path": "pages/students/2023/ronald_zalacko/README.md", + "source_url": "https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko", + "metadata_field": "author" + }, + { + "person": "Yehor Piliavin", + "start_year": 2024, + "source_path": "pages/students/2024/yehor_piliavin/README.md", + "source_url": "https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin", + "metadata_field": "literal_start_year", + "chunk_id": "pages/students/2024/yehor_piliavin/README.md::chunk-0", + "quote": "rok začiatku štúdia: 2024" + }, + { + "person": "Yehor Piliavin", + "start_year": 2024, + "source_path": "pages/students/2024/yehor_piliavin/README.md", + "source_url": "https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin", + "metadata_field": "literal_start_year", + "chunk_id": "pages/students/2024/yehor_piliavin/README.md::chunk-0", + "quote": "rok začiatku štúdia: 2024" + } + ], + "checks": "Exact comparison to frozen checkpoint of owner, work, phase, time and source. Literal checkpoint text was separately checked against original Markdown. This is provenance validation, not a claim of perfect semantic completeness." + }, + "runtime": { + "sqlite_integrity": "ok", + "foreign_key_errors": 0, + "counts": { + "chunks": 947, + "chunks_fts": 947 + }, + "metadata_versions_equal": true, + "content_graph_version_matches_checkpoint": true, + "content_graph_hash_matches_checkpoint": true, + "deployed_files_match_workspace": true, + "capabilities": { + "text_search": { + "ready": true, + "status": "ready", + "reason": null + }, + "metadata_graph": { + "ready": true, + "status": "ready", + "reason": null + }, + "knowledge": { + "ready": true, + "status": "ready", + "reason": null + }, + "generation": { + "configured": true, + "ready": true, + "status": "ready", + "reason": null + } + } + }, + "oauth": { + "passed": true, + "steps": [ + "challenge and protected-resource discovery", + "S256 PKCE", + "dynamic client registration", + "owner login and consent", + "invalid password rejection", + "invalid verifier rejection", + "authorization code exchange and replay rejection", + "authenticated MCP discovery and readiness", + "refresh rotation", + "grant revocation" + ], + "test_grants_revoked": true + }, + "public_mcp": { + "url": "https://pressed-extra-graduation-readings.trycloudflare.com/mcp", + "tools": [ + "zpwiki_rag", + "zpwiki_graphrag", + "zpwiki_health" + ], + "checks": [ + { + "tool": "zpwiki_health", + "status": "ok", + "expected": null, + "passed": true, + "seconds": 0.1 + }, + { + "tool": "zpwiki_rag", + "status": "ok", + "expected": "ok", + "passed": true, + "seconds": 0.8 + }, + { + "tool": "zpwiki_graphrag", + "status": "ok", + "expected": "ok", + "passed": true, + "seconds": 5.7 + }, + { + "tool": "zpwiki_graphrag", + "status": "no_answer", + "expected": "no_answer", + "passed": true, + "seconds": 2.8 + }, + { + "tool": "zpwiki_rag", + "status": "ok", + "expected": "ok", + "passed": true, + "seconds": 3.2 + }, + { + "tool": "zpwiki_graphrag", + "status": "ok", + "expected": "ok", + "passed": true, + "seconds": 9.9 + } + ], + "actual_claude_ui_verified_this_run": false + } + }, + "implemented_steps": [ + { + "step": "Consistent source and publication", + "verification": "Read-only local wiki revision matches the shared index; SQLite, metadata graph and content checkpoint/graph agree. No source wiki writes." + }, + { + "step": "Scoped extraction", + "verification": "All 6293 units have a decision and source binding; actor, work, phase and time invariants checked. Uncertain facts remain uncertain. Activated arbitrary LLM concept relations are conservative and sparse." + }, + { + "step": "Common typed question plan", + "verification": "Regression cases cover quote selectors, multiple people, style clauses, inflected/typo names, roles and unknown conditions; no arbitrary model Cypher." + }, + { + "step": "Hybrid RAG", + "verification": "Content path uses independent FTS5/BM25 and E5 candidate ranks with RRF; a unit regression requires a semantic-only candidate to survive a lexical hit. Named finite scopes expand without dropping goals." + }, + { + "step": "Local GraphRAG", + "verification": "Content path uses bounded Neo4j person/work/activity/evidence traversal with checkpoint validation. Metadata stays on the graph route; no model-generated Cypher." + }, + { + "step": "Controlled prose and exact metadata", + "verification": "Descriptions use guarded synthesis; unsafe prose may fall back to source-backed extraction. Exact named metadata has deterministic source evidence and a tested MCP client contract." + }, + { + "step": "Evaluation", + "verification": "Original consumed 96-question TEST is retained byte-for-byte. Current 44 known regressions and 14 contract replays do not replace an independent accuracy estimate." + } + ], + "retrieval_routes_in_live_regression": { + "metadata_or_clarification": 18, + "bm25_e5_rrf_assertions": 12, + "bounded_local_graph": 13, + "content_checkpoint": 1 + }, + "synthesis_outcomes_in_live_regression": { + "not_applicable": 31, + "fallback": 7, + "accepted": 6 + }, + "case_results": [ + { + "id": "holp_title", + "engine": "rag", + "query": "Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?", + "passed": true, + "status": "ok", + "answer": "Ján Holp má diplomovú prácu s názvom **Systém získavania informácií v slovenskom jazyku** (rok práce: **2021**).\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + }, + { + "id": "quoted_project", + "engine": "rag", + "query": "Pri osobe Adrián Ondra over názov záznamu \"Bakalársky projekt 2027\" a pripoj evidovaný rok práce.", + "passed": true, + "status": "ok", + "answer": "Adrián Ondra má bakalársky projekt s názvom **Automatická tvorba myšlienkových máp** (rok práce: **2027**).\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra" + ] + }, + { + "id": "holp_title", + "engine": "graphrag", + "query": "Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?", + "passed": true, + "status": "ok", + "answer": "Ján Holp má diplomovú prácu s názvom **Systém získavania informácií v slovenskom jazyku** (rok práce: **2021**).\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + }, + { + "id": "quoted_project", + "engine": "graphrag", + "query": "Pri osobe Adrián Ondra over názov záznamu \"Bakalársky projekt 2027\" a pripoj evidovaný rok práce.", + "passed": true, + "status": "ok", + "answer": "Adrián Ondra má bakalársky projekt s názvom **Automatická tvorba myšlienkových máp** (rok práce: **2027**).\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra" + ] + }, + { + "id": "goals", + "engine": "rag", + "query": "Aké sú ciele záznamu \"Bakalársky projekt 2027\" osoby Adrián Ondra? Zhrň všetky doložené body vlastnými slovami a nezamieňaj ich za hotové výsledky.", + "passed": true, + "status": "ok", + "answer": "**Adrián Ondra — Bakalársky projekt 2027**\n\nZadanie a ciele:\n\n- Vytvoriť vizualizáciu myšlienkovej mapy z automaticky extrahovaných entít. (navrhovaný cieľ)\n\n- Vytvoriť nástroj ktorý sa dá využiť pri vzdelávaní. (navrhovaný cieľ)\n\n- Vyhodnotiť kvalitu vytvoreného grafu. Ako veľmi je graf pravdivý vzhľadom na pôvvodný text. (navrhovaný cieľ)\n\nAdrián Ondra — Bakalársky projekt 2027: zdroj tieto body označuje ako možné/navrhované ciele, nie ako potvrdené splnené výsledky.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra" + ] + }, + { + "id": "goals", + "engine": "graphrag", + "query": "Aké sú ciele záznamu \"Bakalársky projekt 2027\" osoby Adrián Ondra? Zhrň všetky doložené body vlastnými slovami a nezamieňaj ich za hotové výsledky.", + "passed": true, + "status": "ok", + "answer": "**Adrián Ondra — Bakalársky projekt 2027**\n\nZadanie a ciele:\n\n- Vytvoriť vizualizáciu myšlienkovej mapy z automaticky extrahovaných entít. (navrhovaný cieľ)\n\n- Vytvoriť nástroj ktorý sa dá využiť pri vzdelávaní. (navrhovaný cieľ)\n\n- Vyhodnotiť kvalitu vytvoreného grafu. Ako veľmi je graf pravdivý vzhľadom na pôvvodný text. (navrhovaný cieľ)\n\nAdrián Ondra — Bakalársky projekt 2027: zdroj tieto body označuje ako možné/navrhované ciele, nie ako potvrdené splnené výsledky.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra" + ] + }, + { + "id": "discovery", + "engine": "rag", + "query": "Kto už vyskúšal hierarchickú sumarizáciu na problém s dlhým kontextom? Stačí jeden doložený príklad.", + "passed": true, + "status": "ok", + "answer": "**Simona Bobrovčanová**\n\n- Téma alebo názov doložený pri tej istej práci: Automatická sumarizácia slovenského textu\n\n- Bakalárska práca 2026; zaznamenaný výsledok; záznam 2026-02-27: Vyskúšaná hierarchická sumarizácia - text je rozdelený na časti a potom sa vykoná sumár zo sumárov. Funguje to \"dobre\".\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova" + ] + }, + { + "id": "discovery", + "engine": "graphrag", + "query": "Kto už vyskúšal hierarchickú sumarizáciu na problém s dlhým kontextom? Stačí jeden doložený príklad.", + "passed": true, + "status": "ok", + "answer": "**Simona Bobrovčanová**\n\n- Bakalárska práca 2026; zaznamenaný výsledok; záznam 2026-02-27: Vyskúšaná hierarchická sumarizácia - text je rozdelený na časti a potom sa vykoná sumár zo sumárov. Funguje to \"dobre\".\n\n- Bakalárska práca 2026; priebežný stav; záznam 2025-12-05: Vyzerá to tak, že problémom je dĺžka kontextu. Modely často vynechajú informácie na konci.\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova\n\nPrechod grafom dosiahol limit výberu. Zúž otázku na prácu, tému alebo obdobie.", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova" + ] + }, + { + "id": "technologies", + "engine": "rag", + "query": "Aké technológie má Ronald Zalacko na frontende, backende a v databáze a čo je doložené o funkčnosti aplikácie?", + "passed": true, + "status": "ok", + "answer": "**Ronald Zalacko — Bakalárska práca 2026**\n\nZaznamenané výsledky (dátum záznamu: 2025-12-08):\n- Funkčnosť aplikácie je doložená tým, že aplikácia funguje. [1]\n\nZameranie (dátum záznamu: 2025-12-08):\n- Na fronte sa používa React s Vite JS, na backende Flask a v databáze Postgres so SQLAlchemy. [1]\n- Aplikácia využíva Websocket cez socket.io a JWT Token na autentifikáciu. [1]\n\nZdroje: [1](https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko)\n\n[1]: https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko" + ] + }, + { + "id": "technologies", + "engine": "graphrag", + "query": "Aké technológie má Ronald Zalacko na frontende, backende a v databáze a čo je doložené o funkčnosti aplikácie?", + "passed": true, + "status": "ok", + "answer": "**Ronald Zalacko — Bakalárska práca 2026**\n\nZaznamenané výsledky (dátum záznamu: 2025-12-08):\n- Funkčnosť celej aplikácie je doložená tým, že aplikácia funguje. [1]\n\nZameranie (dátum záznamu: 2025-12-08):\n- Na fronte sa používa React s Vite JS, na backende Flask a v databáze Postgres so SQLAlchemy. [1]\n- Komunikácia v reálnom čase je riešená pomocou Websocket a socket.io. [1]\n- Autentifikácia je zabezpečená prostredníctvom JWT Token. [1]\n\nZdroje: [1](https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko)\n\n[1]: https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko" + ] + }, + { + "id": "problem_remedy", + "engine": "graphrag", + "query": "S akým problémom pri dlhých textoch sa stretla Simona Bobrovčanová a aký prístup už vyskúšala na jeho riešenie?", + "passed": true, + "status": "ok", + "answer": "**Simona Bobrovčanová — Bakalárska práca 2026**\n\nRiešené problémy:\n\n- Vyzerá to tak, že problémom je dĺžka kontextu. Modely často vynechajú informácie na konci. (priebežný stav)\n\nMetódy a technológie:\n\n- Vyskúšaná hierarchická sumarizácia - text je rozdelený na časti a potom sa vykoná sumár zo sumárov. Funguje to \"dobre\". (zaznamenaný výsledok)\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova" + ] + }, + { + "id": "problem_remedy", + "engine": "rag", + "query": "S akým problémom pri dlhých textoch sa stretla Simona Bobrovčanová a aký prístup už vyskúšala na jeho riešenie?", + "passed": true, + "status": "ok", + "answer": "**Simona Bobrovčanová — Bakalárska práca 2026**\n\nRiešené problémy:\n\n- Vyzerá to tak, že problémom je dĺžka kontextu. Modely často vynechajú informácie na konci. (priebežný stav)\n\nMetódy a technológie:\n\n- Vyskúšaná hierarchická sumarizácia - text je rozdelený na časti a potom sa vykoná sumár zo sumárov. Funguje to \"dobre\". (zaznamenaný výsledok)\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova" + ] + }, + { + "id": "all_tasks", + "engine": "rag", + "query": "Aké úlohy dostal Denis Le Thanh podľa záznamu zo stretnutia 3.10.2025? Odlíš budúce úlohy od toho, čo už bolo urobené.", + "passed": true, + "status": "ok", + "answer": "**Denis Le Thanh — Bakalárska práca 2026**\n\nPlánované úlohy:\n\n- Naštudovať a vyskúšať PEFT-QLORA. (plán)\n\n- Vyskúšajte si skript pre dotrénovanie Slovak Mistral.(poslal som cez Teams) (plán)\n\n- Oboznámte sa s Huggingface TRL. (plán)\n\n- Oboznámte sa s knižnicou \"unsloth\". (plán)\n\n- Oboznámte sa s [odkaz neoverený] (plán)\n\n- Pracujte na teoretickej časti: opíšte základné pojmy, metódy a dátové množiny. Používajte google scholar a bibliografické odkazy. (plán)\n\n- Najprv pracujte s domácou GPU, ak nebude stačiť pracujte s Google Coolab, ak nebude stačiť požiadajte konzultanta. (plán)\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/denis_le_thanh\n\nRepozitárový odkaz zo zdroja sa nepodarilo verejne overiť, preto ho neuvádzam. Nedostupnosť ani HTTP 404 samy nerozlišujú zmazaný a súkromný repozitár.", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/denis_le_thanh" + ] + }, + { + "id": "all_tasks", + "engine": "graphrag", + "query": "Aké úlohy dostal Denis Le Thanh podľa záznamu zo stretnutia 3.10.2025? Odlíš budúce úlohy od toho, čo už bolo urobené.", + "passed": true, + "status": "ok", + "answer": "**Denis Le Thanh — Bakalárska práca 2026**\n\nPlánované úlohy:\n\n- Naštudovať a vyskúšať PEFT-QLORA. (plán)\n\n- Vyskúšajte si skript pre dotrénovanie Slovak Mistral.(poslal som cez Teams) (plán)\n\n- Oboznámte sa s Huggingface TRL. (plán)\n\n- Oboznámte sa s knižnicou \"unsloth\". (plán)\n\n- Oboznámte sa s [odkaz neoverený] (plán)\n\n- Pracujte na teoretickej časti: opíšte základné pojmy, metódy a dátové množiny. Používajte google scholar a bibliografické odkazy. (plán)\n\n- Najprv pracujte s domácou GPU, ak nebude stačiť pracujte s Google Coolab, ak nebude stačiť požiadajte konzultanta. (plán)\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/denis_le_thanh\n\nRepozitárový odkaz zo zdroja sa nepodarilo verejne overiť, preto ho neuvádzam. Nedostupnosť ani HTTP 404 samy nerozlišujú zmazaný a súkromný repozitár.", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/denis_le_thanh" + ] + }, + { + "id": "comparison", + "engine": "rag", + "query": "Porovnaj zadanie osoby Adrián Ondra pri \"Bakalársky projekt 2027\" so zadaním osoby Samuel Štucka pri \"Bakalársky projekt 2027\". Vysvetli rozdiel v cieľoch a uveď oba zdroje.", + "passed": true, + "status": "ok", + "answer": "| Oblasť | Adrián Ondra — Bakalársky projekt 2027 | Samuel Štucka — Bakalársky projekt 2027 |\n| --- | --- | --- |\n| Plánované úlohy | Adrián Ondra má vytvoriť vizualizáciu myšlienkovej mapy z automaticky extrahovaných entít. [1] Jeho cieľom je vyvinúť nástroj využiteľný pri vzdelávaní. [1] Plánuje vyhodnotiť kvalitu vytvoreného grafu a jeho pravdivosť vzhľadom na pôvodný text. [1] | Samuel Štucka má pripraviť čistý korpus novinových článkov s metainformáciami. [2] Korpus má uchovávať HTML aj obrázky bez reklám a rozhrania, pričom musí extrahovať údaje ako autor, zdroj, dátum vydania, sekcia, kľúčové slová, nadpis a perex. [2] Má implementovať pipeline na extrakciu a normalizáciu textu z rôznych zdrojov. [2] |\n\nAdrián Ondra — Bakalársky projekt 2027: zdroj tieto body označuje ako možné/navrhované ciele, nie ako potvrdené splnené výsledky.\n\nSamuel Štucka — Bakalársky projekt 2027: zdroj tieto body označuje ako možné/navrhované ciele, nie ako potvrdené splnené výsledky.\n\nZdroje: [1](https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra), [2](https://zp.kemt.fei.tuke.sk/students/2024/samuel_stucka)\n\n[1]: https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra\n[2]: https://zp.kemt.fei.tuke.sk/students/2024/samuel_stucka", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra", + "https://zp.kemt.fei.tuke.sk/students/2024/samuel_stucka" + ] + }, + { + "id": "comparison", + "engine": "graphrag", + "query": "Porovnaj zadanie osoby Adrián Ondra pri \"Bakalársky projekt 2027\" so zadaním osoby Samuel Štucka pri \"Bakalársky projekt 2027\". Vysvetli rozdiel v cieľoch a uveď oba zdroje.", + "passed": true, + "status": "ok", + "answer": "| Oblasť | Adrián Ondra — Bakalársky projekt 2027 | Samuel Štucka — Bakalársky projekt 2027 |\n| --- | --- | --- |\n| Plánované úlohy | Cieľom Adriána Ondru je vytvoriť vizualizáciu myšlienkovej mapy z automaticky extrahovaných entít. [1] Adrián Ondra plánuje vyvinúť nástroj určený na využitie pri vzdelávaní. [1] Plán Adriána Ondru zahŕňa vyhodnotenie kvality vytvoreného grafu a jeho pravdivosti vzhľadom na pôvodný text. [1] | Cieľom Samuela Štucku je pripraviť čistý korpus novinových článkov s metainformáciami. [2] Samuel Štucka plánuje, aby korpus uchovával HTML a obrázky z článkov bez reklám a rozhrania, pričom má extrahovať čo najviac metainformácií ako autor, zdroj, dátum vydania, sekcia, kľúčové slová, nadpis a perex, a zachovať odkazy na obrázky s ich titulkami a popisom. [2] Samuel Štucka má implementovať pipeline na extrakciu a normalizáciu textu z rôznych zdrojov. [2] |\n\nAdrián Ondra — Bakalársky projekt 2027: zdroj tieto body označuje ako možné/navrhované ciele, nie ako potvrdené splnené výsledky.\n\nSamuel Štucka — Bakalársky projekt 2027: zdroj tieto body označuje ako možné/navrhované ciele, nie ako potvrdené splnené výsledky.\n\nZdroje: [1](https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra), [2](https://zp.kemt.fei.tuke.sk/students/2024/samuel_stucka)\n\n[1]: https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra\n[2]: https://zp.kemt.fei.tuke.sk/students/2024/samuel_stucka", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra", + "https://zp.kemt.fei.tuke.sk/students/2024/samuel_stucka" + ] + }, + { + "id": "similar", + "engine": "rag", + "query": "Kto sa zaoberal obsahovo podobnou témou ako Ján Pták? Vysvetli konkrétne prepojenie a dolož obe osoby.", + "passed": true, + "status": "ok", + "answer": "**Oleh Poiasnik**\n\n- Diplomová práca 2027; podklad: Opíšte spôsoby vyhľadávania pomcou znalostných grafov.\n\n**Valerii Kutsenko**\n\n- Diplomová práca 2027; plán; záznam 27.4. (rok neuvedený): Získajte vzorový znalostný graf ktorý sa bude dať použiť pri výskume.\n\n- Diplomová práca 2027; plán; záznam 27.4. (rok neuvedený): Navrhnúť spôsob vyhľadávania s pomocou existujúceho znalostného grafu.\n\nPodklady k porovnávanej osobe:\n\n- Ján Pták; Diplomový projekt 2026: Agent pre manažment záverečných prác\n\n- Ján Pták; Diplomová práca 2027: Agent pre manažment záverečných prác\n\n- Ján Pták; Diplomová práca 2027: - Funguje vektorové a textové vyhľadávanie. Ale vyzerá to tak, že nevie synonymy.\n\n- Ján Pták; Diplomová práca 2027: - Pokračujte v GraphRAG - Naprogramujte načítanie existujúcich Markdown textov do Neo4j — vytvorte uzly pre každú záverečnú prácu, autora, tému a kľúčové slová. Vytvorte grafové relácie medzi uzlami (napr. rovnaký autor → rovnaká téma, rovnaký študijný program)\n\n- Ján Pták; Diplomová práca 2027: - Agenta rozdeľte na samostatné komponenty (nástroje - tools alebo skills). Každý nástroj dajte do osobitného adresára na GIte. Každý nástroj by sa mal z gitu dať nainštalovať do OpenWebUI.\n\n- Ján Pták; Diplomový projekt 2026: - Odovzdané nejaké zdrojové kódy na https://git.kemt.fei.tuke.sk/jp170na/dp-zp-agent - načítanie z markdown, indexovanie do SQLite a FastAPI.\n\nIde o obsahovú podobnosť podloženú uvedenými úryvkami, nie dôkaz spolupráce ani rovnakej fázy dokončenia.\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik, https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko, https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik", + "https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak" + ] + }, + { + "id": "shared_tags", + "engine": "rag", + "query": "Ktoré tagy spájajú osoby Yehor Piliavin a Kristián Sokolský? Uveď iba spoločný prienik a zdroje.", + "passed": true, + "status": "ok", + "answer": "Spoločné kľúčové slová: **alignment**, **llm**.\nDoložené pri: **Yehor Piliavin**, **Kristián Sokolský**.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin, https://zp.kemt.fei.tuke.sk/students/2024/kristian_sokolsky", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin", + "https://zp.kemt.fei.tuke.sk/students/2024/kristian_sokolsky" + ] + }, + { + "id": "shared_tags", + "engine": "graphrag", + "query": "Ktoré tagy spájajú osoby Yehor Piliavin a Kristián Sokolský? Uveď iba spoločný prienik a zdroje.", + "passed": true, + "status": "ok", + "answer": "Spoločné kľúčové slová: **LLM**, **alignment**.\nDoložené pri: **Kristián Sokolský**, **Yehor Piliavin**.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2024/kristian_sokolsky, https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/kristian_sokolsky", + "https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin" + ] + }, + { + "id": "shared_years", + "engine": "rag", + "query": "V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt? Najprv uveď spoločné roky, potom príslušné práce s typom a názvom.", + "passed": true, + "status": "ok", + "answer": "Spoločný evidovaný rok je **2020**.\n\nJán Holp má v dostupných podkladoch tieto práce a projekty:\n\n1. **Diplomový projekt 2 2020** — názov v zdroji nie je uvedený.\n2. **Diplomový projekt 2020** — názov v zdroji nie je uvedený.\n\nOndrej Megela má v dostupných podkladoch tieto práce a projekty:\n\n3. **Bakalárska práca 2020** — **Neurónové jazykové modelovanie typu BERT**.\n4. **Bakalársky projekt 2020** — názov v zdroji nie je uvedený.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp, https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela" + ] + }, + { + "id": "shared_years", + "engine": "graphrag", + "query": "V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt? Najprv uveď spoločné roky, potom príslušné práce s typom a názvom.", + "passed": true, + "status": "ok", + "answer": "Spoločný evidovaný rok je **2020**.\n\nOndrej Megela má v dostupných podkladoch tieto práce a projekty:\n\n1. **Bakalárska práca 2020** — **Neurónové jazykové modelovanie typu BERT**.\n2. **Bakalársky projekt 2020** — názov v zdroji nie je uvedený.\n\nJán Holp má v dostupných podkladoch tieto práce a projekty:\n\n3. **Diplomový projekt 2 2020** — názov v zdroji nie je uvedený.\n4. **Diplomový projekt 2020** — názov v zdroji nie je uvedený.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela, https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela", + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + }, + { + "id": "unknown", + "engine": "rag", + "query": "Aké práce má Xaver Neexistujúci?", + "passed": true, + "status": "no_answer", + "answer": "Meno „Xaver Neexistujúci“ sa nepodarilo spoľahlivo priradiť k osobe v dostupných podkladoch. Uveď presné celé meno.", + "source_urls": [] + }, + { + "id": "unknown", + "engine": "graphrag", + "query": "Aké práce má Xaver Neexistujúci?", + "passed": true, + "status": "no_answer", + "answer": "Meno „Xaver Neexistujúci“ sa nepodarilo spoľahlivo priradiť k osobe v dostupných podkladoch. Uveď presné celé meno.", + "source_urls": [] + }, + { + "id": "partial_name", + "engine": "rag", + "query": "Čo sa podarilo Samuelovi pri jeho práci?", + "passed": true, + "status": "ambiguous", + "answer": "Meno „Samuelovi“ nie je jednoznačné. Ktorú osobu máš na mysli? Možnosti: Samuel Širotník [pages/students/2018/samuel_sirotnik/README.md]; Samuel Horáni [pages/students/2019/samuel_horani/README.md]; Samuel Vasiľ [pages/students/2022/samuel_vasil/README.md]; Samuel Džurina [pages/students/2023/samuel_dzurina/README.md]; Samuel Štucka [pages/students/2024/samuel_stucka/README.md]. Uveď celé meno.", + "source_urls": [] + }, + { + "id": "partial_name", + "engine": "graphrag", + "query": "Čo sa podarilo Samuelovi pri jeho práci?", + "passed": true, + "status": "ambiguous", + "answer": "Meno „Samuelovi“ nie je jednoznačné. Ktorú osobu máš na mysli? Možnosti: Samuel Širotník [pages/students/2018/samuel_sirotnik/README.md]; Samuel Horáni [pages/students/2019/samuel_horani/README.md]; Samuel Vasiľ [pages/students/2022/samuel_vasil/README.md]; Samuel Džurina [pages/students/2023/samuel_dzurina/README.md]; Samuel Štucka [pages/students/2024/samuel_stucka/README.md]. Uveď celé meno.", + "source_urls": [] + }, + { + "id": "wrong_year", + "engine": "rag", + "query": "Akú prácu mal Ján Holp v roku 1999?", + "passed": true, + "status": "no_answer", + "answer": "Ján Holp: požadované údaje sa v dostupných podkladoch nenašli.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + }, + { + "id": "wrong_year", + "engine": "graphrag", + "query": "Akú prácu mal Ján Holp v roku 1999?", + "passed": true, + "status": "no_answer", + "answer": "Ján Holp: v dostupných podkladoch sa nenašiel záznam práce alebo projektu z roku **1999**.", + "source_urls": [] + }, + { + "id": "similar", + "engine": "graphrag", + "query": "Kto sa zaoberal obsahovo podobnou témou ako Ján Pták? Vysvetli konkrétne prepojenie a dolož obe osoby.", + "passed": true, + "status": "ok", + "answer": "Ján Pták a Matej Ščišľak: doložená spoločná téma — GraphRAG, Neo4j. [1] [3]\n\nJán Pták a Oleh Poiasnik: doložená spoločná téma — znalostný graf. [1] [2]\n\nJán Pták a Valerii Kutsenko: doložená spoločná téma — znalostný graf. [1] [4]\n\nJán Pták a Maksym Zatirka: doložená spoločná téma — znalostný graf. [1] [6]\n\nJán Pták a Ján Malinovský: doložená spoločná téma — znalostný graf. [1] [5]\n\n**Ján Pták — Diplomová práca 2027**\n\nZaznamenaný priebeh (dátum záznamu: 2026-09-30):\n- Ján Pták začal pracovať na systéme GraphRAG a grafovej databáze Neo4J. [1]\n\nPlánované úlohy (dátum záznamu: 2026-11-05):\n- Ján Pták plánuje oboznámiť sa s pojmom znalostný graf a napísať o ňom vlastnými slovami. [1]\n\n**Oleh Poiasnik — Diplomový projekt 2026**\n\nZaznamenané výsledky (dátum záznamu: 14.5. (rok neuvedený)):\n- Oleh Poiasnik vytvoril pomocou LightRAG znalostný graf pre 500 liekov. [2]\n\n**Matej Ščišľak — Diplomová práca 2026**\n\nZaznamenané výsledky (dátum záznamu: 2025-03-30):\n- Matej Ščišľak vytvoril systém GraphRAG zložený z FAISS a Neo4J, kde Neo4J beží lokálne a komunikuje so serverom. [3]\n\n**Valerii Kutsenko — Diplomová práca 2027**\n\nZaznamenaný priebeh (dátum záznamu: 15.5. (rok neuvedený)):\n- Valerii Kutsenko zaindexoval anglické dáta z knihy Alice in Wonderland pomocou LightRAG a zostavil z nich znalostný graf. [4]\n\n**Ján Malinovský — Diplomový projekt 2026**\n\nZadanie a ciele:\n- Ján Malinovský má za úlohu zistiť, čo je to znalostný graf. [5]\n\n**Maksym Zatirka — Bakalárska práca 2026**\n\nZadanie a ciele:\n- Maksym Zatirka má za úlohu zostaviť znalostný graf pre slovenské právo a vložiť ho do vybranej grafovej databázy. [6]\n\nPrepojenie dokladá uvedenú tému alebo činnosť; nepreukazuje spoluprácu ani rovnaký stupeň dokončenia.\n\nPrechod grafom dosiahol limit výberu. Zúž otázku na prácu, tému alebo obdobie.\n\nZdroje: [1](https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak), [2](https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik), [3](https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak), [4](https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko), [5](https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky), [6](https://zp.kemt.fei.tuke.sk/students/2023/maksym_zatirka)\n\n[1]: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak\n[2]: https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik\n[3]: https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak\n[4]: https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko\n[5]: https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky\n[6]: https://zp.kemt.fei.tuke.sk/students/2023/maksym_zatirka", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak", + "https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik", + "https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak", + "https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky", + "https://zp.kemt.fei.tuke.sk/students/2023/maksym_zatirka" + ] + }, + { + "id": "supervisor", + "engine": "rag", + "query": "Kto vedie bakalársku prácu Ronalda Zalacka a kto je pri profile uvedený ako autor dokumentu? Tieto roly odlíš.", + "passed": true, + "status": "ok", + "answer": "**Ronald Zalacko — Bakalárska práca 2026**\n\nNázov / téma zo zdroja: Skórovací online systém pre robotické súťaže\n\nVedúci v hlavičke profilu: Matúš Pleva.\n\nAutor dokumentu nie je automaticky vedúci ani konzultant práce.\n\nAutor dokumentu profilu Ronald Zalacko: Daniel Hladek.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko" + ] + }, + { + "id": "supervisor", + "engine": "graphrag", + "query": "Kto vedie bakalársku prácu Ronalda Zalacka a kto je pri profile uvedený ako autor dokumentu? Tieto roly odlíš.", + "passed": true, + "status": "ok", + "answer": "**Ronald Zalacko — Bakalárska práca 2026**\n\nNázov / téma zo zdroja: Skórovací online systém pre robotické súťaže\n\nVedúci v hlavičke profilu: Matúš Pleva.\n\nAutor dokumentu nie je automaticky vedúci ani konzultant práce.\n\nAutor dokumentu profilu Ronald Zalacko: Daniel Hladek.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko" + ] + }, + { + "id": "typo", + "engine": "rag", + "query": "Zisti názov práce alebo projektu osoby Sebastián Kiš a evidovaný rok. Pri oprave mena povedz, koho si rozpoznal.", + "passed": true, + "status": "ok", + "answer": "Rozpoznané meno: **Sebastián Kišš** (v otázke „Sebastián Kiš“).\n\nSebastián Kišš má v dostupných podkladoch tieto práce a projekty:\n\n1. **Bakalársky projekt 2027** — **Dáta pre vyhodnotenie multimodálnych jazykových modelov**.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2024/sebastian_kiss", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/sebastian_kiss" + ] + }, + { + "id": "typo", + "engine": "graphrag", + "query": "Zisti názov práce alebo projektu osoby Sebastián Kiš a evidovaný rok. Pri oprave mena povedz, koho si rozpoznal.", + "passed": true, + "status": "ok", + "answer": "Rozpoznané meno: **Sebastián Kišš** (v otázke „Sebastián Kiš“).\n\nSebastián Kišš má v dostupných podkladoch tieto práce a projekty:\n\n1. **Bakalársky projekt 2027** — **Dáta pre vyhodnotenie multimodálnych jazykových modelov**.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2024/sebastian_kiss", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/sebastian_kiss" + ] + }, + { + "id": "both_projects", + "engine": "rag", + "query": "Aké metódy používal Ján Holp pri oboch diplomových projektoch v roku 2020? Uveď obe práce samostatne.", + "passed": true, + "status": "ok", + "answer": "**Ján Holp — Diplomový projekt 2 2020**\n\nZaznamenané výsledky (dátum záznamu: 2020-11-13):\n- Vyriešil technické problémy s Cassandra a umožnil indexovanie z Cassandra do Elasticsearch. [1]\n\n**Ján Holp — Diplomový projekt 2020**\n\nZaznamenané výsledky (dátum záznamu: 2020-06-23):\n- Zabezpečil funkčnosť databázy Cassandra. [1]\n- Zabezpečil funkčnosť indexovania zo súboru. [1]\n\nZdroje: [1](https://zp.kemt.fei.tuke.sk/students/2016/jan_holp)\n\n[1]: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + }, + { + "id": "record_count", + "engine": "rag", + "query": "Koľko rôznych prác a projektov má evidovaných Rostyslav Rodzhuk? Spočítaj akademické záznamy.", + "passed": true, + "status": "ok", + "answer": "Počet evidovaných prác a projektov: **2**.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/rostyslav_rodzhuk", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/rostyslav_rodzhuk" + ] + }, + { + "id": "both_projects", + "engine": "graphrag", + "query": "Aké metódy používal Ján Holp pri oboch diplomových projektoch v roku 2020? Uveď obe práce samostatne.", + "passed": true, + "status": "ok", + "answer": "**Ján Holp — Diplomový projekt 2 2020**\n\nMetódy a technológie:\n\n- Vyriešené technické problémy s cassandrou. Vieme indexovať z Cassandra do ES. (zaznamenaný výsledok)\n\n**Ján Holp — Diplomový projekt 2020**\n\nMetódy a technológie:\n\n- funguje databáza Cassandra (zaznamenaný výsledok)\n\n- funguje indexovanie zo súboru. (zaznamenaný výsledok)\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + }, + { + "id": "time_roles", + "engine": "rag", + "query": "Pri Yehorovi Piliavinovi odlíš rok začiatku štúdia, rok bakalárskeho projektu a najnovší úplný dátum stretnutia. Znamená rok projektu dátum vykonania činnosti?", + "passed": true, + "status": "ok", + "answer": "**Yehor Piliavin — Bakalársky projekt 2027**\nPosledný záznam s úplným dátumom: 2026-10-15.\n\nExplicitný rok začiatku štúdia — Yehor Piliavin: 2024.\nRok práce, dátum záznamu a dátum vykonania činnosti sú odlišné údaje.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin" + ] + }, + { + "id": "record_count", + "engine": "graphrag", + "query": "Koľko rôznych prác a projektov má evidovaných Rostyslav Rodzhuk? Spočítaj akademické záznamy.", + "passed": true, + "status": "ok", + "answer": "Rostyslav Rodzhuk má prácu (rok práce: **2026**).\nRostyslav Rodzhuk má prácu (rok práce: **2025**).\nPočet evidovaných prác a projektov: 2.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2023/rostyslav_rodzhuk", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/rostyslav_rodzhuk" + ] + }, + { + "id": "time_roles", + "engine": "graphrag", + "query": "Pri Yehorovi Piliavinovi odlíš rok začiatku štúdia, rok bakalárskeho projektu a najnovší úplný dátum stretnutia. Znamená rok projektu dátum vykonania činnosti?", + "passed": true, + "status": "ok", + "answer": "**Yehor Piliavin — Bakalársky projekt 2027**\nPosledný záznam s úplným dátumom: 2026-10-15.\n\nExplicitný rok začiatku štúdia — Yehor Piliavin: 2024.\nRok práce, dátum záznamu a dátum vykonania činnosti sú odlišné údaje.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/yehor_piliavin" + ] + }, + { + "id": "annotation", + "engine": "graphrag", + "query": "Kto sa venoval anotácii pomenovaných entít?", + "passed": true, + "status": "ok", + "answer": "**Than Trung Thanh**\n\n- Bakalársky projekt 2020; podklad: Pomocou nástroja Prodigy anotujte dostatočné množstvo textu pre výskyt pomenovaných entít.\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2017/than_trung_thanh\n\nPrechod grafom dosiahol limit výberu. Zúž otázku na prácu, tému alebo obdobie.", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2017/than_trung_thanh" + ] + }, + { + "id": "annotation", + "engine": "rag", + "query": "Kto sa venoval anotácii pomenovaných entít?", + "passed": true, + "status": "ok", + "answer": "**Jakub Maruniak**\n\n- Diplomový projekt 2; zaznamenaný výsledok; záznam 2020-09-25: Oboznámenie sa procesom anotácie NER\n\n**Than Trung Thanh**\n\n- Bakalársky projekt 2020; podklad: Pomocou nástroja Prodigy anotujte dostatočné množstvo textu pre výskyt pomenovaných entít.\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak, https://zp.kemt.fei.tuke.sk/students/2017/than_trung_thanh", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak", + "https://zp.kemt.fei.tuke.sk/students/2017/than_trung_thanh" + ] + }, + { + "id": "recognition", + "engine": "rag", + "query": "Kto sa venoval rozpoznávaniu pomenovaných entít?", + "passed": true, + "status": "ok", + "answer": "**Adrián Ondra**\n\n- Bakalársky projekt 2027; zaznamenaný výsledok; záznam 2026-11-05: Vyskúšaný model SlovakBERT-NER (Agarský) na rozpoznávanie pomenovaných entít.\n\n**Martin Wencel**\n\n- Bakalársky projekt 2020; zaznamenaný výsledok: Aplikácia vie robiť pomocou spacy tokenizácia, morfologická analýza, parsing a named entity recognition pre slovenčinu a angličtinu.\n\n**Samuel Horáni**\n\n- Bakalársky projekt 2021; úloha označená ako splnená; záznam 2021-12-17: [x] Pridajte schopnosť rozlišovať pomenované entity typu LOC. Spacy model by to mal vedieť.\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra, https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel, https://zp.kemt.fei.tuke.sk/students/2019/samuel_horani", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra", + "https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel", + "https://zp.kemt.fei.tuke.sk/students/2019/samuel_horani" + ] + }, + { + "id": "recognition", + "engine": "graphrag", + "query": "Kto sa venoval rozpoznávaniu pomenovaných entít?", + "passed": true, + "status": "ok", + "answer": "**Adrián Ondra**\n\n- Bakalársky projekt 2027; zaznamenaný výsledok; záznam 2026-11-05: Vyskúšaný model SlovakBERT-NER (Agarský) na rozpoznávanie pomenovaných entít.\n\n**Martin Wencel**\n\n- Bakalársky projekt 2020; zaznamenaný výsledok: Aplikácia vie robiť pomocou spacy tokenizácia, morfologická analýza, parsing a named entity recognition pre slovenčinu a angličtinu.\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra, https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel\n\nPrechod grafom dosiahol limit výberu. Zúž otázku na prácu, tému alebo obdobie.", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2024/adrian_ondra", + "https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel" + ] + }, + { + "id": "chatbot", + "engine": "graphrag", + "query": "Kto vytvoril chatbota?", + "passed": true, + "status": "ok", + "answer": "**Samuel Džurina**\n\n- Bakalárska práca 2026; zaznamenaný výsledok; záznam 2025-11-27: Vytvorený modelový OpenAI chatbot. Agent používa openai api, gpt3.5 turbo a vie odpovedať na otázky. Asi má problém s vyhľadávaním na internete.\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/samuel_dzurina\n\nPrechod grafom dosiahol limit výberu. Zúž otázku na prácu, tému alebo obdobie.", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/samuel_dzurina" + ] + }, + { + "id": "chatbot", + "engine": "rag", + "query": "Kto vytvoril chatbota?", + "passed": true, + "status": "ok", + "answer": "**Samuel Džurina**\n\n- Bakalárska práca 2026; zaznamenaný výsledok; záznam 2025-11-27: Vytvorený modelový OpenAI chatbot. Agent používa openai api, gpt3.5 turbo a vie odpovedať na otázky. Asi má problém s vyhľadávaním na internete.\n\n- Bakalárska práca 2026; priebežný stav; záznam 2025-12-17: Je urobená prvá verzia chatbota - vie hľadať PDF a pristupovať na internet cez DUCKDUCK. Rozhranie je cez streamlit.\n\nZobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. Výber môžeš zúžiť témou alebo obdobím.\n\nZdroje: https://zp.kemt.fei.tuke.sk/students/2023/samuel_dzurina", + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/samuel_dzurina" + ] + } + ], + "remaining_limits": [ + "Actual new Claude-chat presentation and follow-up need the user to check; protocol success alone does not verify that UI.", + "No new claim of 94% or general accuracy. The earlier TEST is consumed and its first-run conservative score remains unchanged; its model judge had documented reliability problems.", + "Selected global discovery/similarity answers are examples, not complete lists. Graph traversal and evidence budgets are enforced with concrete notices.", + "Conservative synthesis can return literal source wording when generated prose is not fully supported.", + "Only two arbitrary LLM-extracted concept-to-concept relations are activated in this checkpoint; the graph also uses many source-backed structural work/person/evidence/concept paths.", + "72 profile sources were checked automatically against Markdown; no human-reviewed source audit is claimed.", + "Quick Tunnel URLs are temporary; old DNS disappeared and the tunnel was restored. The laptop, Docker and tunnel must stay running." + ], + "frozen_test_files_unchanged": [ + "test.json", + "test_manifest.json", + "test_results.jsonl", + "test_summary.json", + "review.json" + ], + "runtime_file_sha256": { + "tools/answers/answer_coverage.py": "746ab3b89ba65f263b63037be55ee02ee25aaaf685dae6e95f7bd6a5536fc6fa", + "tools/answers/rag_metadata.py": "a16486932e66aed571fd25cb59c5656892613488ea90afe80c41b0c1d15afcf1", + "tools/answers/checked_generation.py": "e36def33fb83e5c973fa632685b4782fa411a7eeb7151ef8c14007f58fa1f375", + "tools/answers/answer_delivery.py": "2520ab327670e35511fb578ef1a447f0c011fb04fa3a341a36696631a2ca499e", + "tools/answers/presentation.py": "954c65288ddc3c8f873fec3ef99176adb70785806043ac1814c7a01a267b4aad", + "tools/answers/content_selection.py": "8e09f4423f2fd3b253f5779f62849b11b9570a112a472ea07392468901e3d80e", + "tools/answers/synthesis.py": "d669136a99d71b9046119debc077f2ab51336ab8ffe1bcd5e5853cfa8ba8e748", + "tools/answers/graph_context.py": "5b14150b7f6903cd92c004d56943d949167be8859acf4f97aed14a713e59faff", + "tools/answers/model_client.py": "f595eb1b594364faff21ce2a6842c8c7e40a5ca4e6a2c3085cdc4a4bae28ae04", + "tools/answers/answer_errors.py": "d1541b5c31b5d64e0bfa729f5e9a29adddc4b55bdd24508490f39cfc0a44b240", + "tools/answers/context_policy.py": "526f5de0b9112e23abd76e403799f156d0150c83f03577251af23d1685991327", + "tools/answers/verified_rag.py": "b2e9777b269f2b19c16ad9e1aa86644f09a9ad96489358a1294e51d21025611e", + "tools/answers/answer_format.py": "533cfe3a30c835d22441d877df63fe08e366d01ed6d15474fe6fcff26bde259b", + "tools/answers/evidence_selection.py": "1007a85b8838e1eb3c34ee610a4f81dfff57410a90213e02b8ccde90a0fedbd2", + "tools/answers/__init__.py": "8b8ab7f996121e5e5b1b594cf08f4c3b4f16f1b434b74276c803a3a83681d2cb", + "tools/answers/rag_preparation.py": "98468b111c9f8f6d423f3c6a07300c3ad7a7e9fef5bae1d1385b91a04771e069", + "tools/answers/rag_evidence.py": "351b99eb5757991696dbe10dcb794cf02aea7c2232c2b7eabab4e3bf1a7ad687", + "tools/answers/reference_links.py": "eb59206bcb3b79912daafcd591e7adb504ec592e61ea32e4e133cb6a65fe0b57", + "tools/answers/render_answer.py": "953cae2cdbdbf05a1b0c1f2746c4dc5f0d85c9bcfd02c105df38a3ff1d2451af", + "tools/answers/question_plan.py": "d95b6f071845f4c2c8846bccc0f131cc80cf58cb0d28804f36e5f29ccf4b7387", + "tools/answers/question_intent.py": "2d009205f0168b80a0a90b0fc86e52a605ddb367d24ecb727387f6944f16669b", + "tools/knowledge/concepts.py": "de0d005ab298c3d86020b0cb09fc695852303d8a5f411d0a3828b58131713b8d", + "tools/knowledge/sources.py": "709c2201288a920e99578fe4e58e191a742deb5f15170104820680d156280c69", + "tools/knowledge/publication.py": "a5a29af22fd73a57f53da47c687200f0ad392d48ce94a23250f00c7a53731148", + "tools/knowledge/academic.py": "0ae9a611792b042c64c6ba9c3019bb95d8b51e4320b1df20844c8e9f6f006b60", + "tools/knowledge/__init__.py": "905a19d7cf5d95379800b2ad917ad759fa376f23d2f8a244527b9dc187555019", + "tools/knowledge/evidence.py": "b0eb7d3e0f6a97083829a9940b3a64dade96d4f98435eb5ebfb43adaf10fac18", + "tools/knowledge/graph_answers.py": "fd599b7ff65485ae11e77022cc007b99e5bd3b533383cc64dc5b7aa3dabf81fa", + "tools/knowledge/store.py": "fa22cd357df8c3cde4cd1b6a4f0a9b78d12721f3970ec896584205f8a5d5d7c1", + "tools/knowledge/extraction.py": "c1819afc5072a5999c985b4547809db997ecf3e1112b031534ae34ed295bd106", + "tools/knowledge/local_graph.py": "cefdc9de3d265a37636fdc2836c59e929743bddf17a3408d04668fa31b80dc4d", + "tools/knowledge/graph_projection.py": "dac3058008fb43c833d239b6a7ced27ef5419004cc768b246741eb2e1a69b12a", + "tools/knowledge/text_retrieval.py": "bcb2ee2c4de7c4c074e1677137a2e11995b36ffa934fc03de96b59710ad89307", + "tools/knowledge/query.py": "4b3c3333a9aae229de9b3ce83f61f5aa278cf82bebe7a1b0e8518f1ccf8e76ab", + "tools/knowledge/build.py": "4666b1cd2d42533b7289f5e2ecaac11c5cc16e09f3ea4b9a2714d17cb8c2f9f6", + "tools/knowledge/planned_answers.py": "e5820cb6c8d6e15315ae94e263e2886b0309b6c7eee8e09338750adbede2aa8c", + "tools/graphrag/graph_search.py": "eff65f570d3a52a9cb4132db8f3c4b59dc08840001ae1bfbab6c7ab5eda09bb0", + "tools/retrieval/person_names.py": "6e3eb293fee2d136f3b6e082a166dbd02cde9b36ab47777eb01a11404d63c1ac", + "tools/rag/llm_context.py": "8f195ff16639de4379c41108f7a3a456d90d985307ebde3691c1406b3f76dfa5" + } +} diff --git a/test/test_answer_repair_regressions.py b/test/test_answer_repair_regressions.py new file mode 100644 index 0000000..bea86dd --- /dev/null +++ b/test/test_answer_repair_regressions.py @@ -0,0 +1,240 @@ +"""Previously failed user formulations are regressions, never a fresh TEST.""" +import base64 +import zlib +from types import SimpleNamespace +import numpy as np +import pytest + +from tools.answers.question_plan import build_question_plan,QuestionPlan,PersonRequest +from tools.retrieval.person_names import resolve_person_names +from tools.knowledge.text_retrieval import retrieve_units + +CATALOG=[{'kind':'Person','name':name,'source_document':f'pages/students/2024/{i}/README.md'} + for i,name in enumerate(['Adrián Ondra','Sebastián Kišš','Samuel Štucka','Samuel Džurina'])] + + +@pytest.mark.parametrize('suffix',[ + 'Zhrň všetky doložené body a nezamieňaj ich za hotové výsledky.', + 'Vysvetli konkrétne prepojenie a dolož obe osoby.', + 'Pri každej osobe príslušné práce uveď v bodoch.', +]) +def test_presentation_clause_is_not_an_additional_unknown_identity(suffix): + r=resolve_person_names('Na čom pracoval Adrián Ondra? '+suffix,CATALOG) + assert r['status']=='ok' and [m['name'] for m in r['matches']]==['Adrián Ondra'] + + +def test_inflected_partial_first_name_asks_for_real_candidates_without_model(): + p=build_question_plan('Čo sa podarilo Samuelovi pri jeho práci?',CATALOG) + assert p.reason=='incomplete_person_name' and p.resolution['status']=='ambiguous' + assert {c['name'] for c in p.resolution['candidates']}=={'Samuel Štucka','Samuel Džurina'} + + +def test_one_short_surname_typo_still_requires_an_exact_given_name_anchor(): + assert resolve_person_names('Akú prácu má Sebastián Kiš?',CATALOG)['matches'][0]['name']=='Sebastián Kišš' + assert resolve_person_names('Akú prácu má Sebastan Kš?',CATALOG)['status']!='ok' + + +@pytest.mark.parametrize('question',[ + 'Kto už vyskúšal hierarchickú sumarizáciu na problém s dlhým kontextom?', + 'U koho je doložené vytvorenie crawlera pomocou Scrapy?', + 'Ktorý študent sa venoval agentovi pre špeciálneho pedagóga? Bez domýšľania výsledkov.', +]) +def test_global_discovery_does_not_require_a_known_student(question): + p=build_question_plan(question,CATALOG) + assert not p.clarification and p.spec.intent=='discovery' and not p.spec.people + + +def test_grounding_directive_does_not_add_a_results_facet_to_goals(): + p=build_question_plan('Aké sú ciele projektu osoby Adrián Ondra? Nezamieňaj ich za hotové výsledky.',CATALOG) + assert p.spec.facets==['assignment'] and not p.clarification + + +def index_rows(): + rows=[{'id':str(i),'person':'Eva Nová','source_path':'eva','line':i+1, + 'text':'Všeobecná poznámka.' if i else 'Použitá technológia Cassandra.', + 'concepts':[],'work_label':'Diplomová práca 2026','work_type':'diploma_thesis','work_year':2026} + for i in range(100)] + matrix=np.array([[0.,1.]]+[[1.,0.]]*99,dtype='scores['0'] + assert trace['lexical_candidates']==1 and trace['vector_candidates']==80 + + +def test_named_scope_expansion_preserves_every_goal_and_excludes_other_people(monkeypatch): + index=index_rows();index.pop('vectors') + index['units'].append({**index['units'][0],'id':'other','person':'Dana Iná','source_path':'dana'}) + plan=QuestionPlan(question='Všetky ciele Evy',intent='facets',people=[PersonRequest(name='Eva Nová',source_document='eva')]) + rows,_,trace=retrieve_units(plan.question,index,plan) + assert len(rows)==100 and all(r['person']=='Eva Nová' for r in rows) + assert trace['named_scope_expansion'] + + +def metadata_payload(presentation='list',fields=None): + from tools.rag.rag_utils import build_context_text + source={'source_id':'S1','title':'Adrián Ondra','document_path':CATALOG[0]['source_document'], + 'source_url':'https://example.test/adrian','tags':['rag','nlp'], + 'text':'# Bakalársky projekt 2027\nAutomatická tvorba myšlienkových máp\nMožné ciele semestrálneho projektu:\n- Vytvoriť vizualizáciu.\n# Diplomový projekt 2029\nStav:\n- Vyskúšaný model.'} + plan=QuestionPlan(question='Zostav prehľad prác Adriána Ondru.',people=[PersonRequest(name='Adrián Ondra',source_document=source['document_path'])], + fields=fields if fields is not None else ['work_type','title','year'],presentation=presentation) + return {'sources':[source],'question_plan':plan.model_dump(),'context':build_context_text([source]), + 'named_scope_complete':True,'index_version':{'version':'fixture'},'truncation_reasons':[]} + + +def test_exact_metadata_keeps_distinct_untitled_records_without_a_model(): + from tools.answers.verified_rag import generate_verified_rag + result=generate_verified_rag('Zostav prehľad prác Adriána Ondru.',metadata_payload(),lambda _:pytest.fail('No model for exact complete records')) + assert result['status']=='ok' and len(result['claims'])==2 + assert len({r['record_id'] for r in result['claims']})==2 + assert 'myšlienkových máp' in result['answer'] and 'názov v zdroji nie je uvedený' in result['answer'] + + +def test_exact_count_counts_records_not_source_windows_or_people(): + from tools.answers.verified_rag import generate_verified_rag + payload=metadata_payload('count',[]) + result=generate_verified_rag('Zostav prehľad prác Adriána Ondru.',payload,lambda _:pytest.fail('No model for exact count')) + assert result['count']==2 and 'prác a projektov' in result['answer'] + + +@pytest.mark.parametrize('presentation',['list','count']) +def test_exact_metadata_is_accepted_by_the_actual_claude_client(monkeypatch,presentation): + from packages.claude import client + from tools.answers.verified_rag import generate_verified_rag + question='Zostav prehľad prác Adriána Ondru.' + result=generate_verified_rag(question,metadata_payload(presentation),lambda _:pytest.fail('No model')) + monkeypatch.setattr(client,'request_json',lambda *a,**kw:{**result,'query':question,'engine':'rag'}) + delivered=client.query_backend(question,'rag',config=client.Config('http://localhost:8000','test')) + assert delivered['status']=='ok' and delivered['accepted_for_delivery'] + assert delivered['answer']==result['answer'] + + +def test_complete_scope_flag_cannot_override_context_clipping(): + from tools.answers.rag_metadata import exact_metadata_answer + from tools.answers.rag_evidence import rag_evidence_context + payload=metadata_payload();payload['truncation_reasons']=['llm_context_budget'] + assert exact_metadata_answer(payload,rag_evidence_context('Zostav prehľad prác Adriána Ondru.',payload)) is None + + +def test_document_authorship_is_not_interpreted_as_an_advisor_filter(): + p=build_question_plan('Kto vedie bakalársku prácu Adriána Ondru a kto je autor dokumentu?',CATALOG) + assert p.spec.intent=='advisors' and not p.clarification + assert 'document_author' in p.spec.fields and not p.spec.conditions + + +def test_profile_metadata_has_its_own_time_role_and_snapshot(monkeypatch,tmp_path): + import sqlite3 + from contextlib import nullcontext + import tools.sync.index_state as state + from tools.knowledge.academic import with_profile_metadata + db=tmp_path/'db.sqlite' + with sqlite3.connect(db) as conn: + conn.execute('CREATE TABLE documents(path,author,published)') + conn.execute('CREATE TABLE chunks(chunk_id,document_path,text,published)') + path=CATALOG[0]['source_document'] + conn.execute('INSERT INTO documents VALUES (?,?,1)',(path,'Daniel Hladek')) + conn.execute('INSERT INTO chunks VALUES (?,?,?,1)',('c1',path,'rok začiatku štúdia: 2024')) + monkeypatch.setattr(state,'index_snapshot',lambda *a,**k:nullcontext()) + monkeypatch.setattr(state,'read_version',lambda *a:{'version':'v'}) + spec=QuestionPlan(question='Odlíš roly a roky.',people=[PersonRequest(name='Adrián Ondra',source_document=path)],fields=['document_author','start_year','year']) + plan=SimpleNamespace(spec=spec) + original={'status':'ok','accepted_for_delivery':True,'index_version':{'version':'v'}, + 'answer':'Vedúci: Matúš Pleva. Rok práce: 2027.\n\nZdroje: https://example.test/adrian','source_urls':['https://example.test/adrian']} + result=with_profile_metadata(original,plan,db) + assert 'Matúš Pleva' in result['answer'] and 'Autor dokumentu profilu Adrián Ondra: Daniel Hladek.' in result['answer'] + assert 'začiatku štúdia — Adrián Ondra: 2024' in result['answer'] + assert any(r.get('quote')=='rok začiatku štúdia: 2024' for r in result['metadata_evidence']) + with pytest.raises(RuntimeError,match='snapshot'): + with_profile_metadata({**original,'index_version':{'version':'other'}},plan,db) + + +def test_graph_work_count_is_not_the_number_of_authors(): + from tools.graphrag import graph_search as gs + from tools.graphrag.tests.test_graph_search import FakeSession + from tools.graphrag.tests.test_graph_search_entity_v2 import catalog,thesis + q='Koľko prác má Alena Nová?' + bound=build_question_plan(q,catalog()) + rows=[{**thesis(),'thesis_id':'a','heading':'Bakalárska práca 2024'}, + {**thesis(),'thesis_id':'b','heading':'Bakalárska práca 2025','year':2025}] + session=FakeSession({gs.CATALOG_QUERY:catalog(),gs.THESIS_FACTS_QUERY:rows}) + result=gs.search_graph_with_session(session,q,question_plan=bound.spec) + assert result['aggregation']['count']==2 and result['aggregation']['people']==['Alena Nová'] + assert result['aggregation']['count_domain']=='academic_records' + + +def test_declarative_stack_is_a_method_description_without_promoting_its_phase(): + from tools.answers.content_selection import facets_for + q='Aké technológie má študent na frontende, backende a čo je doložené o funkčnosti?' + assert 'methods' in facets_for('Frontend React, backend Flask, DB Postgres.','other',q,'unknown') + assert 'methods' not in facets_for('Frontend bude React, backend bude Flask.','task',q,'planned') + assert 'results' in facets_for('Aplikácia funguje.','progress',q,'result') + + +@pytest.mark.parametrize('extra_work,expected', [('',True),('\n# Diplomová práca 2029\nStav:\n- Funguje systém.',False)]) +def test_profile_header_role_is_qualified_and_never_broadcast_to_multiple_works(extra_work,expected): + from tools.knowledge.sources import profile_units + from tools.knowledge.academic import answer_academic + path=CATALOG[0]['source_document'] + units=profile_units({'title':'Adrián Ondra','path':path,'content':'Vedúci: Matúš Pleva\n# Bakalárska práca 2027\nStav:\n- Funguje aplikácia.'+extra_work},include_records=True) + bound=build_question_plan('Kto vedie bakalársku prácu Adriána Ondru?',CATALOG) + index={'index_version':{'version':'v'},'version':'content','academic_schema':1,'units':units} + result=answer_academic(bound.spec.question,index,engine='rag',mode='advisors',plan=bound) + assert ('Matúš Pleva' in result['answer']) is expected + if expected: + assert 'v hlavičke profilu' in result['answer'] + assert any(r['kind']=='supervisor' and r.get('work_id') is None for r in result['evidence']) + + +@pytest.mark.parametrize('consultant_profile',[False,True]) +def test_mentioning_a_consultant_does_not_turn_the_supervisor_into_a_consultant(consultant_profile): + consultant={'kind':'DocumentAuthor','name':'Kristián Sopkovič'} + if consultant_profile:consultant.update(kind='Person',source_document='pages/students/2020/kristian/README.md') + catalog=[{'kind':'DocumentAuthor','name':'Daniel Hladek'}, consultant] + plan=build_question_plan('Ktoré práce vedie Daniel Hladek a konzultuje Kristián Sopkovič?',catalog) + assert [(p.name,p.role) for p in plan.spec.people]==[('Daniel Hladek','supervisor'),('Kristián Sopkovič','consultant')] + + +def test_created_chatbot_is_not_satisfied_by_a_future_assignment(): + from tools.knowledge.query import activity_supported + q='Kto vytvoril chatbota?' + assert not activity_supported(q,{'text':'Vytvorte chatbota.','phase':'assignment'}) + assert activity_supported(q,{'text':'Vytvorený modelový chatbot.','phase':'result'}) + + +def test_ner_is_not_proved_by_generic_form_slot_recognition_or_model_concepts(): + from tools.knowledge.query import activity_supported + q='Kto sa venoval rozpoznávaniu pomenovaných entít?' + row={'text':'Nedarí sa rozpoznať typ priznania za psa vo Form.','phase':'progress','concepts':['NER']} + assert not activity_supported(q,row) + assert activity_supported(q,{'text':'Vyskúšaný model SlovakBERT-NER na rozpoznávanie pomenovaných entít.','phase':'result'}) + + +def test_possible_goals_keep_the_literal_qualification_for_prose_rendering(): + from tools.knowledge.sources import profile_units + from tools.knowledge.planned_answers import answer_facets + rows=profile_units({'title':'Adrián Ondra','path':CATALOG[0]['source_document'], + 'content':'# Bakalársky projekt 2027\nMožné ciele semestrálneho projektu:\n- Vytvoriť vizualizáciu.'},include_records=True) + bound=build_question_plan('Aké sú ciele projektu Adriána Ondru?',CATALOG) + index={'version':'c','index_version':{'version':'v'},'units':rows} + result=answer_facets(bound,index,rows,engine='rag',limit=5) + assert 'Vytvoriť vizualizáciu' in result['answer'] + assert any('možné/navrhované ciele' in note for note in result['presentation_notes']) + + +def test_comparing_time_roles_does_not_dump_the_meeting_agenda(): + from tools.knowledge.sources import profile_units + from tools.knowledge.academic import answer_academic + path=CATALOG[0]['source_document'] + rows=profile_units({'title':'Adrián Ondra','path':path,'content':'# Bakalársky projekt 2027\nStretnutie 5.11.2026\nStav:\n- Implementovaný nevyžiadaný detail.'},include_records=True) + p=build_question_plan('Odlíš rok začiatku štúdia Adriána Ondru, rok práce a najnovší dátum stretnutia.',CATALOG) + result=answer_academic(p.spec.question,{'index_version':{'version':'v'},'version':'c','academic_schema':1,'units':rows},engine='rag',mode='meeting',plan=p) + assert '2026-11-05' in result['answer'] and '2027' in result['answer'] + assert 'nevyžiadaný detail' not in result['answer'] diff --git a/tools/answers/checked_generation.py b/tools/answers/checked_generation.py index 0248c72..f468f56 100644 --- a/tools/answers/checked_generation.py +++ b/tools/answers/checked_generation.py @@ -19,7 +19,7 @@ def generate_checked_answer(question: str, context: dict, prompt: str, complete) aggregation = context.get("aggregation") if aggregation is not None: sent["aggregation"] = { - k: v for k, v in aggregation.items() if k in {"count", "people", "complete"} + k: v for k, v in aggregation.items() if k in {"count", "count_domain", "people", "complete"} } policy = "\nDo claims nevkladaj source_urls ani evidence_documents; source_urls patrí iba na vrchnú úroveň odpovede." if ( diff --git a/tools/answers/content_selection.py b/tools/answers/content_selection.py index 6684764..c68d4be 100644 --- a/tools/answers/content_selection.py +++ b/tools/answers/content_selection.py @@ -137,13 +137,14 @@ def facets_for(text: str, role: str, question: str, phase: str | None = None) -> ) and role not in {"assignment", "task"}: facets.append("problems") if re.search( - r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*|indexov\w*|databaz\w*|webov\w* rozhrani\w*)\b", + r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*|indexov\w*|databaz\w*|webov\w* rozhrani\w*|frontend\w*|backend\w*|vyskus\w*|sumarizac\w*)\b", n, ): actual_use = actual_methods_requested(question) actual_statement = not re.search(NOT_COMPLETED, n) and ( - re.search(COMPLETED, n) + re.search(COMPLETED, n) or re.search(r'\bvyskus(?:any|ana|ane|al|ala|ali)\b',n) or re.search(r"\b(?:pouziva\w*|pouzil\w*|vyuziva\w*|vyuzil\w*|vieme indexov\w*)\b", n) + or (not re.search(PLANNED,n) and re.search(r'\b(?:frontend|backend)\s+\w+',n)) ) if not actual_use or (role not in {"assignment", "task"} and actual_statement): facets.append("methods") diff --git a/tools/answers/question_intent.py b/tools/answers/question_intent.py index 9daa747..a87c615 100644 --- a/tools/answers/question_intent.py +++ b/tools/answers/question_intent.py @@ -7,10 +7,10 @@ FACET_PATTERNS = { "assignment": r"\b(?:zadani\w*|ciel\w*)\b", "plans": r"\b(?:planovan\w*|dalsi\w*\s+uloh\w*|uloh\w*)\b", "problems": r"\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*)\b", - "methods": r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*)\b", + "methods": r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*|pristup\w*|frontend\w*|backend\w*)\b", "experiments": r"\b(?:experiment\w*|merani\w*|testovani\w*|vyhodnocov\w*)\b", "progress": r"\b(?:priebezn\w*\s+)?stav\w*\b", - "results": r"\b(?:vysledk\w*|dosiahn\w*)\b", + "results": r"\b(?:vysledk\w*|dosiahn\w*|funkcnost\w*)\b", } FACET_LABELS = { "assignment": "Zadanie a ciele", @@ -25,6 +25,7 @@ FACET_LABELS = { def requested_facets(question: str) -> list[str]: text = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', " ", question)) + text = re.sub(r'\bnezamienaj\s+(?:ich|to|ciele|zadanie|plany|ulohy)\s+za\s+(?:hotove\s+)?vysled\w*|\bbez domyslania vysled\w*',' ',text) matches = [ (m.start(), key) for key, pattern in FACET_PATTERNS.items() @@ -54,7 +55,8 @@ def facet_mode(question: str) -> bool: def actual_methods_requested(question: str) -> bool: return bool( re.search( - r"\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*)\b|" + r"\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*|vyskus(?:al|ala|ali)\w*)\b|" + r"\bake technolog\w* ma\b|" r"\b(?:pouzit\w*|aplikovan\w*|vyuzit\w*)\s+(?:metod\w*|technolog\w*|nastroj\w*)\b", normalized(question), ) diff --git a/tools/answers/question_plan.py b/tools/answers/question_plan.py index 76f38e4..50461ec 100644 --- a/tools/answers/question_plan.py +++ b/tools/answers/question_plan.py @@ -126,17 +126,24 @@ def direct_spec(question, resolution): selector=parse_work_selector(q) mode=intent(q) facets=requested_facets(q) + global_discovery=bool(re.search(r'\b(?:kto|ktori studenti|ktory student|u koho)\b',n) + and re.search(r'\b(?:venov\w*|zaober\w*|pracov\w*|vyskus\w*|vytvor\w*|pouz\w*|ries\w*|implement\w*)\b',n)) + if mode not in {'similar','supervisor','advisors','completion','meeting','period'} and global_discovery: + mode='discovery' # Facet questions without an academic lifecycle intent share one execution # path, including GraphRAG's content evidence. Explicit metadata stays metadata. if (not mode or mode=='progress' and facets) and content_question(q):mode='facets' years=requested_years(q) people=[] - staff_query=bool(re.search(r'\b(?:pod veden\w*|vedie|viedol|veduc\w*|skolitel\w*|konzult(?:ant|oval)\w*)\b',n)) + staff_query=bool(re.search(r'\b(?:pod veden\w*|vedie|viedol|veduc\w*|skolitel\w*|konzult\w*)\b',n)) for m in resolution.get('matches',[]): role='document_author' if not m.get('source_document') else 'student' - explicit_staff=bool(re.search(r'(?:pod vedenim|vedie|viedol|veduca|veduci|konzultant)\s+'+re.escape(normalized(m['name'])),n)) + explicit_staff=bool(re.search(r'(?:pod vedenim|vedie|viedol|veduca|veduci|konzult\w*)\s+'+re.escape(normalized(m['name'])),n)) if staff_query and (not m.get('source_document') or explicit_staff): - role='consultant' if re.search(r'\bkonzult(?:ant|oval)\w*',n) else 'supervisor' + name=re.escape(normalized(m['name'])) + consultant=bool(re.search(r'\bkonzult\w*(?:\s+s)?\s+'+name+r'\b',n) + or re.search(r'\b'+name+r'\s+konzult\w*\b',n)) + role='consultant' if consultant else 'supervisor' people.append(PersonRequest(name=m['name'],role=role,source_document=m.get('source_document'))) from tools.retrieval.work_selector import QUOTED,HEADING,label_key types_text=QUOTED.sub(lambda m:m[0] if HEADING.fullmatch(label_key(m[1])) else ' ',selector.positive_text) @@ -178,7 +185,8 @@ def direct_spec(question, resolution): if any(len(m[1])!=4 for m in re.finditer(r'\brok(?:u|och)?\s+(\d+)\b',n)):unsupported.append('invalid_year') full_dates=re.findall(r'\d{1,2}\.\d{1,2}\.[12]\d{3}',q) if len(full_dates)<2 and re.search(r'\b(?:kvartal\w*|minuly rok|minulom roku|v marci|v januari|v februari|v aprili|v maji|v juni|v juli|v auguste|v septembri|v oktobri|v novembri|v decembri)\b',n):unsupported.append('period_needs_dates') - if re.search(r'\bbez\b',n) and not (selector.unnumbered or selector.excluded_numbers or selector.excluded_labels):unsupported.append('exclusion') + exclusion_text=re.sub(r'\bbez\s+(?:domyslania|dohadov|spekulacii|nepodlozenych tvrdeni)(?:\s+vysledkov)?\b',' ',n) + if re.search(r'\bbez\b',exclusion_text) and not (selector.unnumbered or selector.excluded_numbers or selector.excluded_labels):unsupported.append('exclusion') if unresolved_followup(q):unsupported.append('missing_followup_context') if re.search(r'\bsemest\w*\b',n) and len(re.findall(r'\d{1,2}\.\d{1,2}\.[12]\d{3}',q))<2:unsupported.append('enrollment_not_established' if re.search(r'\b(?:aktual\w*|teraz|v akom|v ktorom)\b',n) else 'period_needs_dates') code=unsupported_constraint(q) @@ -203,6 +211,7 @@ def direct_spec(question, resolution): conditions=[] for pattern,kind in [(r'\btag\w*\s+[^,;?]+','tag'),(r'\bklucov\w*\s+slov\w*\s+[^,;?]+','tag'), (r'\bkategori\w*\s+[^,;?]+','category'),(r'\bautor\w* dokument\w*[^,;?]*','document_author')]: + if kind=='document_author' and not any(p.role=='document_author' for p in people):continue conditions.extend(Condition(kind=kind,quote=m[0]) for m in re.finditer(pattern,n)) if mode and mode!='metadata' and conditions:unsupported.append('unsupported_content_join') conditions.extend(Condition(kind='current_status',quote=m[0]) for m in re.finditer(r'\b(?:aktual\w*|teraz|dnes)\b',n)) @@ -222,7 +231,7 @@ def direct_spec(question, resolution): work.years = [year for year in work.years if year not in study_values | verification_values] if year_verification: conditions.append(Condition(kind='year_verification', quote=q)) - fields=[field for field,pattern in [('title',r'\bnaz(?:ov|v)\w*'),('year',r'\b(?:ktoreho roku|ktorych rokoch|rok\w*)'),('work_type',r'\btyp\w*'),('keyword',r'\b(?:tag\w*|klucov\w*)'),('category',r'\bkategor\w*'),('document_author',r'\bautor\w* dokument\w*'),('supervisor',r'\b(?:vedie|veduc\w*|veden\w*)'),('consultant',r'\bkonzult(?:ant|oval)\w*'),('status',r'\b(?:ukoncil\w*|obhaj\w*|skoncil\w*)'),('date',r'\b(?:kedy|datum\w*)')] if re.search(pattern,n)] + fields=[field for field,pattern in [('title',r'\bnaz(?:ov|v)\w*'),('year',r'\b(?:ktoreho roku|ktorych rokoch|rok\w*)'),('work_type',r'\btyp\w*'),('keyword',r'\b(?:tag\w*|klucov\w*)'),('category',r'\bkategor\w*'),('document_author',r'\bautor\w* dokument\w*'),('supervisor',r'\b(?:vedie|veduc\w*|veden\w*)'),('consultant',r'\bkonzult\w*'),('status',r'\b(?:ukoncil\w*|obhaj\w*|skoncil\w*)'),('date',r'\b(?:kedy|datum\w*)')] if re.search(pattern,n)] if study_start: fields.append('start_year') return QuestionPlan(fields=fields,question=q,intent=mode or 'metadata',people=people,works=works,years=years,period=period, diff --git a/tools/answers/rag_evidence.py b/tools/answers/rag_evidence.py index 939634a..bb61dd6 100644 --- a/tools/answers/rag_evidence.py +++ b/tools/answers/rag_evidence.py @@ -603,6 +603,7 @@ def rag_evidence_context(question: str, payload: dict) -> dict: if project_number(question) is None or (scope is not None and project_number(label or "") == project_number(question)) ] + labels={(scope,text):label for scope,label,text in academic_windows_metadata(source['body'])} unscoped = [] for section, text in windows: titles = [] @@ -652,7 +653,7 @@ def rag_evidence_context(question: str, payload: dict) -> dict: if ( lead and re.match( - r"^(?:Cie[lľe]+\s*:|Zadanie\s*:|Myšlienky\s*:|1[.)]\s)", next_line, re.I + r"^(?:(?:Možné\s+)?Cie[lľe]+\b|Zadanie\s*:|Myšlienky\s*:|1[.)]\s)", next_line, re.I ) and not re.match(r"^(?:[-*+]\s|\d+[.)]\s|#{1,6}\s)", lead) and not normalized(lead).startswith( @@ -676,6 +677,9 @@ def rag_evidence_context(question: str, payload: dict) -> dict: if title and len(normalized(title).split()) >= 3: titles.append(title) record = {**base, "work_type": kind} + if labels.get((section,text)): + from tools.knowledge.sources import identifier + record.update(work_label=labels[(section,text)],record_id=identifier(source['document_path'],labels[(section,text)])) if year is not None: record["year"] = year for title in titles or [None]: @@ -811,7 +815,8 @@ def rag_evidence_context(question: str, payload: dict) -> dict: requirements["project_number"] = project_number(question) requirements["exact_work_label"] = selector.labels[0] if len(selector.labels) == 1 else None if plan: - facts=[row for row in facts if not row.get("work_type") or any((not w.work_type or w.work_type==row.get("work_type")) and (not w.years or row.get("year") in w.years) for w in plan.works if w.person in {None,row["person"]})] + facts=[row for row in facts if not row.get('work_type') or plan.matches_work( + row['person'],row.get('work_label'),row.get('work_type'),row.get('year'))] if requested: facts = [row for row in facts if row["person"] in requested] requirements["available_fields"] = { diff --git a/tools/answers/rag_metadata.py b/tools/answers/rag_metadata.py new file mode 100644 index 0000000..2ee530d --- /dev/null +++ b/tools/answers/rag_metadata.py @@ -0,0 +1,68 @@ +"""Exact metadata answers from delivered text, never from Neo4j or a gold set.""" +import re +from tools.answers.context_policy import normalized,citation_check +from tools.answers.question_plan import QuestionPlan +from tools.answers.render_answer import render_graph_answer + + +def exact_metadata_answer(payload,context): + if not payload.get('question_plan'):return None + spec=QuestionPlan.model_validate(payload['question_plan']) + people=[p.name for p in spec.people if p.role=='student'] + if spec.intent!='metadata' or not people or any(p.role!='student' for p in spec.people):return None + if set(spec.fields)&{'supervisor','consultant','status','date'}:return None + q=normalized(spec.question) + shared=bool(re.search(r'\b(?:spoloc\w*|prienik|obaja|spajaj\w*)\b',q) and len(people)>1) + keyword=shared and bool(set(spec.fields)&{'keyword','category','document_author'}) + if spec.conditions and not keyword:return None + sources={s.get('title'):s for s in context['sources'] if s.get('document_path') in {p.source_document for p in spec.people}} + if not set(people)<=set(sources):return None + if not keyword and (not payload.get('named_scope_complete') or 'llm_context_budget' in payload.get('truncation_reasons',[])): + return None + facts=context['facts'] + record_map={} + for row in facts: + if row.get('record_id') and row['person'] in people and spec.matches_work( + row['person'],row.get('work_label'),row.get('work_type'),row.get('year')): + old=record_map.get(row['record_id'],{}) + if old.get('title') and row.get('title') and normalized(old['title'])!=normalized(row['title']):return None + record_map[row['record_id']]={**row,**{k:v for k,v in old.items() if v is not None}} + records=list(record_map.values()) + claims=[];extra={};status='ok' + if keyword: + fields=set(spec.fields)&{'keyword','category','document_author'} + for field in fields: + sets=[{r[field] for r in facts if r['person']==p and r.get(field) is not None} for p in people] + for value in sorted(set.intersection(*sets)): + extra.setdefault('shared_properties',[]).append({'field':field,'value':value,'people':people}) + claims.extend({'person':p,field:value} for p in people) + if not claims:status='no_answer' + elif shared and 'year' in spec.fields: + years=set.intersection(*[{r['year'] for r in records if r['person']==p and r.get('year') is not None} for p in people]) + records=[r for r in records if r.get('year') in years] + extra['academic_year_comparison']={'people':people,'shared_years':sorted(years)} + if not years:status='no_answer' + if not keyword: + listing=spec.presentation in {'list','count'} or shared or 'work_type' in spec.fields and spec.scope=='across_works' or 'title' in spec.fields and any(not r.get('title') for r in records) + for row in records: + keys=['person','record_id','work_label','work_type','title','year'] if listing else ['person','work_type',*(k for k in ['title','year'] if k in spec.fields)] + claims.append({k:row[k] for k in keys if k in row}) + if not claims:status='no_answer' + urls=[sources[p]['source_url'] for p in people] + out={'status':status,'claims':claims,'source_urls':list(dict.fromkeys(urls))} + rendered_context={**context,**extra,'complete':True,'truncated':False,'requested_people':people} + if spec.presentation=='count': + # Every record has an explicit heading in the fully delivered named + # profile scope. Never count top-k hits, meetings or taxonomy years. + count=len({r['record_id'] for r in records}) + status='ok' + out.update(status='ok',count=count) + out['answer']='Počet evidovaných prác a projektov: **'+str(count)+'**.\n\nZdroje: '+', '.join(out['source_urls']) + else:out=render_graph_answer(out,rendered_context) + if not citation_check(out,set(urls))['citations_ok']:raise ValueError('Exact metadata citations rejected') + return {**out,'accepted_for_delivery':True,'attempts':[],'complete':True,'truncated':False, + 'truncation_reasons':[],'index_version':context['index_version'], + 'completeness_scope':'explicit_named_profiles', + 'answer_coverage':{'status':'supported' if status=='ok' else 'not_found', + 'scope':'count' if spec.presentation=='count' else 'list' if keyword or listing else 'fact', + 'warnings':[],'limitations':[]}} diff --git a/tools/answers/rag_preparation.py b/tools/answers/rag_preparation.py index 4a947d9..a3a496c 100644 --- a/tools/answers/rag_preparation.py +++ b/tools/answers/rag_preparation.py @@ -35,7 +35,9 @@ def prepare_named_content(db_file, question: str, payload: dict) -> dict: from tools.answers.question_plan import QuestionPlan, build_question_plan plan=QuestionPlan.model_validate(payload['question_plan']) if payload.get('question_plan') else None selector = parse_work_selector(question) - if not content_question(question) and not selector.specific: + metadata_scope=bool(plan and plan.intent=='metadata' and any(p.role=='student' for p in plan.people) + and (set(plan.fields)&{'title','year','work_type'} or plan.presentation=='count')) + if not content_question(question) and not selector.specific and not metadata_scope: return payload if re.search( r"\b(?:s tagom|s klucovym slovom|z kategorie|s autorom dokumentu)\b", normalized(question) @@ -128,10 +130,16 @@ def prepare_named_content(db_file, question: str, payload: dict) -> dict: ) if not selector.matches(label): continue + if metadata_scope and (scope,normalized(label)) in records: + continue records[(scope, normalized(label))] = label if (scope, normalized(label), text) not in seen: seen.add((scope, normalized(label), text)) - frames.append("Sekcia: " + label + "\n" + text) + # Exact metadata needs every record's source heading + # and introduction, not all subsequent meeting notes. + body='\n'.join(line for line in text.splitlines() if line.strip()) if metadata_scope else text + if metadata_scope:body='\n'.join(body.splitlines()[:6]) + frames.append("Sekcia: " + label + "\n" + body) if selector.multiple and ( any( label not in {label_key(v) for v in records.values()} @@ -202,4 +210,5 @@ def prepare_named_content(db_file, question: str, payload: dict) -> dict: "named_subjects": [p["title"] for p in named], "work_scopes": scopes, "question_plan": plan.model_dump(), + "named_scope_complete": True, } diff --git a/tools/answers/render_answer.py b/tools/answers/render_answer.py index 16c8672..9bc6fa5 100644 --- a/tools/answers/render_answer.py +++ b/tools/answers/render_answer.py @@ -122,8 +122,9 @@ def render_graph_answer(answer: dict, context: dict) -> dict: lines.append(fact_sentences(extra)) lines.append("") if out.get("count") is not None: - lines.append(f'Počet vyhovujúcich osôb: {out["count"]}.') aggregation = context.get("aggregation") or {} + label='Počet evidovaných prác a projektov' if aggregation.get('count_domain')=='academic_records' else 'Počet vyhovujúcich osôb' + lines.append(label+f': {out["count"]}.') if aggregation.get("complete") is True and out["count"] == aggregation.get("count"): # A full total is backed by the inspected candidate set, not # just the one person left visible after a presentation cap. diff --git a/tools/answers/verified_rag.py b/tools/answers/verified_rag.py index 553b0c4..899b22c 100644 --- a/tools/answers/verified_rag.py +++ b/tools/answers/verified_rag.py @@ -808,6 +808,9 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: "truncation_reasons": [], "index_version": context.get("index_version"), } + from tools.answers.rag_metadata import exact_metadata_answer + exact=exact_metadata_answer(payload,context) + if exact is not None:return exact if ( context["requirements"].get("facets") and context["requirements"]["response_mode"] == "quotes" diff --git a/tools/graphrag/graph_search.py b/tools/graphrag/graph_search.py index a0c7493..9d897f3 100644 --- a/tools/graphrag/graph_search.py +++ b/tools/graphrag/graph_search.py @@ -1841,6 +1841,15 @@ def search_graph_with_session( kinds={work.work_type for work in spec.works if work.work_type} constraints['thesis_types']=sorted(k for k in kinds if k.endswith('_thesis')) constraints['work_types']=sorted(k for k in kinds if not k.endswith('_thesis')) + constraints['mentions_thesis']=bool(constraints['thesis_types']) + constraints['mentions_work']=bool(constraints['work_types']) + constraints['mentions_count']=spec.presentation=='count' + constraints['mentions_work_listing']=bool( + spec.presentation=='list' and spec.scope=='across_works' + and not set(spec.fields)&{'keyword','category','document_author','start_year'} + or 'work_type' in spec.fields and spec.scope=='across_works') + if spec.presentation=='count': + constraints['mentions_academic_year']=True # fetch academic records, not meetings constraints["published_only"] = published_only # Explicit tag filters are resolved separately. Incidental names such as the # tag 'zp' in 'ZP Wiki' must never become additional query constraints. @@ -2022,6 +2031,8 @@ def search_graph_with_session( normalize_for_match(clean_query[: match.start()]), ) ] + from tools.retrieval.work_selector import HEADING,label_key + quoted_titles=[title for title in quoted_titles if not HEADING.fullmatch(label_key(title))] if quoted_titles and (thesis_specific or work_specific) and not thesis_ids and not work_ids: blocked_reasons.append("unknown_title") constraints["unresolved_title"] = True @@ -2201,8 +2212,12 @@ def search_graph_with_session( aggregation = None if constraints["mentions_count"] and not blocked: matching = sorted({str(row["person"]) for row in thesis_rows + work_rows}) + record_count=bool(re.search(r'\b(?:kolko|pocet)\s+(?:roznych\s+|evidovanych\s+)?(?:prac\w*|projekt\w*)\b',normalize_for_match(clean_query))) + records={row.get('thesis_id') or row.get('work_id') or row.get('thesis_element_id') or row.get('work_element_id') for row in thesis_rows+work_rows} + records.discard(None) aggregation = { - "count": len(matching), + "count": len(records) if record_count else len(matching), + "count_domain":'academic_records' if record_count else 'people', "people": matching, "candidate_count": len(person_ids), "complete": not (entity_limit_reached or fetch_capped), diff --git a/tools/knowledge/academic.py b/tools/knowledge/academic.py index 6a36d9f..2d8c24c 100644 --- a/tools/knowledge/academic.py +++ b/tools/knowledge/academic.py @@ -12,6 +12,45 @@ import re from tools.answers.context_policy import normalized from tools.knowledge.sources import advisor_name + + +def with_profile_metadata(result,plan,db_file): + """Keep indexed document authorship and literal study start separate from work events.""" + import sqlite3 + from tools.sync.index_state import index_snapshot,read_version + from tools.rag.search_core import make_source_url + if result.get('status')!='ok' or not result.get('accepted_for_delivery'):return result + with index_snapshot(db_file.parent,db_file=db_file): + if read_version(db_file.parent)!=result['index_version']:raise RuntimeError('Authorship snapshot changed') + with sqlite3.connect(db_file.resolve().as_uri()+'?mode=ro',uri=True) as conn: + rows=[] + for person in plan.spec.people: + if person.role!='student' or not person.source_document:continue + row=conn.execute('SELECT author FROM documents WHERE path=? AND COALESCE(published,1)=1', + (person.source_document,)).fetchone() + if 'document_author' in plan.spec.fields and row and row[0]:rows.append({'person':person.name,'document_author':row[0], + 'source_path':person.source_document,'source_url':make_source_url(person.source_document),'metadata_field':'author'}) + if 'start_year' in plan.spec.fields: + starts={} + for chunk_id,text in conn.execute('SELECT chunk_id,text FROM chunks WHERE document_path=? AND COALESCE(published,1)=1',(person.source_document,)): + for line in text.splitlines(): + match=re.fullmatch(r'rok zaciatku studia ([12]\d{3})',normalized(line)) + if match:starts[int(match[1])]=(chunk_id,line) + if len(starts)==1: + year=next(iter(starts));chunk_id,quote=starts[year] + rows.append({'person':person.name,'start_year':year,'source_path':person.source_document, + 'source_url':make_source_url(person.source_document),'metadata_field':'literal_start_year','chunk_id':chunk_id,'quote':quote}) + if not rows:return result + body,separator,sources=result['answer'].rpartition('\n\nZdroje:') + if not separator:body=result['answer'] + lines=[('Autor dokumentu profilu '+r['person']+': '+r['document_author']+'.') if 'document_author' in r else + ('Explicitný rok začiatku štúdia — '+r['person']+': '+str(r['start_year'])+'.') for r in rows] + if 'start_year' in plan.spec.fields and 'year' in plan.spec.fields: + lines.append('Rok práce, dátum záznamu a dátum vykonania činnosti sú odlišné údaje.') + urls=list(dict.fromkeys([*result['source_urls'],*(r['source_url'] for r in rows)])) + answer=body+'\n\n'+'\n'.join(lines)+'\n\nZdroje: '+', '.join(urls) + return {**result,'answer':answer,'metadata_evidence':rows, + 'source_urls':urls} from tools.retrieval.person_names import resolve_person_names from tools.retrieval.work_selector import parse_work_selector @@ -33,7 +72,7 @@ def academic_intent(question): return "meeting" if re.search(r"\bkto\b.*\b(?:vedie|viedol|viedla)\b", text): return "advisors" - if re.search(r"\b(?:veduc\w*|skolitel\w*|konzult(?:ant|oval)\w*)\b", text): + if re.search(r"\b(?:veduc\w*|skolitel\w*|konzult\w*)\b", text): return "advisors" return None @@ -205,7 +244,7 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None, role_filter = bool( staff and re.search( - r"\b(?:pod vedenim|vedie|viedol|veden\w*|veduc\w*|skolitel\w*|konzult(?:ant|oval)\w*)\b", text + r"\b(?:pod vedenim|vedie|viedol|veden\w*|veduc\w*|skolitel\w*|konzult\w*)\b", text ) ) if role_filter: @@ -276,6 +315,13 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None, reason="completion_year_scope", ) selected_paths = {m["source_document"] for m in students} + profile_roles=defaultdict(list) + for row in rows: + if row['kind'] in {'supervisor','consultant'} and not row.get('work_id'): + profile_roles[row['source_path']].append(row) + profile_work_ids=defaultdict(set) + for row in index['units']: + if row['kind']=='work_record' and row.get('work_id'):profile_work_ids[row['source_path']].add(row['work_id']) groups = defaultdict(list) for r in rows: if plan and not plan.spec.matches_work(r['person'],r.get('work_label'),r.get('work_type'),r.get('work_year')):continue @@ -301,7 +347,7 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None, groups[(r["source_path"], r.get("work_label"))].append(r) requested_role = ( "consultant" - if re.search(r"\bkonzult(?:ant|oval)\w*\b", text) and not re.search(r"\bveduc\w*|\bveden\w*", text) + if re.search(r"\bkonzult\w*\b", text) and not re.search(r"\bveduc\w*|\bveden\w*", text) else "supervisor" ) if role_filter: @@ -412,6 +458,9 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None, else f'**{marker["person"]}** — záznam: {marker["date_label"]}; rok nie je uvedený.' ) evidence.append(marker) + if (plan and {'start_year','date'}<=set(plan.spec.fields) and not plan.spec.facets + and not re.search(r'\bo com\b|\bco sa\b',text)): + continue # A comparison of time roles does not ask for the meeting agenda. notes = [ r for r in part @@ -481,7 +530,12 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None, ("supervisor", "Vedúci práce"), ("consultant", "Konzultant"), ]: + requested=set(plan.spec.fields)&{'supervisor','consultant'} if plan else set() + if requested and role not in requested:continue roles = [r for r in part if advisor_name(r, role)] + if not roles and len(profile_work_ids[key[0]])==1: + roles=[r for r in profile_roles[key[0]] if advisor_name(r,role)] + if roles:name_label='Vedúci v hlavičke profilu' if role=='supervisor' else 'Konzultant v hlavičke profilu' names = list(dict.fromkeys(advisor_name(r, role) for r in roles)) lines.append( name_label diff --git a/tools/knowledge/local_graph.py b/tools/knowledge/local_graph.py index ba39fc6..a9c2a24 100644 --- a/tools/knowledge/local_graph.py +++ b/tools/knowledge/local_graph.py @@ -175,7 +175,7 @@ def retrieve_local_graph(question, index, bound, *, limit=5): 'supervisors':[store.identifier(normalized(p.name)) for p in spec.people if p.role=='supervisor'], 'consultants':[store.identifier(normalized(p.name)) for p in spec.people if p.role=='consultant'], 'date_field':period.field if period else None,'start':period.start if period else None,'end':period.end if period else None, - 'profile_records':not any(w.work_type or w.years or w.labels or w.numbers or + 'profile_records':spec.intent=='advisors' or not any(w.work_type or w.years or w.labels or w.numbers or w.excluded_labels or w.excluded_numbers or w.unnumbered for w in spec.works), 'budget':EVIDENCE_LIMIT+1,'peers':min(61,max(11,limit*3+1))} settings=get_neo4j_settings() diff --git a/tools/knowledge/planned_answers.py b/tools/knowledge/planned_answers.py index 3c6b3aa..1ef0362 100644 --- a/tools/knowledge/planned_answers.py +++ b/tools/knowledge/planned_answers.py @@ -46,6 +46,16 @@ def answer_facets(bound, index, units, *, engine, limit): keys=[part[i] for i in range(max((len(v) for v in by_person.values()),default=0)) for part in by_person.values() if ilimit;size=0;notes=[] cells=defaultdict(list) for key in keys[:limit]: @@ -58,8 +68,23 @@ def answer_facets(bound, index, units, *, engine, limit): for row in groups[key]: if row['kind'] in {'reference','work_record','meeting','supervisor','consultant','lifecycle','topic'}:continue role={'assignment':'assignment','planned':'task','result':'progress','progress':'progress'}.get(row['phase'],'other') - if facet in facets_for(row['text'],role,plan.question,row['phase']):candidates.append(row) - candidates.sort(key=lambda r:(-unit_score(r['text'], {'result':'progress','progress':'progress','planned':'task'}.get(r['phase'],r['phase']),facet),r['line'])) + proposal=(facet=='assignment' and row['phase']=='planned' and row.get('section_scope')=='work' + and any(ntake:truncated=True for row in candidates[:take]: if size+len(row['text'])>18000 or work_size+len(row['text'])>work_budget: @@ -76,9 +101,13 @@ def answer_facets(bound, index, units, *, engine, limit): size+=len(row['text']);work_size+=len(row['text']) quote=re.sub(r'^\s*(?:[-*+]\s+|\d+[.)]\s+)','',row['text']) phase={'assignment':'zadanie','planned':'plán','progress':'priebežný stav','result':'zaznamenaný výsledok'}.get(row['phase'],'podklad') + if facet=='assignment' and row['phase']=='planned':phase='navrhovaný cieľ' lines.append('- '+quote+' ('+phase+')') cells[(key,facet)].append(quote+' ('+phase+')') if row['id'] not in used:evidence.append(row);used.add(row['id']) + if facet=='assignment' and candidates and key in tentative_goals: + note=person+' — '+label+': zdroj tieto body označuje ako možné/navrhované ciele, nie ako potvrdené splnené výsledky.' + notes.append(note);lines.append(note) if not cells[(key,facet)]:lines.append('Doložené výňatky sa nezmestili do tohto výpisu.') if plan.presentation in {'comparison','table'} and keys: from tools.answers.presentation import comparison_table diff --git a/tools/knowledge/query.py b/tools/knowledge/query.py index 92d0dcd..d21bf1f 100644 --- a/tools/knowledge/query.py +++ b/tools/knowledge/query.py @@ -161,9 +161,17 @@ def activity_supported(question, row): compares the workflow to NER. These are distinct research tasks in this wiki. """ wording = normalized(question) + achieved=re.search(r'\b(?:vytvoril\w*|implementoval\w*|nasadil\w*|pouzil\w*|vyhodnotil\w*|vyskus(?:al|ala|ali)\w*)\b',wording) + if achieved and row.get('phase') not in {'result','progress'}: + return False named_entities = bool( re.search(r"\bpomenovan\w* entit\w*\b|\bnamed entit\w*\b|\bner\b", wording) ) + if named_entities: + literal=row['text']+' '+' '.join(u['text'] for u in row.get('context_quotes',[])) + if 'ner' not in concept_families(literal) and not re.search( + r'\bpomenovan\w* entit\w*\b|\bnamed entit\w*\b',normalized(literal)): + return False if named_entities and re.search(r'\brozpoznav\w*|\brozpoznava\w*|\bnamed entity recognition\b|\bner\b', wording): text = normalized(row['text']) if re.search(r'\banot\w*\b', text) and not re.search( @@ -1035,6 +1043,11 @@ def deliver_content(question: str, *, engine: str, limit: int, db_file, complete if (note["source_path"], note.get("work_label"), note["text"]) not in existing ), ] + retrieval_trace=None + retrieved_relevance=None + if engine=='rag' and plan and mode not in {'advisors','completion','supervisor'}: + from tools.knowledge.text_retrieval import retrieve_units + units,retrieved_relevance,retrieval_trace=retrieve_units(question,index,plan.spec,limit=limit) def traced(result): if not include_diagnostics: return result @@ -1044,7 +1057,8 @@ def deliver_content(question: str, *, engine: str, limit: int, db_file, complete pool=[row for row in pool if (not people or row['person'] in people) and plan.spec.matches_work(row['person'],row.get('work_label'),row.get('work_type'),row.get('work_year'))] return {**result,'retrieval_diagnostics':{ - 'strategy':'bounded_local_graph' if local is not None else 'content_checkpoint', + **(retrieval_trace or {}), + 'strategy':'bounded_local_graph' if local is not None else (retrieval_trace or {}).get('strategy','content_checkpoint'), 'stage':'loaded_scoped_units_before_answer_selection', 'candidate_ids':list(dict.fromkeys(row['id'] for row in pool)), 'candidate_count':len(pool)}} @@ -1080,8 +1094,8 @@ def deliver_content(question: str, *, engine: str, limit: int, db_file, complete from tools.knowledge.sources import with_work_context units = with_work_context(units) - relevance = None - if local is None and mode in {"discovery", "similar"} and index.get("vectors"): + relevance = retrieved_relevance + if local is None and relevance is None and mode in {"discovery", "similar"} and index.get("vectors"): import base64 import zlib import numpy as np @@ -1129,6 +1143,9 @@ def deliver_content(question: str, *, engine: str, limit: int, db_file, complete plan=plan, local_graph=local, ) + if mode in {'advisors','meeting'} and plan and set(plan.spec.fields)&{'document_author','start_year'}: + from tools.knowledge.academic import with_profile_metadata + result=with_profile_metadata(result,plan,db_file) if local is not None: from tools.knowledge.graph_answers import graph_details result = graph_details(result,local) diff --git a/tools/knowledge/text_retrieval.py b/tools/knowledge/text_retrieval.py new file mode 100644 index 0000000..bfcdc23 --- /dev/null +++ b/tools/knowledge/text_retrieval.py @@ -0,0 +1,81 @@ +"""Independent FTS5/BM25 and E5 candidates over witnessed source assertions. + +The checkpoint is an evidence store, not an alternative to text retrieval. +Named finite scopes are expanded after ranking so a bounded candidate list +cannot silently remove an assignment goal or one side of a comparison. +""" +from functools import lru_cache +import base64 +import sqlite3 +import threading +import zlib + +from tools.answers.context_policy import normalized + +_lock = threading.RLock() + + +@lru_cache(maxsize=2) +def lexical_index(version, texts): + connection = sqlite3.connect(':memory:', check_same_thread=False) + connection.execute('CREATE VIRTUAL TABLE evidence USING fts5(id UNINDEXED, text, tokenize="unicode61 remove_diacritics 2")') + connection.executemany('INSERT INTO evidence VALUES (?,?)',texts) + return connection + + +def retrieve_units(question, index, spec, *, limit=5): + from tools.knowledge.query import _terms + from tools.knowledge.evidence import claim_eligible + people={p.name for p in spec.people if p.role=='student'} + similar=spec.intent=='similar' + def in_scope(row): + if not spec.matches_work(row['person'],row.get('work_label'),row.get('work_type'),row.get('work_year')):return False + if spec.period and spec.period.field!='work_year': + value=row.get(spec.period.field) + if not value or not spec.period.start<=value<=spec.period.end:return False + return similar or not people or row['person'] in people + candidates=[r for r in index['units'] if in_scope(r)] + eligible={r['id'] for r in candidates} + texts=tuple((r['id'],r['text']+' '+' '.join(r.get('concepts',[]))) for r in index['units']) + connection=lexical_index(index['version'],texts) + terms=_terms(question) + if similar: + # The reference's actual substantive topics seed the peer search. + from tools.knowledge.concepts import substantive_concept + terms={normalized(c) for r in candidates if r['person'] in people and claim_eligible(r) + for c in r.get('concepts',[]) if substantive_concept(c)} or terms + expression=' OR '.join('"'+term.replace('"','""')+'"*' for term in sorted(terms)) + with _lock: + lexical=[r[0] for r in connection.execute( + 'SELECT id FROM evidence WHERE evidence MATCH ? ORDER BY bm25(evidence),id', + (expression,)) if r[0] in eligible] if expression else [] + semantic={} + if index.get('vectors'): + import numpy as np + from tools.rag.embedding_utils import embed_query,model_profile,embedding_model_name + if model_profile(embedding_model_name())['fingerprint']!=index['embedding_fingerprint']: + raise RuntimeError('Content embeddings belong to another model') + matrix=np.frombuffer(zlib.decompress(base64.b64decode(index['vectors'])),dtype='len(rows)} diff --git a/tools/rag/llm_context.py b/tools/rag/llm_context.py index 31c523e..b2258e2 100644 --- a/tools/rag/llm_context.py +++ b/tools/rag/llm_context.py @@ -37,6 +37,7 @@ FIELDS = ( "clarification", "named_subjects", "work_scopes", + "named_scope_complete", "question_plan", "resolved_query", "name_resolution", diff --git a/tools/retrieval/person_names.py b/tools/retrieval/person_names.py index 4322350..6f30b87 100644 --- a/tools/retrieval/person_names.py +++ b/tools/retrieval/person_names.py @@ -73,7 +73,7 @@ def _score(words: list[str], canonical: list[str]) -> int | None: if ( not actual.isalpha() or not expected.isalpha() - or min(len(actual), len(expected)) < 4 + or min(len(actual), len(expected)) < 3 ): return None cap = 2 if min(len(actual), len(expected)) >= 8 else 1 @@ -189,7 +189,7 @@ def resolve_person_names(query: str, catalog: list[dict]) -> dict: } author_lookup = bool( re.search( - r"\b(?:autor\w*|dokument\w*|vedie|viedol|veduc\w*|skolitel\w*|konzult(?:ant|oval)\w*)\b", normalize(query) + r"\b(?:autor\w*|dokument\w*|vedie|viedol|veduc\w*|skolitel\w*|konzult\w*)\b", normalize(query) ) ) people = [ @@ -345,7 +345,14 @@ def resolve_person_names(query: str, catalog: list[dict]) -> dict: and not token[0].isupper() and not tokens[i + 1][0].isupper() ) - comparison_subject = bool(selected and pair and re.search(r"\b(?:a|porovnaj)\s*$", prefix)) + # A conjunction introduces another identity only while enumerating + # names. "Ján Holp ... a nezamieňaj ich" is a presentation request. + adjacent_name = any(right <= token.start() and re.fullmatch( + r"\s*(?:,|a)\s*", query[right:token.start()]) for _,right,_ in selected) + comparison_subject = bool(selected and pair and (adjacent_name or re.fullmatch(r"porovnaj\s*",prefix))) + identity_role = bool(re.search(r"\b(?:osob\w*|student\w*|kandidat\w*|autor(?:a|om)?)\s*$",prefix)) and not reference + owner_slot = role and not selected and not re.match( + r'^(?:kto|ktori|ktory|u koho)\b', normalize(query)) # Lower-case unknown entries in an explicit candidate list must still # fail closed, even if they follow a comma or a known person. list_subject = explicit_list and any( @@ -353,8 +360,8 @@ def resolve_person_names(query: str, catalog: list[dict]) -> dict: for _, right, _ in selected ) if not ( - role - or (pair and words[i] in first_names) + owner_slot or (identity_role and pair) + or words[i] in first_names or upper_pair or comparison_subject or list_subject