diff --git a/README.md b/README.md index 80d0202..f789d3c 100644 --- a/README.md +++ b/README.md @@ -695,6 +695,18 @@ vyvolá rollback. Pri nedokončenom prepnutí po páde zostane vyhľadávanie nedostupné do úspešného opakovania synchronizácie. Podrobnosti a zotavenie: [Spoločná synchronizácia indexov](docs/synchronizacia_indexov.md). +## Publikovanie a dostupnosť nástrojov + +Oba nástroje predvolene vylučujú iba explicitné `published=false`. +Chýbajúca hodnota sa považuje za publikovanú. Parameter +`published_only=false` výslovne zahrnie aj nepublikované dokumenty pri +autorizovanom volaní; CLI používa `--include-unpublished`. + +`/health` uvádza samostatne `tools.rag`, `tools.neo4j` a `tools.graphrag`. +Globálne `ready` vyžaduje oba vyhľadávacie nástroje; pri výpadku Neo4j +môže klient ďalej používať RAG podľa `tools.rag.ready`. +[Politika, migrácia a príklady](docs/publikovanie_a_dostupnost.md). + ## Hybridné vyhľadávanie Vyhľadávanie kombinuje: @@ -747,7 +759,7 @@ curl -X POST http://127.0.0.1:8000/search \ -d '{ "query": "strojový preklad", "limit": 3, - "published_only": false, + "published_only": true, "max_per_document": 1 }' ``` @@ -798,7 +810,7 @@ curl -X POST http://127.0.0.1:8000/rag \ -d '{ "query": "V akom roku robil Ján Holp diplomovú prácu?", "limit": 5, - "published_only": false, + "published_only": true, "max_per_document": 1 }' ``` diff --git a/app/main.py b/app/main.py index 1055a7b..3c3aad2 100644 --- a/app/main.py +++ b/app/main.py @@ -41,15 +41,17 @@ OPENWEBUI_ORIGIN = ( @asynccontextmanager async def lifespan( - _: FastAPI, + application: FastAPI, ): validate_security_configuration() + application.state.embedding_ready = False try: await asyncio.to_thread( embed_query, "warmup", ) + application.state.embedding_ready = True except Exception: logger.exception( diff --git a/app/routes.py b/app/routes.py index f0be588..6205fb0 100644 --- a/app/routes.py +++ b/app/routes.py @@ -46,7 +46,8 @@ from tools.rag.rebuild_index import ( rebuild_index, ) from tools.rag.search_utils import search_database -from tools.index_state import index_snapshot, IndexUnavailableError +from tools.publication import DEFAULT_PUBLISHED_ONLY +from tools.tool_health import tools_readiness logger = logging.getLogger(__name__) @@ -104,7 +105,7 @@ class SearchRequest( le=50, ) - published_only: bool = False + published_only: bool = DEFAULT_PUBLISHED_ONLY max_per_document: int = Field( default=1, @@ -137,10 +138,10 @@ class RagRequest( ) published_only: bool = Field( - default=False, + default=DEFAULT_PUBLISHED_ONLY, description=( "Ak je true, použijú sa iba " - "publikované dokumenty." + "dokumenty okrem explicitného published=false; chýbajúca hodnota sa zahrnie." ), ) @@ -184,6 +185,7 @@ class HealthResponse(BaseModel): security_configured: bool index_version: dict[str, Any] | None = None + tools: dict[str, Any] class RagSource(BaseModel): @@ -515,7 +517,7 @@ async def read_webhook_body( include_in_schema=False, response_model=HealthResponse, ) -def health() -> dict[str, Any]: +def health(request: Request) -> dict[str, Any]: database_exists = ( DB_FILE.exists() ) @@ -532,22 +534,15 @@ def health() -> dict[str, Any]: except RuntimeError: security_configured = False - ready = ( - database_exists - and zpwiki_exists - and security_configured - ) - version = None - try: - with index_snapshot(DB_FILE.parent, db_file=DB_FILE) as version: - pass - except (IndexUnavailableError, sqlite3.Error): - ready = False + tools, version = tools_readiness(DB_FILE, security_configured=security_configured, + embedding_ready=getattr(request.app.state, 'embedding_ready', False)) + ready = tools['rag']['ready'] and tools['graphrag']['ready'] return { "status": "ok", "ready": ready, "index_version": version, + "tools": tools, "database_exists": ( database_exists ), @@ -623,9 +618,10 @@ def health() -> dict[str, Any]: ) def rag( request: RagRequest, + http_request: Request, ) -> dict[str, Any]: try: - return build_rag_context( + result = build_rag_context( DB_FILE, request.query, limit=request.limit, @@ -636,6 +632,8 @@ def rag( request.max_per_document ), ) + http_request.app.state.embedding_ready = True + return result except Exception as error: raise_retrieval_error( @@ -655,6 +653,7 @@ def rag( ) def search( request: SearchRequest, + http_request: Request, ) -> dict[str, Any]: try: response = search_database( @@ -668,6 +667,7 @@ def search( request.max_per_document ), ) + http_request.app.state.embedding_ready = True except Exception as error: raise_retrieval_error( @@ -690,6 +690,7 @@ def search( results ), "results": results, + **({'index_version': response['index_version']} if response.get('index_version') else {}), } @@ -705,6 +706,7 @@ def search( ) def sync( request: SyncRequest, + http_request: Request, ) -> dict[str, Any]: try: result = rebuild_index( @@ -712,6 +714,7 @@ def sync( request.pull_git ) ) + http_request.app.state.embedding_ready = True except Exception as error: raise_reindex_error( @@ -890,6 +893,7 @@ async def gitea_webhook( webhook_should_pull_git() ), ) + request.app.state.embedding_ready = True except Exception as error: raise_reindex_error( diff --git a/docs/publikovanie_a_dostupnost.md b/docs/publikovanie_a_dostupnost.md new file mode 100644 index 0000000..6c95b81 --- /dev/null +++ b/docs/publikovanie_a_dostupnost.md @@ -0,0 +1,102 @@ +# Publikovanie dokumentov a dostupnosť nástrojov + +## Jednotná politika + +Používateľ zvolil zahrnutie dokumentov bez `published`. RAG aj GraphRAG preto +predvolene používajú `published_only=true` s týmto významom: + +| Hodnota zdroja | Predvolené vyhľadávanie | +|---|---| +| `published: true` | Zahrnúť | +| Chýbajúce `published` alebo normalizované `null` | Zahrnúť | +| `published: false` | Vylúčiť | + +Chýbajúca hodnota zodpovedá predvolenému významu príznaku v +[Grav](https://learn.getgrav.org/2/content/headers). Táto politika určuje +výber podľa `published`; API naďalej používa svoju existujúcu autentifikáciu. +V aktuálnom korpuse nie sú nastavené `publish_date` ani `unpublish_date`. + +SQL používa `COALESCE(published, 1) = 1`, graf efektívnu publikačnú hodnotu. +Zdrojový JSON a SQLite zachovávajú pôvodné `null`; grafový builder ho +normalizuje na efektívne `true`. Všetkých 16 pôvodne neoznačených dokumentov +je teda dostupných v oboch vetvách. Jeden explicitne nepublikovaný dokument +sa predvolene vylúči. + +Pri RAG sa filter aplikuje na FTS, embeddingy aj všetky rozšírenia kontextu. +Pri grafe platí pre katalóg entít, fakty, spoločné vlastnosti, počítanie, +prechádzanie grafu a zoznam zdrojov. Entity doložené iba skrytými dokumentmi +sa nezaradia do verejného katalógu. Pôvod názvu témy a canonical profilu +osoby sa kontroluje podľa príslušného zdrojového dokumentu. + +Výslovné zahrnutie nepublikovaných dokumentov pre autorizované volanie: + +```json +{"query": "vyhľadávací dotaz", "published_only": false} +``` + +Python funkcie používajú rovnaký parameter `published_only=False`. +Oba vyhľadávacie CLI podporujú `--include-unpublished`. Pred pripojením +nástrojov do OpenWebUI sa používa aktuálna OpenAPI schéma s novým defaultom. + +## Migrácia uloženého grafu + +Po nasadení sa vykoná spoločná synchronizácia. Tá opraví staré zamenenie +chýbajúcej hodnoty s `false` a doplní pôvod názvu témy. Verzia indexov obsahuje +`publication_policy: explicit-false-hidden-v1`. Pred migráciou predvolené +grafové vyhľadávanie vyžaduje synchronizáciu a health uvádza +`publication_sync_required`. Existujúce historické merania sa neprepisujú; +časť klasického baseline používala aj nepublikované dokumenty. + +## `/health` + +HTTP 200 a `status=ok` označujú živý API proces. `ready` znamená pripravenosť +oboch vyhľadávacích nástrojov. Klient/agenta pri výbere konkrétneho nástroja +zaujímajú samostatné položky `tools`: + +- `tools.rag`: čitateľný SQLite/FTS5/embedding index, zhoda verzie a známa + úspešná inicializácia embeddingového modelu. Úspešné vyhľadanie alebo sync + obnoví stav modelu po prípadnom zlyhaní inicializácie. +- `tools.neo4j`: dostupné spojenie a vykonanie read-only sondy v databáze. + Sonda používa krátke časové limity a nevykonáva automatické opakovania transakcií. +- `tools.graphrag`: dostupné Neo4j, neprázdny graf, zhodná generácia dát + a migrovaná publikačná politika. + +Výpadok Neo4j nespôsobí nedostupnosť RAG. Neplatný SQLite index môže vyradiť +RAG, pričom graf ostane dostupný. Počas nejasného prepnutia generácie sa +vyhľadávacie nástroje označia ako nedostupné; stav samotného Neo4j sa uvádza +samostatne. Odpovede neobsahujú prihlasovacie údaje ani surové chyby drivera. + +Príklad výpadku Neo4j, skrátený na rozhodujúce polia: + +```json +{ + "status": "ok", + "ready": false, + "tools": { + "rag": {"ready": true, "status": "ready", "reason": null}, + "neo4j": {"ready": false, "status": "unavailable", "reason": "unreachable"}, + "graphrag": {"ready": false, "status": "unavailable", "reason": "neo4j_unavailable"} + } +} +``` + +Ide o dostupnosť retrieval nástrojov. Health neposiela otázky školskému LLM +a nemeria dostupnosť externého generatívneho modelu ani presnosť odpovedí. + +## Overenie nasadenia 25. 10. 2026 + +- Kompletná regresná sada: **485 úspešných testov, 4 preskočené**. Zahŕňa + skutočné Cypher testy filtrovania, agregácie a grafových prechodov nad + izolovaným Neo4j grafom s rollbackom. +- Docker image bol znovu zostavený a API nasadené. Autentifikovaný `/sync` + bez Git pullu obnovil 114 dokumentov, 887 chunkov a graf 447 uzlov/879 vzťahov. +- Reálne `/search`, `/rag` a `search_graph()` overili predvolené vylúčenie + skrytého dokumentu aj jeho zahrnutie pri `published_only=false`. +- HTTP `/health` hlási všetky tri stavy pripravené; SQLite, súborová verzia + a Neo4j používajú rovnakú novú generáciu dát. +- Sonda s nefunkčnou adresou Neo4j potvrdila dostupnosť RAG pri nedostupnom + grafe. Sonda s chýbajúcim SQLite indexom potvrdila dostupnosť grafu pri + nedostupnom RAG. Produkčné služby sa pri týchto sondách nevypínali. + +Strojovo čitateľný výsledok a identifikácia dát: +[publikovanie_a_dostupnost_overenie.json](publikovanie_a_dostupnost_overenie.json). diff --git a/docs/publikovanie_a_dostupnost_overenie.json b/docs/publikovanie_a_dostupnost_overenie.json new file mode 100644 index 0000000..92faceb --- /dev/null +++ b/docs/publikovanie_a_dostupnost_overenie.json @@ -0,0 +1,162 @@ +{ + "status": "ok", + "index_version": { + "version": "ff682d98134d406081040902f234c44f", + "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e", + "created_at_utc": "2026-10-25T15:06:03.279424+00:00", + "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363", + "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52", + "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9", + "embedding_model": "intfloat/multilingual-e5-small", + "publication_policy": "explicit-false-hidden-v1" + }, + "health": { + "status": "ok", + "ready": true, + "database_exists": true, + "search_engine": "hybrid_fts5_embeddings", + "rag_enabled": true, + "zpwiki_exists": true, + "security_configured": true, + "index_version": { + "version": "ff682d98134d406081040902f234c44f", + "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e", + "created_at_utc": "2026-10-25T15:06:03.279424+00:00", + "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363", + "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52", + "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9", + "embedding_model": "intfloat/multilingual-e5-small", + "publication_policy": "explicit-false-hidden-v1" + }, + "tools": { + "rag": { + "ready": true, + "status": "ready", + "reason": null, + "embedding_model": "intfloat/multilingual-e5-small", + "dimensions": 384, + "index_version": { + "version": "ff682d98134d406081040902f234c44f", + "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e", + "created_at_utc": "2026-10-25T15:06:03.279424+00:00", + "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363", + "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52", + "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9", + "embedding_model": "intfloat/multilingual-e5-small", + "publication_policy": "explicit-false-hidden-v1" + } + }, + "neo4j": { + "ready": true, + "status": "ready", + "reason": null + }, + "graphrag": { + "ready": true, + "status": "ready", + "reason": null, + "index_version": { + "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9", + "created_at_utc": "2026-10-25T15:06:03.279424+00:00", + "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52", + "embedding_model": "intfloat/multilingual-e5-small", + "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e", + "publication_policy": "explicit-false-hidden-v1", + "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363", + "version": "ff682d98134d406081040902f234c44f" + }, + "nodes": 447 + } + } + }, + "publication_counts": { + "published_true": 97, + "missing": 16, + "published_false": 1, + "default_visible": 113 + }, + "graph_document_counts": { + "True": 113, + "False": 1 + }, + "live_search_checks": { + "/search": { + "default_hides_unpublished": true, + "explicit_opt_out_includes_it": true + }, + "/rag": { + "default_hides_unpublished": true, + "explicit_opt_out_includes_it": true + }, + "graphrag": { + "default_hides_unpublished": true, + "explicit_opt_out_includes_it": true, + "default_status": "ok" + } + }, + "neo4j_outage_probe": { + "rag": { + "ready": true, + "status": "ready", + "reason": null, + "embedding_model": "intfloat/multilingual-e5-small", + "dimensions": 384, + "index_version": { + "version": "ff682d98134d406081040902f234c44f", + "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e", + "created_at_utc": "2026-10-25T15:06:03.279424+00:00", + "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363", + "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52", + "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9", + "embedding_model": "intfloat/multilingual-e5-small", + "publication_policy": "explicit-false-hidden-v1" + } + }, + "neo4j": { + "ready": false, + "status": "unavailable", + "reason": "unreachable" + }, + "graphrag": { + "ready": false, + "status": "unavailable", + "reason": "neo4j_unavailable" + } + }, + "rag_index_outage_probe": { + "rag": { + "ready": false, + "status": "unavailable", + "reason": "index_missing" + }, + "neo4j": { + "ready": true, + "status": "ready", + "reason": null + }, + "graphrag": { + "ready": true, + "status": "ready", + "reason": null, + "index_version": { + "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9", + "created_at_utc": "2026-10-25T15:06:03.279424+00:00", + "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52", + "embedding_model": "intfloat/multilingual-e5-small", + "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e", + "publication_policy": "explicit-false-hidden-v1", + "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363", + "version": "ff682d98134d406081040902f234c44f" + }, + "nodes": 447 + } + }, + "checked_at_utc": "2026-10-25T15:09:42.459584+00:00", + "regression": { + "passed": 485, + "skipped": 4, + "failed": 0, + "includes_real_neo4j_publication_tests": true + }, + "scope": "Publikačný filter a dostupnosť retrieval nástrojov; nejde o nové meranie presnosti LLM." +} diff --git a/test/test_publication_health.py b/test/test_publication_health.py new file mode 100644 index 0000000..f24b7c2 --- /dev/null +++ b/test/test_publication_health.py @@ -0,0 +1,166 @@ +import json +import sqlite3 +from types import SimpleNamespace + +import numpy as np +import pytest +from fastapi.testclient import TestClient + +import app.main as main +import app.routes as routes +import tools.tool_health as health +import tools.rag.build_sqlite_index as indexer +import tools.rag.search_core as search_core +from tools.index_state import durable_json, PENDING_FILE, VERSION_FILE +from tools.publication import PUBLICATION_POLICY_VERSION, effective_published +from tools.rag.search_utils import search_database +from tools.rag.rag_utils import build_rag_context + + +@pytest.fixture +def corpus(tmp_path, monkeypatch): + docs = []; chunks = [] + paths = ['pages/students/2020/eva/README.md', 'pages/topics/missing/README.md', + 'pages/topics/hidden/README.md', 'pages/students/2020/eva/private/README.md'] + for index, (path, published) in enumerate(zip(paths, [True, None, False, False])): + title = ['Verejný profil', 'Chýbajúca značka', 'Skrytý dokument', 'Skrytá poznámka'][index] + text = 'spoločný overovací výraz ' + ('SKRYTY_OVEROVACI_OBSAH' if published is False else title) + docs.append({'path': path, 'title': title, 'published': published}) + chunks.append({'chunk_id': path+'::0', 'document_path': path, 'title': title, + 'published': published, 'chunk_index': 0, 'heading_paths': [['Úvod']], + 'text': text, 'text_length': len(text), 'token_count': 10, + 'content_hash': str(index), 'tags': [], 'categories': []}) + data = tmp_path / 'data'; data.mkdir() + for name, rows in [('documents.json', docs), ('chunks.json', chunks)]: + (data/name).write_text(json.dumps(rows)) + monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.ones((len(texts), 4), dtype=np.float32)) + monkeypatch.setattr(search_core, 'embed_query', lambda *args, **kwargs: np.ones(4, dtype=np.float32)) + db = data / 'zp_index.sqlite' + indexer.build_database(documents_file=data/'documents.json', chunks_file=data/'chunks.json', db_file=db) + version = {'version': 'test-generation', 'publication_policy': PUBLICATION_POLICY_VERSION} + with sqlite3.connect(db) as conn: + conn.execute('CREATE TABLE index_state (id INTEGER PRIMARY KEY, metadata_json TEXT)') + conn.execute('INSERT INTO index_state VALUES (1,?)', (json.dumps(version),)) + durable_json(data/VERSION_FILE, version) + monkeypatch.setattr(routes, 'DB_FILE', db) + monkeypatch.setattr(main, 'embed_query', lambda *args: None) + return db, version, paths + + +@pytest.mark.parametrize('value,visible', [(True, True), (None, True), (False, False)]) +def test_publication_policy(value, visible): + assert effective_published(value) is visible + + +def test_rag_default_filters_both_fts_and_vectors_but_keeps_null(corpus): + db, _, paths = corpus + result = search_database(db, 'spoločný overovací výraz', limit=10) + assert {r['document_path'] for r in result['results']} == set(paths[:2]) + assert any(r['published'] is None for r in result['results']) + all_rows = search_database(db, 'spoločný overovací výraz', limit=10, published_only=False) + assert {r['document_path'] for r in all_rows['results']} == set(paths) + + +@pytest.mark.parametrize('query', ['spoločný overovací výraz', 'Skrytý dokument', 'Skrytá poznámka']) +def test_rag_expansions_do_not_reintroduce_hidden_text(corpus, query): + db, _, paths = corpus + result = build_rag_context(db, query, limit=10) + assert all(s['document_path'] not in paths[2:] for s in result['sources']) + assert 'SKRYTY_OVEROVACI_OBSAH' not in result['context'] + all_rows = build_rag_context(db, query, limit=10, published_only=False) + assert 'SKRYTY_OVEROVACI_OBSAH' in all_rows['context'] + + +def healthy_graph(version): + return health.readiness(True), health.readiness(True, index_version=version, nodes=2) + + +def test_neo4j_outage_does_not_disable_rag(corpus, monkeypatch): + _, _, paths = corpus + monkeypatch.setattr(health, 'neo4j_readiness', lambda v: ( + health.readiness(False, 'unreachable'), health.readiness(False, 'neo4j_unavailable'))) + with TestClient(main.app) as client: + payload = client.get('/health').json() + assert payload['tools']['rag']['ready'] is True + assert payload['tools']['neo4j']['ready'] is False + assert payload['tools']['graphrag']['ready'] is False + assert payload['ready'] is False + response = client.post('/rag', json={'query': 'spoločný overovací výraz'}, + headers={'Authorization': 'Bearer '+'a'*64}) + assert response.status_code == 200 + assert {s['document_path'] for s in response.json()['sources']} == set(paths[:2]) + + +def test_rag_index_outage_does_not_disable_graph(corpus, monkeypatch): + db, version, _ = corpus + monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph) + db.unlink() + with TestClient(main.app) as client: + payload = client.get('/health').json() + assert payload['tools']['rag']['ready'] is False + assert payload['tools']['neo4j']['ready'] is True + assert payload['tools']['graphrag']['ready'] is True + assert payload['ready'] is False + + +def test_pending_sync_blocks_both_tools_but_reports_neo4j_service(corpus, monkeypatch): + db, _, _ = corpus + durable_json(db.parent/PENDING_FILE, {'target_version': 'next'}) + monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph) + with TestClient(main.app) as client: + tools = client.get('/health').json()['tools'] + assert tools['rag']['reason'] == tools['graphrag']['reason'] == 'snapshot_unavailable' + assert tools['neo4j']['ready'] is True + + +def test_embedding_startup_failure_and_successful_query_recovery(corpus, monkeypatch): + monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph) + def broken(*args): + raise RuntimeError('transient warmup failure') + monkeypatch.setattr(main, 'embed_query', broken) + with TestClient(main.app) as client: + tools = client.get('/health').json()['tools'] + assert tools['rag']['reason'] == 'embedding_not_ready' + assert tools['graphrag']['ready'] is True + response = client.post('/rag', json={'query': 'spoločný overovací výraz'}, + headers={'Authorization': 'Bearer '+'a'*64}) + assert response.status_code == 200 + assert client.get('/health').json()['tools']['rag']['ready'] is True + + +def test_neo4j_probe_is_bounded_and_does_not_expose_errors(monkeypatch): + monkeypatch.setattr(health, 'get_neo4j_settings', lambda: SimpleNamespace(database='neo4j')) + captured = {} + def fail(settings, **options): + captured.update(options) + raise RuntimeError('password=never-return-this') + monkeypatch.setattr(health, 'create_neo4j_driver', fail) + result = health.neo4j_readiness(None) + assert not result[0]['ready'] and not result[1]['ready'] + assert captured['connection_timeout'] == captured['connection_acquisition_timeout'] == 2.0 + assert captured['max_transaction_retry_time'] == 0 + assert 'never-return-this' not in json.dumps(result) + + +@pytest.mark.parametrize('actual,nodes,reason', [ + ({'version': 'different'}, 2, 'version_mismatch'), + (None, 0, 'graph_empty'), + ({'version': 'old'}, 2, 'publication_sync_required'), +]) +def test_connection_readiness_is_distinct_from_graph_readiness(monkeypatch, actual, nodes, reason): + class Result: + def single(self): + return {'nodes': nodes, 'metadata': json.dumps(actual) if actual else None} + class Connection: + def __enter__(self): return self + def __exit__(self, *args): pass + def verify_connectivity(self): pass + def session(self, **kwargs): return self + def run(self, query, **params): + assert query.timeout == 2.0 + return Result() + monkeypatch.setattr(health, 'get_neo4j_settings', lambda: SimpleNamespace(database='neo4j')) + monkeypatch.setattr(health, 'create_neo4j_driver', lambda *args, **kwargs: Connection()) + expected = {'version': 'expected'} if reason=='version_mismatch' else actual + neo, graph = health.neo4j_readiness(expected) + assert neo['ready'] is True and graph['ready'] is False and graph['reason']==reason diff --git a/tools/graphrag/graph_db.py b/tools/graphrag/graph_db.py index 18c8e0f..830c756 100644 --- a/tools/graphrag/graph_db.py +++ b/tools/graphrag/graph_db.py @@ -39,6 +39,7 @@ def get_neo4j_settings() -> Neo4jSettings: def create_neo4j_driver( settings: Neo4jSettings | None = None, + **driver_options, ): resolved = settings or get_neo4j_settings() @@ -48,6 +49,7 @@ def create_neo4j_driver( resolved.user, resolved.password, ), + **driver_options, ) diff --git a/tools/graphrag/graph_search.py b/tools/graphrag/graph_search.py index bf6cef8..0047a82 100644 --- a/tools/graphrag/graph_search.py +++ b/tools/graphrag/graph_search.py @@ -5,6 +5,8 @@ import json import re import unicodedata from typing import Any, Iterable +from tools.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION +from .publication import public_query GRAPH_SCOPE = "zpwiki" @@ -2286,6 +2288,23 @@ RETURN document.path AS document_path, document.source_url AS source_url ORDER BY document_path """ +# Restrict the complete retrieval pipeline, including vocabulary and traversal. +CATALOG_QUERY = public_query(CATALOG_QUERY, nodes=('n',)) +NEIGHBORHOOD_QUERY = public_query(NEIGHBORHOOD_QUERY, path=True) +PAIR_PATH_QUERY = public_query(PAIR_PATH_QUERY, path=True) +THESIS_FACTS_QUERY = public_query(THESIS_FACTS_QUERY, nodes=('person',), documents=('document',)) +WORK_FACTS_QUERY = public_query(WORK_FACTS_QUERY, nodes=('person',), documents=('document',)) +THESIS_PATHS_QUERY = public_query(THESIS_PATHS_QUERY, nodes=('person',), documents=('document',)) +WORK_PATHS_QUERY = public_query(WORK_PATHS_QUERY, nodes=('person',), documents=('document',)) +START_YEAR_FACTS_QUERY = public_query(START_YEAR_FACTS_QUERY, nodes=('person',)) +PERSON_KEYWORD_FACTS_QUERY = public_query(PERSON_KEYWORD_FACTS_QUERY, nodes=('person',), documents=('document',)) +PERSON_CATEGORY_FACTS_QUERY = public_query(PERSON_CATEGORY_FACTS_QUERY, nodes=('person',), documents=('document',)) +PERSON_DOCUMENT_AUTHOR_FACTS_QUERY = public_query(PERSON_DOCUMENT_AUTHOR_FACTS_QUERY, nodes=('person',), documents=('document',)) +SHARED_KEYWORD_FACTS_QUERY = public_query(SHARED_KEYWORD_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document')) +SHARED_CATEGORY_FACTS_QUERY = public_query(SHARED_CATEGORY_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document')) +SHARED_DOCUMENT_AUTHOR_FACTS_QUERY = public_query(SHARED_DOCUMENT_AUTHOR_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document')) +DOCUMENT_SOURCES_QUERY = public_query(DOCUMENT_SOURCES_QUERY, documents=('document',)) + def search_graph_with_session( session: Any, @@ -2293,6 +2312,7 @@ def search_graph_with_session( *, limit: int = DEFAULT_LIMIT, max_entities: int = DEFAULT_MAX_ENTITIES, + published_only: bool = DEFAULT_PUBLISHED_ONLY, ) -> dict[str, Any]: clean_query = str(query or "").strip() if not clean_query: @@ -2303,7 +2323,8 @@ def search_graph_with_session( raise ValueError("max_entities musí byť aspoň 1") constraints = extract_query_constraints(clean_query) - catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE) + catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE, published_only=published_only) + constraints['published_only'] = published_only # Explicit tag filters are resolved separately. Incidental names such as the # tag 'zp' in 'ZP Wiki' must never become additional query constraints. phrase = constraints["keyword_phrase"] @@ -2425,7 +2446,8 @@ def search_graph_with_session( # the presentation limit. The catalog bounds the number of academic nodes; # relational lookups use a conservative bound covering the managed graph. fetch_limit = max(limit, len(catalog) ** 2) - common = {"scope": GRAPH_SCOPE, "person_ids": person_ids, "limit": fetch_limit} + common = {"scope": GRAPH_SCOPE, "person_ids": person_ids, "limit": fetch_limit, + "published_only": published_only} blocked_reasons = [] if constraints["unresolved_keyword"]: blocked_reasons.append("unknown_keyword") @@ -2507,7 +2529,8 @@ def search_graph_with_session( )) if not structured and resolved_entities and not blocked: start_ids = [str(row["element_id"]) for row in resolved_entities] - params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit} + params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit, + "published_only": published_only} if constraints["mentions_shared"] and len(start_ids) >= 2: paths.extend(run_rows(session, PAIR_PATH_QUERY, **params)) else: @@ -2543,8 +2566,11 @@ def search_graph_with_session( # actual stored URLs rather than manufacturing URLs from expected answers. documents = collect_documents(resolved_entities, paths, facts) sources = run_rows(session, DOCUMENT_SOURCES_QUERY, scope=GRAPH_SCOPE, - paths=[row["document_path"] for row in documents]) if documents else [] + paths=[row["document_path"] for row in documents], + published_only=published_only) if documents else [] by_path = {row["document_path"]: row.get("source_url") for row in sources} + if published_only: + documents = [row for row in documents if row['document_path'] in by_path] for document in documents: if not document.get("source_url"): document["source_url"] = by_path.get(document["document_path"]) @@ -2594,6 +2620,7 @@ def search_graph( max_entities: int = ( DEFAULT_MAX_ENTITIES ), + published_only: bool = DEFAULT_PUBLISHED_ONLY, ) -> dict[str, Any]: from .graph_db import ( create_neo4j_driver, @@ -2607,12 +2634,15 @@ def search_graph( ) with index_snapshot(DATA_DIR) as version: + if published_only and (version is None or version.get('publication_policy') != PUBLICATION_POLICY_VERSION): + raise IndexUnavailableError('Graf vyžaduje synchronizáciu pravidiel publikovania.') with create_neo4j_driver(settings) as driver: driver.verify_connectivity() with driver.session(database=settings.database) as session: if version is not None and graph_version(session) != version: raise IndexUnavailableError('Neo4j graf nezodpovedá aktívnej verzii dát.') - result = search_graph_with_session(session, query, limit=limit, max_entities=max_entities) + result = search_graph_with_session(session, query, limit=limit, max_entities=max_entities, + published_only=published_only) if version is not None: result['index_version'] = version return result @@ -2641,6 +2671,8 @@ def main() -> None: ), ) + parser.add_argument('--include-unpublished', action='store_true', + help='Výslovne zahrnie aj nepublikované dokumenty.') args = parser.parse_args() result = search_graph( @@ -2649,6 +2681,7 @@ def main() -> None: max_entities=( args.max_entities ), + published_only=not args.include_unpublished, ) print( diff --git a/tools/graphrag/graphrag.py b/tools/graphrag/graphrag.py index a3827e7..2fccbf6 100644 --- a/tools/graphrag/graphrag.py +++ b/tools/graphrag/graphrag.py @@ -1,5 +1,7 @@ from __future__ import annotations +from tools.publication import effective_published + import re import unicodedata from collections import defaultdict @@ -1498,12 +1500,7 @@ def build_graph_payload( ) ), "document_kind": kind, - "published": bool( - document.get( - "published", - False, - ) - ), + "published": effective_published(document.get("published")), "categories": ( doc_categories ), @@ -1616,6 +1613,7 @@ def build_graph_payload( ] = { "id": described_topic, "name": title, + "source_document": path, "graph_scope": ( GRAPH_SCOPE ), diff --git a/tools/graphrag/publication.py b/tools/graphrag/publication.py new file mode 100644 index 0000000..1604444 --- /dev/null +++ b/tools/graphrag/publication.py @@ -0,0 +1,44 @@ +"""Publication predicates applied before entity resolution, joins and LIMIT. + +The query wrappers fail at import if their insertion point changes. Filtering +only final citations would still expose hidden facts and incorrect counts. +""" + + +def visible_node(node: str) -> str: + # Identifiers come only from the fixed query definitions, never user input. + return f'''CASE + WHEN {node}:Document THEN coalesce({node}.published, true) + WHEN {node}:Person OR ({node}:Topic AND {node}.source_document IS NOT NULL) THEN EXISTS {{ + MATCH (public_source:Document) + WHERE public_source.graph_scope=$scope AND public_source.path={node}.source_document + AND coalesce(public_source.published, true) + }} + WHEN {node}:Thesis OR {node}:Work THEN EXISTS {{ + MATCH ({node})-[:EVIDENCED_BY]->(public_source:Document) + WHERE public_source.graph_scope=$scope AND coalesce(public_source.published, true) + }} AND ({node}.source_document IS NULL OR EXISTS {{ + MATCH (origin:Document) WHERE origin.graph_scope=$scope + AND origin.path={node}.source_document AND coalesce(origin.published, true) + }}) + ELSE EXISTS {{ + MATCH ({node})--(public_source:Document) + WHERE public_source.graph_scope=$scope AND coalesce(public_source.published, true) + }} END''' + + +def public_query(query: str, *, nodes: tuple[str, ...] = (), documents: tuple[str, ...] = (), + path: bool = False) -> str: + if path: + anchor = 'WHERE all(node IN nodes(p) WHERE node.graph_scope = $scope)' + predicate = f'all(public_node IN nodes(p) WHERE {visible_node("public_node")})' + if query.count(anchor) != 1: + raise ValueError('Missing unique path publication guard position') + return query.replace(anchor, anchor + f' AND (NOT $published_only OR ({predicate}))', 1) + predicates = [visible_node(node) for node in nodes] + predicates.extend(f'coalesce({node}.published, true)' for node in documents) + anchor = '\nWHERE ' + if anchor not in query or not predicates: + raise ValueError('Missing publication guard position') + condition = ' AND '.join(f'({p})' for p in predicates) + return query.replace(anchor, f'\nWHERE (NOT $published_only OR ({condition})) AND ', 1) diff --git a/tools/graphrag/tests/test_graph_publication.py b/tools/graphrag/tests/test_graph_publication.py new file mode 100644 index 0000000..2797327 --- /dev/null +++ b/tools/graphrag/tests/test_graph_publication.py @@ -0,0 +1,101 @@ +"""Publication is enforced by actual Cypher, before aggregation/traversal.""" +import json +import os +import uuid + +import pytest + +from tools.graphrag import graph_search as gs +from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings + +pytestmark = pytest.mark.skipif(os.getenv('RUN_NEO4J_TESTS') != '1', reason='Requires local Neo4j.') + + +@pytest.fixture +def graph(monkeypatch): + scope = 'publication-' + uuid.uuid4().hex + monkeypatch.setattr(gs, 'GRAPH_SCOPE', scope) + settings = get_neo4j_settings() + with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: + tx = session.begin_transaction() + try: + for key, name, published in [('a','Alena Nová',True),('b','Boris Iný',None),('c','Cyril Malý',False)]: + tx.run('''CREATE (p:Person {id:$id, name:$name, graph_scope:$scope, source_document:$path}) + CREATE (d:Document {path:$path, published:$published, graph_scope:$scope, source_url:$url}) + CREATE (t:Thesis {id:$thesis, title:$title, year:2024, thesis_type:'bachelor_thesis', + source_document:$path, graph_scope:$scope}) + MERGE (k:Keyword {id:$tag, name:'sharedcheck', graph_scope:$scope}) + CREATE (p)-[:HAS_DOCUMENT]->(d)-[:HAS_KEYWORD]->(k) + CREATE (p)-[:AUTHORED]->(t)-[:EVIDENCED_BY]->(d)''', + id=scope+key, name=name, scope=scope, path=scope+'/'+key+'.md', published=published, + url='https://example.test/'+key, thesis=scope+'th'+key, title='Výskum '+key,tag=scope+'shared').consume() + tx.run('''MATCH (p:Person {id:$person}), (public:Document {path:$public}) + CREATE (d:Document {path:$path, published:false, source_url:'https://example.test/private', graph_scope:$scope}) + CREATE (k:Keyword {id:$keyword, name:'hiddentag', graph_scope:$scope}) + CREATE (c:Category {name:$category, graph_scope:$scope}) + CREATE (a:DocumentAuthor {id:$author, name:'Hidden Author', graph_scope:$scope}) + CREATE (topic:Topic {id:$topic, name:'Hidden Topic', source_document:$path, graph_scope:$scope}) + CREATE (p)-[:HAS_DOCUMENT]->(d)-[:HAS_KEYWORD]->(k) + CREATE (d)-[:IN_CATEGORY]->(c) CREATE (a)-[:AUTHORED_DOCUMENT]->(d) + CREATE (d)-[:DESCRIBES]->(topic) CREATE (public)-[:DESCRIBES]->(topic)''', + person=scope+'a',public=scope+'/a.md', path=scope+'/private.md',scope=scope, + keyword=scope+'hidden',category=scope+'hidden-category',author=scope+'author',topic=scope+'topic').consume() + yield tx, scope + finally: + tx.rollback() + + +def test_catalog_hides_private_people_and_private_only_vocabulary(graph): + tx, _ = graph + public = gs.run_rows(tx, gs.CATALOG_QUERY, scope=gs.GRAPH_SCOPE, published_only=True) + names = {r.get('name') for r in public} + assert 'Alena Nová' in names and 'Boris Iný' in names + assert not names & {'Cyril Malý','hiddentag','Hidden Author','Hidden Topic'} + all_rows = gs.run_rows(tx, gs.CATALOG_QUERY, scope=gs.GRAPH_SCOPE, published_only=False) + assert {'Cyril Malý','hiddentag','Hidden Author','Hidden Topic'} <= {r.get('name') for r in all_rows} + + +def test_traversal_and_citations_do_not_expose_hidden_documents(graph): + tx, scope = graph + result = gs.search_graph_with_session(tx, 'Alena Nová') + serialized = json.dumps(result, ensure_ascii=False) + assert scope+'/private.md' not in serialized + assert 'Hidden Topic' not in serialized and 'hiddentag' not in serialized + all_rows = gs.search_graph_with_session(tx, 'Alena Nová', published_only=False) + assert scope+'/private.md' in json.dumps(all_rows) + + +def test_global_count_is_filtered_before_counting(graph): + tx, _ = graph + public = gs.search_graph_with_session(tx, 'Koľko osôb má prácu v roku 2024?') + all_rows = gs.search_graph_with_session(tx, 'Koľko osôb má prácu v roku 2024?', published_only=False) + assert public['aggregation']['count'] == 2 + assert set(public['aggregation']['people']) == {'Alena Nová','Boris Iný'} + assert all_rows['aggregation']['count'] == 3 + + +def test_private_person_is_not_resolved_as_a_public_fact(graph): + tx, scope = graph + result = gs.search_graph_with_session(tx, 'Aký názov práce má osoba Cyril Malý v roku 2024?') + assert result['status']=='no_answer' and result['facts']==[] and result['documents']==[] + all_rows = gs.search_graph_with_session(tx, 'Aký názov práce má osoba Cyril Malý v roku 2024?', published_only=False) + assert all_rows['status']=='ok' and any(f.get('person')=='Cyril Malý' for f in all_rows['facts']) + + +def test_private_child_tag_is_not_used_in_conjunction_or_metadata(graph): + tx, _ = graph + result = gs.search_graph_with_session(tx, + 'Aký názov práce má osoba Alena Nová v roku 2024 s kľúčovým slovom hiddentag?') + assert result['status']=='no_answer' and not result['facts'] + all_rows = gs.search_graph_with_session(tx, + 'Aký názov práce má osoba Alena Nová v roku 2024 s kľúčovým slovom hiddentag?', published_only=False) + assert all_rows['status']=='ok' and any(f.get('keyword')=='hiddentag' for f in all_rows['facts']) + + +def test_shared_keyword_does_not_include_private_person(graph): + tx, _ = graph + question = 'Majú osoby Alena Nová a Cyril Malý spoločné kľúčové slovo sharedcheck?' + public = gs.search_graph_with_session(tx, question) + assert public['status']=='no_answer' and not public['facts'] + all_rows = gs.search_graph_with_session(tx, question, published_only=False) + assert any(f['fact_type']=='shared_keyword' for f in all_rows['facts']) diff --git a/tools/graphrag/tests/test_graph_search.py b/tools/graphrag/tests/test_graph_search.py index b385901..53af936 100644 --- a/tools/graphrag/tests/test_graph_search.py +++ b/tools/graphrag/tests/test_graph_search.py @@ -32,6 +32,21 @@ class FakeSession: ) ) + # Existing unit fixtures omit the separate stored-source lookup. Model + # that lookup as present documents; publication enforcement itself is + # exercised by real Neo4j fixtures with explicit publication flags. + if query == graph_search.DOCUMENT_SOURCES_QUERY and query not in self.responses: + urls = {} + for rows in self.responses.values(): + for row in rows: + for prefix in ['', 'left_', 'right_']: + if row.get(prefix+'document_path'): + urls[row[prefix+'document_path']] = row.get(prefix+'source_url') + for node in row.get('nodes', []): + if node.get('path'): + urls[node['path']] = node.get('source_url') + return [{'document_path': path, 'source_url': urls.get(path)} for path in parameters.get('paths', [])] + return self.responses.get( query, [], diff --git a/tools/graphrag/tests/test_graphrag.py b/tools/graphrag/tests/test_graphrag.py index c2e7ec7..cb1ca81 100644 --- a/tools/graphrag/tests/test_graphrag.py +++ b/tools/graphrag/tests/test_graphrag.py @@ -584,6 +584,7 @@ def test_topic_is_created_from_topic_document() -> None: "Nástroje a zdroje pre " "spracovanie prirodzeného jazyka" ), + "source_document": path, "graph_scope": "zpwiki", } ] diff --git a/tools/publication.py b/tools/publication.py new file mode 100644 index 0000000..2a6f841 --- /dev/null +++ b/tools/publication.py @@ -0,0 +1,11 @@ +"""Shared default: only explicit published=false is excluded. + +Missing/NULL publication metadata follows the wiki default and remains visible. +published_only=False is an explicit opt-out for authenticated/admin workflows. +""" +DEFAULT_PUBLISHED_ONLY = True +PUBLICATION_POLICY_VERSION = 'explicit-false-hidden-v1' + + +def effective_published(value: bool | None) -> bool: + return True if value is None else bool(value) diff --git a/tools/rag/rag_document_expansion.py b/tools/rag/rag_document_expansion.py index 1f4e160..aa3a208 100644 --- a/tools/rag/rag_document_expansion.py +++ b/tools/rag/rag_document_expansion.py @@ -361,7 +361,7 @@ def find_exact_student_document( FROM documents WHERE ( ? = 0 - OR published = 1 + OR COALESCE(published, 1) = 1 ) ORDER BY LENGTH(title) DESC, @@ -676,7 +676,7 @@ def load_best_document_section_chunk( WHERE document_path = ? AND ( ? = 0 - OR published = 1 + OR COALESCE(published, 1) = 1 ) ORDER BY chunk_index ASC, diff --git a/tools/rag/rag_query_evidence.py b/tools/rag/rag_query_evidence.py index 6c55028..8647dfd 100644 --- a/tools/rag/rag_query_evidence.py +++ b/tools/rag/rag_query_evidence.py @@ -1,5 +1,7 @@ from __future__ import annotations +from tools.publication import DEFAULT_PUBLISHED_ONLY + import json import re import sqlite3 @@ -704,7 +706,7 @@ def _load_document_chunks( and "published" in columns ): where_parts.append( - "published = 1" + "COALESCE(published, 1) = 1" ) order_parts = [ @@ -766,7 +768,7 @@ def find_top_query_evidence_chunks( document_path: str, query: str, *, - published_only: bool = False, + published_only: bool = DEFAULT_PUBLISHED_ONLY, top_k: int = DEFAULT_TOP_EVIDENCE_CHUNKS, ) -> list[dict[str, Any]]: if top_k <= 0: @@ -898,7 +900,7 @@ def find_best_query_evidence_chunk( document_path: str, query: str, *, - published_only: bool = False, + published_only: bool = DEFAULT_PUBLISHED_ONLY, ) -> dict[str, Any] | None: chunks = _load_document_chunks( conn, @@ -970,7 +972,7 @@ def expand_results_with_query_evidence( dict[str, Any] ], *, - published_only: bool = False, + published_only: bool = DEFAULT_PUBLISHED_ONLY, top_k: int = DEFAULT_TOP_EVIDENCE_CHUNKS, ) -> list[dict[str, Any]]: base_results = [ diff --git a/tools/rag/rag_supplemental_expansion.py b/tools/rag/rag_supplemental_expansion.py index ff0c6d6..3d182e8 100644 --- a/tools/rag/rag_supplemental_expansion.py +++ b/tools/rag/rag_supplemental_expansion.py @@ -1,5 +1,7 @@ from __future__ import annotations +from tools.publication import DEFAULT_PUBLISHED_ONLY + import json import re import sqlite3 @@ -280,7 +282,7 @@ def _load_chunks( where = "" if published_only and "published" in columns: - where = " WHERE published = 1" + where = " WHERE COALESCE(published, 1) = 1" order = " ORDER BY document_path ASC, chunk_index ASC" if "id" in columns: @@ -651,7 +653,7 @@ def expand_results_with_supplemental_sources( query: str, results: list[dict[str, Any]], *, - published_only: bool = False, + published_only: bool = DEFAULT_PUBLISHED_ONLY, limit: int = 5, ) -> list[dict[str, Any]]: base_results = [dict(item) for item in results] diff --git a/tools/rag/rag_utils.py b/tools/rag/rag_utils.py index f65131d..596751f 100644 --- a/tools/rag/rag_utils.py +++ b/tools/rag/rag_utils.py @@ -1,5 +1,7 @@ from __future__ import annotations +from tools.publication import DEFAULT_PUBLISHED_ONLY + import inspect import json import sqlite3 @@ -310,7 +312,7 @@ def load_section_lead_chunk( FROM chunks WHERE document_path = ? AND chunk_index <= ? - AND (? = 0 OR published = 1) + AND (? = 0 OR COALESCE(published, 1) = 1) ORDER BY chunk_index ASC, id ASC """, ( @@ -698,7 +700,7 @@ def build_rag_context( query: str, *, limit: int = 5, - published_only: bool = False, + published_only: bool = DEFAULT_PUBLISHED_ONLY, max_per_document: int = 1, ) -> dict[str, Any]: response = search_database( diff --git a/tools/rag/search_core.py b/tools/rag/search_core.py index 5216a09..7d82d02 100644 --- a/tools/rag/search_core.py +++ b/tools/rag/search_core.py @@ -423,7 +423,7 @@ def run_fts_query( 'bm25({BM25_SQL})' AND ( ? = 0 - OR chunks.published = 1 + OR COALESCE(chunks.published, 1) = 1 ) ORDER BY chunks_fts.rank ASC, @@ -491,7 +491,7 @@ def run_vector_query( = chunk_embeddings.chunk_id WHERE ( ? = 0 - OR chunks.published = 1 + OR COALESCE(chunks.published, 1) = 1 ) """, ( diff --git a/tools/rag/search_db.py b/tools/rag/search_db.py index 58f3337..8ff2e6f 100644 --- a/tools/rag/search_db.py +++ b/tools/rag/search_db.py @@ -136,9 +136,9 @@ def main() -> None: parser.add_argument( "--published-only", action="store_true", + default=True, help=( - "Vyhľadáva iba v dokumentoch " - "s published: true." + "Vylúči explicitné published=false (predvolené)." ), ) @@ -153,6 +153,8 @@ def main() -> None: ), ) + parser.add_argument("--include-unpublished", action="store_false", dest="published_only", + help="Výslovne zahrnie aj nepublikované dokumenty.") args = parser.parse_args() query = " ".join(args.query) diff --git a/tools/rag/search_utils.py b/tools/rag/search_utils.py index 8001759..ca1a7ca 100644 --- a/tools/rag/search_utils.py +++ b/tools/rag/search_utils.py @@ -1,5 +1,7 @@ from __future__ import annotations +from tools.publication import DEFAULT_PUBLISHED_ONLY + import sqlite3 from pathlib import Path from typing import Any @@ -49,7 +51,7 @@ def search_database( db_file: Path, query: str, limit: int = 10, - published_only: bool = False, + published_only: bool = DEFAULT_PUBLISHED_ONLY, max_per_document: int = 3, ) -> dict[str, Any]: if not db_file.exists(): diff --git a/tools/rag/tests/test_rag.py b/tools/rag/tests/test_rag.py index 085d138..09e9ab0 100644 --- a/tools/rag/tests/test_rag.py +++ b/tools/rag/tests/test_rag.py @@ -1156,7 +1156,7 @@ def test_rag_endpoint( assert ( published_only - is False + is True ) assert ( diff --git a/tools/rag/tests/test_search.py b/tools/rag/tests/test_search.py index d49644d..03c1c22 100644 --- a/tools/rag/tests/test_search.py +++ b/tools/rag/tests/test_search.py @@ -254,6 +254,7 @@ def test_prefix_fallback_handles_slovak_inflection( db_file, "hlboke ucenie", limit=5, + published_only=False, ) assert response["strategies"] == [ @@ -309,6 +310,7 @@ def test_published_only_excludes_unpublished_chunks( db_file, "hlboke ucenie", limit=5, + published_only=False, ) public_results = search_database( diff --git a/tools/sync_indexes.py b/tools/sync_indexes.py index 9f454fb..e8d4615 100644 --- a/tools/sync_indexes.py +++ b/tools/sync_indexes.py @@ -25,6 +25,7 @@ from tools.rag.scan_zpwiki import scan_pages from tools.rag.build_chunks import build_chunks from tools.rag.build_sqlite_index import build_database from tools.rag.embedding_utils import embedding_model_name +from tools.publication import PUBLICATION_POLICY_VERSION from tools.graphrag.graphrag import build_graph_payload from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings from tools.graphrag.build_graphrag import (create_constraints, graph_version, @@ -68,7 +69,7 @@ def prepare_snapshot(wiki_root: Path, stage: Path) -> tuple[dict, dict, dict]: 'created_at_utc': datetime.now(timezone.utc).isoformat(), 'source_sha256': hashlib.sha256(json.dumps(sources, sort_keys=True).encode()).hexdigest(), 'documents_sha256': digest(stage / 'documents.json'), 'chunks_sha256': digest(stage / 'chunks.json'), - 'embedding_model': embedding_model_name()} + 'embedding_model': embedding_model_name(), 'publication_policy': PUBLICATION_POLICY_VERSION} with closing(sqlite3.connect(stage / 'zp_index.sqlite')) as conn, conn: conn.execute('CREATE TABLE index_state (id INTEGER PRIMARY KEY CHECK(id=1), metadata_json TEXT NOT NULL)') conn.execute('INSERT INTO index_state VALUES (1, ?)', (json.dumps(version, sort_keys=True),)) diff --git a/tools/tool_health.py b/tools/tool_health.py new file mode 100644 index 0000000..d67a283 --- /dev/null +++ b/tools/tool_health.py @@ -0,0 +1,89 @@ +"""Independent, sanitized readiness results for the two retrieval backends.""" +from __future__ import annotations + +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +from neo4j import Query + +from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings +from tools.graphrag.graphrag import GRAPH_SCOPE +from tools.index_state import IndexUnavailableError, index_snapshot, sqlite_version +from tools.publication import PUBLICATION_POLICY_VERSION +from tools.rag.search_core import verify_search_schema, embedding_index_info + +PROBE_TIMEOUT_SECONDS = 2.0 + + +def readiness(ready: bool, reason: str | None = None, **details) -> dict: + return {'ready': ready, 'status': 'ready' if ready else 'unavailable', 'reason': reason, **details} + + +def rag_readiness(db_file: Path, version: dict | None, *, embedding_ready: bool) -> dict: + if not db_file.is_file(): + return readiness(False, 'index_missing') + try: + if version is not None and sqlite_version(db_file) != version: + return readiness(False, 'version_mismatch') + with closing(sqlite3.connect(db_file.resolve().as_uri()+'?mode=ro', uri=True, timeout=1)) as conn: + conn.row_factory = sqlite3.Row + verify_search_schema(conn) + model, dimensions = embedding_index_info(conn) + conn.execute('SELECT rowid FROM chunks_fts LIMIT 1').fetchone() + if not embedding_ready: + return readiness(False, 'embedding_not_ready', embedding_model=model) + return readiness(True, embedding_model=model, dimensions=dimensions, index_version=version) + except (sqlite3.Error, RuntimeError, ValueError, OSError): + return readiness(False, 'index_invalid') + + +def neo4j_readiness(version: dict | None) -> tuple[dict, dict]: + try: + settings = get_neo4j_settings() + except RuntimeError: + return readiness(False, 'not_configured'), readiness(False, 'neo4j_not_configured') + try: + with create_neo4j_driver(settings, connection_timeout=PROBE_TIMEOUT_SECONDS, + connection_acquisition_timeout=PROBE_TIMEOUT_SECONDS, + connection_write_timeout=PROBE_TIMEOUT_SECONDS, + max_transaction_retry_time=0, max_connection_pool_size=1) as driver: + driver.verify_connectivity() + with driver.session(database=settings.database) as session: + record = session.run(Query('''MATCH (n) WHERE n.graph_scope=$scope + WITH count(n) AS nodes + OPTIONAL MATCH (state:IndexState {id:$scope}) + RETURN nodes, state.metadata_json AS metadata''', timeout=PROBE_TIMEOUT_SECONDS), + scope=GRAPH_SCOPE).single() + except Exception: + # Credentials, connection strings and raw driver errors stay server-side. + return readiness(False, 'unreachable'), readiness(False, 'neo4j_unavailable') + neo4j = readiness(True) + if not record or record['nodes'] == 0: + return neo4j, readiness(False, 'graph_empty') + try: + actual = json.loads(record['metadata']) if record['metadata'] else None + except (ValueError, TypeError): + return neo4j, readiness(False, 'graph_version_invalid') + if version is not None and actual != version: + return neo4j, readiness(False, 'version_mismatch') + if version is None or version.get('publication_policy') != PUBLICATION_POLICY_VERSION: + return neo4j, readiness(False, 'publication_sync_required') + return neo4j, readiness(True, index_version=actual, nodes=record['nodes']) + + +def tools_readiness(db_file: Path, *, security_configured: bool, embedding_ready: bool) -> tuple[dict, dict | None]: + version = None + try: + with index_snapshot(db_file.parent) as version: + rag = rag_readiness(db_file, version, embedding_ready=embedding_ready) + neo4j, graph = neo4j_readiness(version) + except IndexUnavailableError: + neo4j, _ = neo4j_readiness(None) + rag = readiness(False, 'snapshot_unavailable') + graph = readiness(False, 'snapshot_unavailable') + if not security_configured: + rag = readiness(False, 'api_not_configured') + graph = readiness(False, 'api_not_configured') + return {'rag': rag, 'neo4j': neo4j, 'graphrag': graph}, version