jp170na/dp-zp-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publikovanie a dostupnosť

Browse Source
This commit is contained in:
Ján Pták 2026-10-25 16:09:54 +01:00
parent 86a155691d
commit fee6ffc70f
25 changed files with 803 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@ -695,6 +695,18 @@ vyvolá rollback. Pri nedokončenom prepnutí po páde zostane vyhľadávanie
nedostupné do úspešného opakovania synchronizácie. Podrobnosti a zotavenie: nedostupné do úspešného opakovania synchronizácie. Podrobnosti a zotavenie:
[Spoločná synchronizácia indexov](docs/synchronizacia_indexov.md). [Spoločná synchronizácia indexov](docs/synchronizacia_indexov.md).
## Publikovanie a dostupnosť nástrojov
Oba nástroje predvolene vylučujú iba explicitné `published=false`.
Chýbajúca hodnota sa považuje za publikovanú. Parameter
`published_only=false` výslovne zahrnie aj nepublikované dokumenty pri
autorizovanom volaní; CLI používa `--include-unpublished`.
`/health` uvádza samostatne `tools.rag`, `tools.neo4j` a `tools.graphrag`.
Globálne `ready` vyžaduje oba vyhľadávacie nástroje; pri výpadku Neo4j
môže klient ďalej používať RAG podľa `tools.rag.ready`.
[Politika, migrácia a príklady](docs/publikovanie_a_dostupnost.md).
## Hybridné vyhľadávanie ## Hybridné vyhľadávanie
Vyhľadávanie kombinuje: Vyhľadávanie kombinuje:
@ -747,7 +759,7 @@ curl -X POST http://127.0.0.1:8000/search \
-d '{ -d '{
"query": "strojový preklad", "query": "strojový preklad",
"limit": 3, "limit": 3,
"published_only": false, "published_only": true,
"max_per_document": 1 "max_per_document": 1
}' }'
``` ```
@ -798,7 +810,7 @@ curl -X POST http://127.0.0.1:8000/rag \
-d '{ -d '{
"query": "V akom roku robil Ján Holp diplomovú prácu?", "query": "V akom roku robil Ján Holp diplomovú prácu?",
"limit": 5, "limit": 5,
"published_only": false, "published_only": true,
"max_per_document": 1 "max_per_document": 1
}' }'
``` ```

4
app/main.py
View File

@ -41,15 +41,17 @@ OPENWEBUI_ORIGIN = (
@asynccontextmanager @asynccontextmanager
async def lifespan( async def lifespan(
_: FastAPI, application: FastAPI,
): ):
validate_security_configuration() validate_security_configuration()
application.state.embedding_ready = False
try: try:
await asyncio.to_thread( await asyncio.to_thread(
embed_query, embed_query,
"warmup", "warmup",
) )
application.state.embedding_ready = True
except Exception: except Exception:
logger.exception( logger.exception(

38
app/routes.py
View File

@ -46,7 +46,8 @@ from tools.rag.rebuild_index import (
rebuild_index, rebuild_index,
) )
from tools.rag.search_utils import search_database from tools.rag.search_utils import search_database
from tools.index_state import index_snapshot, IndexUnavailableError from tools.publication import DEFAULT_PUBLISHED_ONLY
from tools.tool_health import tools_readiness
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -104,7 +105,7 @@ class SearchRequest(
le=50, le=50,
) )
published_only: bool = False published_only: bool = DEFAULT_PUBLISHED_ONLY
max_per_document: int = Field( max_per_document: int = Field(
default=1, default=1,
@ -137,10 +138,10 @@ class RagRequest(
) )
published_only: bool = Field( published_only: bool = Field(
default=False, default=DEFAULT_PUBLISHED_ONLY,
description=( description=(
"Ak je true, použijú sa iba " "Ak je true, použijú sa iba "
"publikované dokumenty." "dokumenty okrem explicitného published=false; chýbajúca hodnota sa zahrnie."
), ),
) )
@ -184,6 +185,7 @@ class HealthResponse(BaseModel):
security_configured: bool security_configured: bool
index_version: dict[str, Any] | None = None index_version: dict[str, Any] | None = None
tools: dict[str, Any]
class RagSource(BaseModel): class RagSource(BaseModel):
@ -515,7 +517,7 @@ async def read_webhook_body(
include_in_schema=False, include_in_schema=False,
response_model=HealthResponse, response_model=HealthResponse,
) )
def health() -> dict[str, Any]: def health(request: Request) -> dict[str, Any]:
database_exists = ( database_exists = (
DB_FILE.exists() DB_FILE.exists()
) )
@ -532,22 +534,15 @@ def health() -> dict[str, Any]:
except RuntimeError: except RuntimeError:
security_configured = False security_configured = False
ready = ( tools, version = tools_readiness(DB_FILE, security_configured=security_configured,
database_exists embedding_ready=getattr(request.app.state, 'embedding_ready', False))
and zpwiki_exists ready = tools['rag']['ready'] and tools['graphrag']['ready']
and security_configured
)
version = None
try:
with index_snapshot(DB_FILE.parent, db_file=DB_FILE) as version:
pass
except (IndexUnavailableError, sqlite3.Error):
ready = False
return { return {
"status": "ok", "status": "ok",
"ready": ready, "ready": ready,
"index_version": version, "index_version": version,
"tools": tools,
"database_exists": ( "database_exists": (
database_exists database_exists
), ),
@ -623,9 +618,10 @@ def health() -> dict[str, Any]:
) )
def rag( def rag(
request: RagRequest, request: RagRequest,
http_request: Request,
) -> dict[str, Any]: ) -> dict[str, Any]:
try: try:
return build_rag_context( result = build_rag_context(
DB_FILE, DB_FILE,
request.query, request.query,
limit=request.limit, limit=request.limit,
@ -636,6 +632,8 @@ def rag(
request.max_per_document request.max_per_document
), ),
) )
http_request.app.state.embedding_ready = True
return result
except Exception as error: except Exception as error:
raise_retrieval_error( raise_retrieval_error(
@ -655,6 +653,7 @@ def rag(
) )
def search( def search(
request: SearchRequest, request: SearchRequest,
http_request: Request,
) -> dict[str, Any]: ) -> dict[str, Any]:
try: try:
response = search_database( response = search_database(
@ -668,6 +667,7 @@ def search(
request.max_per_document request.max_per_document
), ),
) )
http_request.app.state.embedding_ready = True
except Exception as error: except Exception as error:
raise_retrieval_error( raise_retrieval_error(
@ -690,6 +690,7 @@ def search(
results results
), ),
"results": results, "results": results,
**({'index_version': response['index_version']} if response.get('index_version') else {}),
} }
@ -705,6 +706,7 @@ def search(
) )
def sync( def sync(
request: SyncRequest, request: SyncRequest,
http_request: Request,
) -> dict[str, Any]: ) -> dict[str, Any]:
try: try:
result = rebuild_index( result = rebuild_index(
@ -712,6 +714,7 @@ def sync(
request.pull_git request.pull_git
) )
) )
http_request.app.state.embedding_ready = True
except Exception as error: except Exception as error:
raise_reindex_error( raise_reindex_error(
@ -890,6 +893,7 @@ async def gitea_webhook(
webhook_should_pull_git() webhook_should_pull_git()
), ),
) )
request.app.state.embedding_ready = True
except Exception as error: except Exception as error:
raise_reindex_error( raise_reindex_error(

102
docs/publikovanie_a_dostupnost.md Normal file
View File

@ -0,0 +1,102 @@
# Publikovanie dokumentov a dostupnosť nástrojov
## Jednotná politika
Používateľ zvolil zahrnutie dokumentov bez `published`. RAG aj GraphRAG preto
predvolene používajú `published_only=true` s týmto významom:
| Hodnota zdroja | Predvolené vyhľadávanie |
|---|---|
| `published: true` | Zahrnúť |
| Chýbajúce `published` alebo normalizované `null` | Zahrnúť |
| `published: false` | Vylúčiť |
Chýbajúca hodnota zodpovedá predvolenému významu príznaku v
[Grav](https://learn.getgrav.org/2/content/headers). Táto politika určuje
výber podľa `published`; API naďalej používa svoju existujúcu autentifikáciu.
V aktuálnom korpuse nie sú nastavené `publish_date` ani `unpublish_date`.
SQL používa `COALESCE(published, 1) = 1`, graf efektívnu publikačnú hodnotu.
Zdrojový JSON a SQLite zachovávajú pôvodné `null`; grafový builder ho
normalizuje na efektívne `true`. Všetkých 16 pôvodne neoznačených dokumentov
je teda dostupných v oboch vetvách. Jeden explicitne nepublikovaný dokument
sa predvolene vylúči.
Pri RAG sa filter aplikuje na FTS, embeddingy aj všetky rozšírenia kontextu.
Pri grafe platí pre katalóg entít, fakty, spoločné vlastnosti, počítanie,
prechádzanie grafu a zoznam zdrojov. Entity doložené iba skrytými dokumentmi
sa nezaradia do verejného katalógu. Pôvod názvu témy a canonical profilu
osoby sa kontroluje podľa príslušného zdrojového dokumentu.
Výslovné zahrnutie nepublikovaných dokumentov pre autorizované volanie:
```json
{"query": "vyhľadávací dotaz", "published_only": false}
```
Python funkcie používajú rovnaký parameter `published_only=False`.
Oba vyhľadávacie CLI podporujú `--include-unpublished`. Pred pripojením
nástrojov do OpenWebUI sa používa aktuálna OpenAPI schéma s novým defaultom.
## Migrácia uloženého grafu
Po nasadení sa vykoná spoločná synchronizácia. Tá opraví staré zamenenie
chýbajúcej hodnoty s `false` a doplní pôvod názvu témy. Verzia indexov obsahuje
`publication_policy: explicit-false-hidden-v1`. Pred migráciou predvolené
grafové vyhľadávanie vyžaduje synchronizáciu a health uvádza
`publication_sync_required`. Existujúce historické merania sa neprepisujú;
časť klasického baseline používala aj nepublikované dokumenty.
## `/health`
HTTP 200 a `status=ok` označujú živý API proces. `ready` znamená pripravenosť
oboch vyhľadávacích nástrojov. Klient/agenta pri výbere konkrétneho nástroja
zaujímajú samostatné položky `tools`:
- `tools.rag`: čitateľný SQLite/FTS5/embedding index, zhoda verzie a známa
úspešná inicializácia embeddingového modelu. Úspešné vyhľadanie alebo sync
obnoví stav modelu po prípadnom zlyhaní inicializácie.
- `tools.neo4j`: dostupné spojenie a vykonanie read-only sondy v databáze.
Sonda používa krátke časové limity a nevykonáva automatické opakovania transakcií.
- `tools.graphrag`: dostupné Neo4j, neprázdny graf, zhodná generácia dát
a migrovaná publikačná politika.
Výpadok Neo4j nespôsobí nedostupnosť RAG. Neplatný SQLite index môže vyradiť
RAG, pričom graf ostane dostupný. Počas nejasného prepnutia generácie sa
vyhľadávacie nástroje označia ako nedostupné; stav samotného Neo4j sa uvádza
samostatne. Odpovede neobsahujú prihlasovacie údaje ani surové chyby drivera.
Príklad výpadku Neo4j, skrátený na rozhodujúce polia:
```json
{
"status": "ok",
"ready": false,
"tools": {
"rag": {"ready": true, "status": "ready", "reason": null},
"neo4j": {"ready": false, "status": "unavailable", "reason": "unreachable"},
"graphrag": {"ready": false, "status": "unavailable", "reason": "neo4j_unavailable"}
}
}
```
Ide o dostupnosť retrieval nástrojov. Health neposiela otázky školskému LLM
a nemeria dostupnosť externého generatívneho modelu ani presnosť odpovedí.
## Overenie nasadenia 25. 10. 2026
- Kompletná regresná sada: **485 úspešných testov, 4 preskočené**. Zahŕňa
skutočné Cypher testy filtrovania, agregácie a grafových prechodov nad
izolovaným Neo4j grafom s rollbackom.
- Docker image bol znovu zostavený a API nasadené. Autentifikovaný `/sync`
bez Git pullu obnovil 114 dokumentov, 887 chunkov a graf 447 uzlov/879 vzťahov.
- Reálne `/search`, `/rag` a `search_graph()` overili predvolené vylúčenie
skrytého dokumentu aj jeho zahrnutie pri `published_only=false`.
- HTTP `/health` hlási všetky tri stavy pripravené; SQLite, súborová verzia
a Neo4j používajú rovnakú novú generáciu dát.
- Sonda s nefunkčnou adresou Neo4j potvrdila dostupnosť RAG pri nedostupnom
grafe. Sonda s chýbajúcim SQLite indexom potvrdila dostupnosť grafu pri
nedostupnom RAG. Produkčné služby sa pri týchto sondách nevypínali.
Strojovo čitateľný výsledok a identifikácia dát:
[publikovanie_a_dostupnost_overenie.json](publikovanie_a_dostupnost_overenie.json).

162
docs/publikovanie_a_dostupnost_overenie.json Normal file
View File

@ -0,0 +1,162 @@
{
"status": "ok",
"index_version": {
"version": "ff682d98134d406081040902f234c44f",
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
"embedding_model": "intfloat/multilingual-e5-small",
"publication_policy": "explicit-false-hidden-v1"
},
"health": {
"status": "ok",
"ready": true,
"database_exists": true,
"search_engine": "hybrid_fts5_embeddings",
"rag_enabled": true,
"zpwiki_exists": true,
"security_configured": true,
"index_version": {
"version": "ff682d98134d406081040902f234c44f",
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
"embedding_model": "intfloat/multilingual-e5-small",
"publication_policy": "explicit-false-hidden-v1"
},
"tools": {
"rag": {
"ready": true,
"status": "ready",
"reason": null,
"embedding_model": "intfloat/multilingual-e5-small",
"dimensions": 384,
"index_version": {
"version": "ff682d98134d406081040902f234c44f",
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
"embedding_model": "intfloat/multilingual-e5-small",
"publication_policy": "explicit-false-hidden-v1"
}
},
"neo4j": {
"ready": true,
"status": "ready",
"reason": null
},
"graphrag": {
"ready": true,
"status": "ready",
"reason": null,
"index_version": {
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
"embedding_model": "intfloat/multilingual-e5-small",
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
"publication_policy": "explicit-false-hidden-v1",
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
"version": "ff682d98134d406081040902f234c44f"
},
"nodes": 447
}
}
},
"publication_counts": {
"published_true": 97,
"missing": 16,
"published_false": 1,
"default_visible": 113
},
"graph_document_counts": {
"True": 113,
"False": 1
},
"live_search_checks": {
"/search": {
"default_hides_unpublished": true,
"explicit_opt_out_includes_it": true
},
"/rag": {
"default_hides_unpublished": true,
"explicit_opt_out_includes_it": true
},
"graphrag": {
"default_hides_unpublished": true,
"explicit_opt_out_includes_it": true,
"default_status": "ok"
}
},
"neo4j_outage_probe": {
"rag": {
"ready": true,
"status": "ready",
"reason": null,
"embedding_model": "intfloat/multilingual-e5-small",
"dimensions": 384,
"index_version": {
"version": "ff682d98134d406081040902f234c44f",
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
"embedding_model": "intfloat/multilingual-e5-small",
"publication_policy": "explicit-false-hidden-v1"
}
},
"neo4j": {
"ready": false,
"status": "unavailable",
"reason": "unreachable"
},
"graphrag": {
"ready": false,
"status": "unavailable",
"reason": "neo4j_unavailable"
}
},
"rag_index_outage_probe": {
"rag": {
"ready": false,
"status": "unavailable",
"reason": "index_missing"
},
"neo4j": {
"ready": true,
"status": "ready",
"reason": null
},
"graphrag": {
"ready": true,
"status": "ready",
"reason": null,
"index_version": {
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
"embedding_model": "intfloat/multilingual-e5-small",
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
"publication_policy": "explicit-false-hidden-v1",
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
"version": "ff682d98134d406081040902f234c44f"
},
"nodes": 447
}
},
"checked_at_utc": "2026-10-25T15:09:42.459584+00:00",
"regression": {
"passed": 485,
"skipped": 4,
"failed": 0,
"includes_real_neo4j_publication_tests": true
},
"scope": "Publikačný filter a dostupnosť retrieval nástrojov; nejde o nové meranie presnosti LLM."
}

166
test/test_publication_health.py Normal file
View File

@ -0,0 +1,166 @@
import json
import sqlite3
from types import SimpleNamespace
import numpy as np
import pytest
from fastapi.testclient import TestClient
import app.main as main
import app.routes as routes
import tools.tool_health as health
import tools.rag.build_sqlite_index as indexer
import tools.rag.search_core as search_core
from tools.index_state import durable_json, PENDING_FILE, VERSION_FILE
from tools.publication import PUBLICATION_POLICY_VERSION, effective_published
from tools.rag.search_utils import search_database
from tools.rag.rag_utils import build_rag_context
@pytest.fixture
def corpus(tmp_path, monkeypatch):
docs = []; chunks = []
paths = ['pages/students/2020/eva/README.md', 'pages/topics/missing/README.md',
'pages/topics/hidden/README.md', 'pages/students/2020/eva/private/README.md']
for index, (path, published) in enumerate(zip(paths, [True, None, False, False])):
title = ['Verejný profil', 'Chýbajúca značka', 'Skrytý dokument', 'Skrytá poznámka'][index]
text = 'spoločný overovací výraz ' + ('SKRYTY_OVEROVACI_OBSAH' if published is False else title)
docs.append({'path': path, 'title': title, 'published': published})
chunks.append({'chunk_id': path+'::0', 'document_path': path, 'title': title,
'published': published, 'chunk_index': 0, 'heading_paths': [['Úvod']],
'text': text, 'text_length': len(text), 'token_count': 10,
'content_hash': str(index), 'tags': [], 'categories': []})
data = tmp_path / 'data'; data.mkdir()
for name, rows in [('documents.json', docs), ('chunks.json', chunks)]:
(data/name).write_text(json.dumps(rows))
monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.ones((len(texts), 4), dtype=np.float32))
monkeypatch.setattr(search_core, 'embed_query', lambda *args, **kwargs: np.ones(4, dtype=np.float32))
db = data / 'zp_index.sqlite'
indexer.build_database(documents_file=data/'documents.json', chunks_file=data/'chunks.json', db_file=db)
version = {'version': 'test-generation', 'publication_policy': PUBLICATION_POLICY_VERSION}
with sqlite3.connect(db) as conn:
conn.execute('CREATE TABLE index_state (id INTEGER PRIMARY KEY, metadata_json TEXT)')
conn.execute('INSERT INTO index_state VALUES (1,?)', (json.dumps(version),))
durable_json(data/VERSION_FILE, version)
monkeypatch.setattr(routes, 'DB_FILE', db)
monkeypatch.setattr(main, 'embed_query', lambda *args: None)
return db, version, paths
@pytest.mark.parametrize('value,visible', [(True, True), (None, True), (False, False)])
def test_publication_policy(value, visible):
assert effective_published(value) is visible
def test_rag_default_filters_both_fts_and_vectors_but_keeps_null(corpus):
db, _, paths = corpus
result = search_database(db, 'spoločný overovací výraz', limit=10)
assert {r['document_path'] for r in result['results']} == set(paths[:2])
assert any(r['published'] is None for r in result['results'])
all_rows = search_database(db, 'spoločný overovací výraz', limit=10, published_only=False)
assert {r['document_path'] for r in all_rows['results']} == set(paths)
@pytest.mark.parametrize('query', ['spoločný overovací výraz', 'Skrytý dokument', 'Skrytá poznámka'])
def test_rag_expansions_do_not_reintroduce_hidden_text(corpus, query):
db, _, paths = corpus
result = build_rag_context(db, query, limit=10)
assert all(s['document_path'] not in paths[2:] for s in result['sources'])
assert 'SKRYTY_OVEROVACI_OBSAH' not in result['context']
all_rows = build_rag_context(db, query, limit=10, published_only=False)
assert 'SKRYTY_OVEROVACI_OBSAH' in all_rows['context']
def healthy_graph(version):
return health.readiness(True), health.readiness(True, index_version=version, nodes=2)
def test_neo4j_outage_does_not_disable_rag(corpus, monkeypatch):
_, _, paths = corpus
monkeypatch.setattr(health, 'neo4j_readiness', lambda v: (
health.readiness(False, 'unreachable'), health.readiness(False, 'neo4j_unavailable')))
with TestClient(main.app) as client:
payload = client.get('/health').json()
assert payload['tools']['rag']['ready'] is True
assert payload['tools']['neo4j']['ready'] is False
assert payload['tools']['graphrag']['ready'] is False
assert payload['ready'] is False
response = client.post('/rag', json={'query': 'spoločný overovací výraz'},
headers={'Authorization': 'Bearer '+'a'*64})
assert response.status_code == 200
assert {s['document_path'] for s in response.json()['sources']} == set(paths[:2])
def test_rag_index_outage_does_not_disable_graph(corpus, monkeypatch):
db, version, _ = corpus
monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph)
db.unlink()
with TestClient(main.app) as client:
payload = client.get('/health').json()
assert payload['tools']['rag']['ready'] is False
assert payload['tools']['neo4j']['ready'] is True
assert payload['tools']['graphrag']['ready'] is True
assert payload['ready'] is False
def test_pending_sync_blocks_both_tools_but_reports_neo4j_service(corpus, monkeypatch):
db, _, _ = corpus
durable_json(db.parent/PENDING_FILE, {'target_version': 'next'})
monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph)
with TestClient(main.app) as client:
tools = client.get('/health').json()['tools']
assert tools['rag']['reason'] == tools['graphrag']['reason'] == 'snapshot_unavailable'
assert tools['neo4j']['ready'] is True
def test_embedding_startup_failure_and_successful_query_recovery(corpus, monkeypatch):
monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph)
def broken(*args):
raise RuntimeError('transient warmup failure')
monkeypatch.setattr(main, 'embed_query', broken)
with TestClient(main.app) as client:
tools = client.get('/health').json()['tools']
assert tools['rag']['reason'] == 'embedding_not_ready'
assert tools['graphrag']['ready'] is True
response = client.post('/rag', json={'query': 'spoločný overovací výraz'},
headers={'Authorization': 'Bearer '+'a'*64})
assert response.status_code == 200
assert client.get('/health').json()['tools']['rag']['ready'] is True
def test_neo4j_probe_is_bounded_and_does_not_expose_errors(monkeypatch):
monkeypatch.setattr(health, 'get_neo4j_settings', lambda: SimpleNamespace(database='neo4j'))
captured = {}
def fail(settings, **options):
captured.update(options)
raise RuntimeError('password=never-return-this')
monkeypatch.setattr(health, 'create_neo4j_driver', fail)
result = health.neo4j_readiness(None)
assert not result[0]['ready'] and not result[1]['ready']
assert captured['connection_timeout'] == captured['connection_acquisition_timeout'] == 2.0
assert captured['max_transaction_retry_time'] == 0
assert 'never-return-this' not in json.dumps(result)
@pytest.mark.parametrize('actual,nodes,reason', [
({'version': 'different'}, 2, 'version_mismatch'),
(None, 0, 'graph_empty'),
({'version': 'old'}, 2, 'publication_sync_required'),
])
def test_connection_readiness_is_distinct_from_graph_readiness(monkeypatch, actual, nodes, reason):
class Result:
def single(self):
return {'nodes': nodes, 'metadata': json.dumps(actual) if actual else None}
class Connection:
def __enter__(self): return self
def __exit__(self, *args): pass
def verify_connectivity(self): pass
def session(self, **kwargs): return self
def run(self, query, **params):
assert query.timeout == 2.0
return Result()
monkeypatch.setattr(health, 'get_neo4j_settings', lambda: SimpleNamespace(database='neo4j'))
monkeypatch.setattr(health, 'create_neo4j_driver', lambda *args, **kwargs: Connection())
expected = {'version': 'expected'} if reason=='version_mismatch' else actual
neo, graph = health.neo4j_readiness(expected)
assert neo['ready'] is True and graph['ready'] is False and graph['reason']==reason

2
tools/graphrag/graph_db.py
View File

@ -39,6 +39,7 @@ def get_neo4j_settings() -> Neo4jSettings:
def create_neo4j_driver( def create_neo4j_driver(
settings: Neo4jSettings | None = None, settings: Neo4jSettings | None = None,
**driver_options,
): ):
resolved = settings or get_neo4j_settings() resolved = settings or get_neo4j_settings()
@ -48,6 +49,7 @@ def create_neo4j_driver(
resolved.user, resolved.user,
resolved.password, resolved.password,
), ),
**driver_options,
) )

43
tools/graphrag/graph_search.py
View File

@ -5,6 +5,8 @@ import json
import re import re
import unicodedata import unicodedata
from typing import Any, Iterable from typing import Any, Iterable
from tools.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION
from .publication import public_query
GRAPH_SCOPE = "zpwiki" GRAPH_SCOPE = "zpwiki"
@ -2286,6 +2288,23 @@ RETURN document.path AS document_path, document.source_url AS source_url
ORDER BY document_path ORDER BY document_path
""" """
# Restrict the complete retrieval pipeline, including vocabulary and traversal.
CATALOG_QUERY = public_query(CATALOG_QUERY, nodes=('n',))
NEIGHBORHOOD_QUERY = public_query(NEIGHBORHOOD_QUERY, path=True)
PAIR_PATH_QUERY = public_query(PAIR_PATH_QUERY, path=True)
THESIS_FACTS_QUERY = public_query(THESIS_FACTS_QUERY, nodes=('person',), documents=('document',))
WORK_FACTS_QUERY = public_query(WORK_FACTS_QUERY, nodes=('person',), documents=('document',))
THESIS_PATHS_QUERY = public_query(THESIS_PATHS_QUERY, nodes=('person',), documents=('document',))
WORK_PATHS_QUERY = public_query(WORK_PATHS_QUERY, nodes=('person',), documents=('document',))
START_YEAR_FACTS_QUERY = public_query(START_YEAR_FACTS_QUERY, nodes=('person',))
PERSON_KEYWORD_FACTS_QUERY = public_query(PERSON_KEYWORD_FACTS_QUERY, nodes=('person',), documents=('document',))
PERSON_CATEGORY_FACTS_QUERY = public_query(PERSON_CATEGORY_FACTS_QUERY, nodes=('person',), documents=('document',))
PERSON_DOCUMENT_AUTHOR_FACTS_QUERY = public_query(PERSON_DOCUMENT_AUTHOR_FACTS_QUERY, nodes=('person',), documents=('document',))
SHARED_KEYWORD_FACTS_QUERY = public_query(SHARED_KEYWORD_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document'))
SHARED_CATEGORY_FACTS_QUERY = public_query(SHARED_CATEGORY_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document'))
SHARED_DOCUMENT_AUTHOR_FACTS_QUERY = public_query(SHARED_DOCUMENT_AUTHOR_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document'))
DOCUMENT_SOURCES_QUERY = public_query(DOCUMENT_SOURCES_QUERY, documents=('document',))
def search_graph_with_session( def search_graph_with_session(
session: Any, session: Any,
@ -2293,6 +2312,7 @@ def search_graph_with_session(
*, *,
limit: int = DEFAULT_LIMIT, limit: int = DEFAULT_LIMIT,
max_entities: int = DEFAULT_MAX_ENTITIES, max_entities: int = DEFAULT_MAX_ENTITIES,
published_only: bool = DEFAULT_PUBLISHED_ONLY,
) -> dict[str, Any]: ) -> dict[str, Any]:
clean_query = str(query or "").strip() clean_query = str(query or "").strip()
if not clean_query: if not clean_query:
@ -2303,7 +2323,8 @@ def search_graph_with_session(
raise ValueError("max_entities musí byť aspoň 1") raise ValueError("max_entities musí byť aspoň 1")
constraints = extract_query_constraints(clean_query) constraints = extract_query_constraints(clean_query)
catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE) catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE, published_only=published_only)
constraints['published_only'] = published_only
# Explicit tag filters are resolved separately. Incidental names such as the # Explicit tag filters are resolved separately. Incidental names such as the
# tag 'zp' in 'ZP Wiki' must never become additional query constraints. # tag 'zp' in 'ZP Wiki' must never become additional query constraints.
phrase = constraints["keyword_phrase"] phrase = constraints["keyword_phrase"]
@ -2425,7 +2446,8 @@ def search_graph_with_session(
# the presentation limit. The catalog bounds the number of academic nodes; # the presentation limit. The catalog bounds the number of academic nodes;
# relational lookups use a conservative bound covering the managed graph. # relational lookups use a conservative bound covering the managed graph.
fetch_limit = max(limit, len(catalog) ** 2) fetch_limit = max(limit, len(catalog) ** 2)
common = {"scope": GRAPH_SCOPE, "person_ids": person_ids, "limit": fetch_limit} common = {"scope": GRAPH_SCOPE, "person_ids": person_ids, "limit": fetch_limit,
"published_only": published_only}
blocked_reasons = [] blocked_reasons = []
if constraints["unresolved_keyword"]: if constraints["unresolved_keyword"]:
blocked_reasons.append("unknown_keyword") blocked_reasons.append("unknown_keyword")
@ -2507,7 +2529,8 @@ def search_graph_with_session(
)) ))
if not structured and resolved_entities and not blocked: if not structured and resolved_entities and not blocked:
start_ids = [str(row["element_id"]) for row in resolved_entities] start_ids = [str(row["element_id"]) for row in resolved_entities]
params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit} params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit,
"published_only": published_only}
if constraints["mentions_shared"] and len(start_ids) >= 2: if constraints["mentions_shared"] and len(start_ids) >= 2:
paths.extend(run_rows(session, PAIR_PATH_QUERY, **params)) paths.extend(run_rows(session, PAIR_PATH_QUERY, **params))
else: else:
@ -2543,8 +2566,11 @@ def search_graph_with_session(
# actual stored URLs rather than manufacturing URLs from expected answers. # actual stored URLs rather than manufacturing URLs from expected answers.
documents = collect_documents(resolved_entities, paths, facts) documents = collect_documents(resolved_entities, paths, facts)
sources = run_rows(session, DOCUMENT_SOURCES_QUERY, scope=GRAPH_SCOPE, sources = run_rows(session, DOCUMENT_SOURCES_QUERY, scope=GRAPH_SCOPE,
paths=[row["document_path"] for row in documents]) if documents else [] paths=[row["document_path"] for row in documents],
published_only=published_only) if documents else []
by_path = {row["document_path"]: row.get("source_url") for row in sources} by_path = {row["document_path"]: row.get("source_url") for row in sources}
if published_only:
documents = [row for row in documents if row['document_path'] in by_path]
for document in documents: for document in documents:
if not document.get("source_url"): if not document.get("source_url"):
document["source_url"] = by_path.get(document["document_path"]) document["source_url"] = by_path.get(document["document_path"])
@ -2594,6 +2620,7 @@ def search_graph(
max_entities: int = ( max_entities: int = (
DEFAULT_MAX_ENTITIES DEFAULT_MAX_ENTITIES
), ),
published_only: bool = DEFAULT_PUBLISHED_ONLY,
) -> dict[str, Any]: ) -> dict[str, Any]:
from .graph_db import ( from .graph_db import (
create_neo4j_driver, create_neo4j_driver,
@ -2607,12 +2634,15 @@ def search_graph(
) )
with index_snapshot(DATA_DIR) as version: with index_snapshot(DATA_DIR) as version:
if published_only and (version is None or version.get('publication_policy') != PUBLICATION_POLICY_VERSION):
raise IndexUnavailableError('Graf vyžaduje synchronizáciu pravidiel publikovania.')
with create_neo4j_driver(settings) as driver: with create_neo4j_driver(settings) as driver:
driver.verify_connectivity() driver.verify_connectivity()
with driver.session(database=settings.database) as session: with driver.session(database=settings.database) as session:
if version is not None and graph_version(session) != version: if version is not None and graph_version(session) != version:
raise IndexUnavailableError('Neo4j graf nezodpovedá aktívnej verzii dát.') raise IndexUnavailableError('Neo4j graf nezodpovedá aktívnej verzii dát.')
result = search_graph_with_session(session, query, limit=limit, max_entities=max_entities) result = search_graph_with_session(session, query, limit=limit, max_entities=max_entities,
published_only=published_only)
if version is not None: if version is not None:
result['index_version'] = version result['index_version'] = version
return result return result
@ -2641,6 +2671,8 @@ def main() -> None:
), ),
) )
parser.add_argument('--include-unpublished', action='store_true',
help='Výslovne zahrnie aj nepublikované dokumenty.')
args = parser.parse_args() args = parser.parse_args()
result = search_graph( result = search_graph(
@ -2649,6 +2681,7 @@ def main() -> None:
max_entities=( max_entities=(
args.max_entities args.max_entities
), ),
published_only=not args.include_unpublished,
) )
print( print(

10
tools/graphrag/graphrag.py
View File

@ -1,5 +1,7 @@
from __future__ import annotations from __future__ import annotations
from tools.publication import effective_published
import re import re
import unicodedata import unicodedata
from collections import defaultdict from collections import defaultdict
@ -1498,12 +1500,7 @@ def build_graph_payload(
) )
), ),
"document_kind": kind, "document_kind": kind,
"published": bool( "published": effective_published(document.get("published")),
document.get(
"published",
False,
)
),
"categories": ( "categories": (
doc_categories doc_categories
), ),
@ -1616,6 +1613,7 @@ def build_graph_payload(
] = { ] = {
"id": described_topic, "id": described_topic,
"name": title, "name": title,
"source_document": path,
"graph_scope": ( "graph_scope": (
GRAPH_SCOPE GRAPH_SCOPE
), ),

44
tools/graphrag/publication.py Normal file
View File

@ -0,0 +1,44 @@
"""Publication predicates applied before entity resolution, joins and LIMIT.
The query wrappers fail at import if their insertion point changes. Filtering
only final citations would still expose hidden facts and incorrect counts.
"""
def visible_node(node: str) -> str:
# Identifiers come only from the fixed query definitions, never user input.
return f'''CASE
WHEN {node}:Document THEN coalesce({node}.published, true)
WHEN {node}:Person OR ({node}:Topic AND {node}.source_document IS NOT NULL) THEN EXISTS {{
MATCH (public_source:Document)
WHERE public_source.graph_scope=$scope AND public_source.path={node}.source_document
AND coalesce(public_source.published, true)
}}
WHEN {node}:Thesis OR {node}:Work THEN EXISTS {{
MATCH ({node})-[:EVIDENCED_BY]->(public_source:Document)
WHERE public_source.graph_scope=$scope AND coalesce(public_source.published, true)
}} AND ({node}.source_document IS NULL OR EXISTS {{
MATCH (origin:Document) WHERE origin.graph_scope=$scope
AND origin.path={node}.source_document AND coalesce(origin.published, true)
}})
ELSE EXISTS {{
MATCH ({node})--(public_source:Document)
WHERE public_source.graph_scope=$scope AND coalesce(public_source.published, true)
}} END'''
def public_query(query: str, *, nodes: tuple[str, ...] = (), documents: tuple[str, ...] = (),
path: bool = False) -> str:
if path:
anchor = 'WHERE all(node IN nodes(p) WHERE node.graph_scope = $scope)'
predicate = f'all(public_node IN nodes(p) WHERE {visible_node("public_node")})'
if query.count(anchor) != 1:
raise ValueError('Missing unique path publication guard position')
return query.replace(anchor, anchor + f' AND (NOT $published_only OR ({predicate}))', 1)
predicates = [visible_node(node) for node in nodes]
predicates.extend(f'coalesce({node}.published, true)' for node in documents)
anchor = '\nWHERE '
if anchor not in query or not predicates:
raise ValueError('Missing publication guard position')
condition = ' AND '.join(f'({p})' for p in predicates)
return query.replace(anchor, f'\nWHERE (NOT $published_only OR ({condition})) AND ', 1)

101
tools/graphrag/tests/test_graph_publication.py Normal file
View File

@ -0,0 +1,101 @@
"""Publication is enforced by actual Cypher, before aggregation/traversal."""
import json
import os
import uuid
import pytest
from tools.graphrag import graph_search as gs
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
pytestmark = pytest.mark.skipif(os.getenv('RUN_NEO4J_TESTS') != '1', reason='Requires local Neo4j.')
@pytest.fixture
def graph(monkeypatch):
scope = 'publication-' + uuid.uuid4().hex
monkeypatch.setattr(gs, 'GRAPH_SCOPE', scope)
settings = get_neo4j_settings()
with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session:
tx = session.begin_transaction()
try:
for key, name, published in [('a','Alena Nová',True),('b','Boris Iný',None),('c','Cyril Malý',False)]:
tx.run('''CREATE (p:Person {id:$id, name:$name, graph_scope:$scope, source_document:$path})
CREATE (d:Document {path:$path, published:$published, graph_scope:$scope, source_url:$url})
CREATE (t:Thesis {id:$thesis, title:$title, year:2024, thesis_type:'bachelor_thesis',
source_document:$path, graph_scope:$scope})
MERGE (k:Keyword {id:$tag, name:'sharedcheck', graph_scope:$scope})
CREATE (p)-[:HAS_DOCUMENT]->(d)-[:HAS_KEYWORD]->(k)
CREATE (p)-[:AUTHORED]->(t)-[:EVIDENCED_BY]->(d)''',
id=scope+key, name=name, scope=scope, path=scope+'/'+key+'.md', published=published,
url='https://example.test/'+key, thesis=scope+'th'+key, title='Výskum '+key,tag=scope+'shared').consume()
tx.run('''MATCH (p:Person {id:$person}), (public:Document {path:$public})
CREATE (d:Document {path:$path, published:false, source_url:'https://example.test/private', graph_scope:$scope})
CREATE (k:Keyword {id:$keyword, name:'hiddentag', graph_scope:$scope})
CREATE (c:Category {name:$category, graph_scope:$scope})
CREATE (a:DocumentAuthor {id:$author, name:'Hidden Author', graph_scope:$scope})
CREATE (topic:Topic {id:$topic, name:'Hidden Topic', source_document:$path, graph_scope:$scope})
CREATE (p)-[:HAS_DOCUMENT]->(d)-[:HAS_KEYWORD]->(k)
CREATE (d)-[:IN_CATEGORY]->(c) CREATE (a)-[:AUTHORED_DOCUMENT]->(d)
CREATE (d)-[:DESCRIBES]->(topic) CREATE (public)-[:DESCRIBES]->(topic)''',
person=scope+'a',public=scope+'/a.md', path=scope+'/private.md',scope=scope,
keyword=scope+'hidden',category=scope+'hidden-category',author=scope+'author',topic=scope+'topic').consume()
yield tx, scope
finally:
tx.rollback()
def test_catalog_hides_private_people_and_private_only_vocabulary(graph):
tx, _ = graph
public = gs.run_rows(tx, gs.CATALOG_QUERY, scope=gs.GRAPH_SCOPE, published_only=True)
names = {r.get('name') for r in public}
assert 'Alena Nová' in names and 'Boris Iný' in names
assert not names & {'Cyril Malý','hiddentag','Hidden Author','Hidden Topic'}
all_rows = gs.run_rows(tx, gs.CATALOG_QUERY, scope=gs.GRAPH_SCOPE, published_only=False)
assert {'Cyril Malý','hiddentag','Hidden Author','Hidden Topic'} <= {r.get('name') for r in all_rows}
def test_traversal_and_citations_do_not_expose_hidden_documents(graph):
tx, scope = graph
result = gs.search_graph_with_session(tx, 'Alena Nová')
serialized = json.dumps(result, ensure_ascii=False)
assert scope+'/private.md' not in serialized
assert 'Hidden Topic' not in serialized and 'hiddentag' not in serialized
all_rows = gs.search_graph_with_session(tx, 'Alena Nová', published_only=False)
assert scope+'/private.md' in json.dumps(all_rows)
def test_global_count_is_filtered_before_counting(graph):
tx, _ = graph
public = gs.search_graph_with_session(tx, 'Koľko osôb má prácu v roku 2024?')
all_rows = gs.search_graph_with_session(tx, 'Koľko osôb má prácu v roku 2024?', published_only=False)
assert public['aggregation']['count'] == 2
assert set(public['aggregation']['people']) == {'Alena Nová','Boris Iný'}
assert all_rows['aggregation']['count'] == 3
def test_private_person_is_not_resolved_as_a_public_fact(graph):
tx, scope = graph
result = gs.search_graph_with_session(tx, 'Aký názov práce má osoba Cyril Malý v roku 2024?')
assert result['status']=='no_answer' and result['facts']==[] and result['documents']==[]
all_rows = gs.search_graph_with_session(tx, 'Aký názov práce má osoba Cyril Malý v roku 2024?', published_only=False)
assert all_rows['status']=='ok' and any(f.get('person')=='Cyril Malý' for f in all_rows['facts'])
def test_private_child_tag_is_not_used_in_conjunction_or_metadata(graph):
tx, _ = graph
result = gs.search_graph_with_session(tx,
'Aký názov práce má osoba Alena Nová v roku 2024 s kľúčovým slovom hiddentag?')
assert result['status']=='no_answer' and not result['facts']
all_rows = gs.search_graph_with_session(tx,
'Aký názov práce má osoba Alena Nová v roku 2024 s kľúčovým slovom hiddentag?', published_only=False)
assert all_rows['status']=='ok' and any(f.get('keyword')=='hiddentag' for f in all_rows['facts'])
def test_shared_keyword_does_not_include_private_person(graph):
tx, _ = graph
question = 'Majú osoby Alena Nová a Cyril Malý spoločné kľúčové slovo sharedcheck?'
public = gs.search_graph_with_session(tx, question)
assert public['status']=='no_answer' and not public['facts']
all_rows = gs.search_graph_with_session(tx, question, published_only=False)
assert any(f['fact_type']=='shared_keyword' for f in all_rows['facts'])

15
tools/graphrag/tests/test_graph_search.py
View File

@ -32,6 +32,21 @@ class FakeSession:
) )
) )
# Existing unit fixtures omit the separate stored-source lookup. Model
# that lookup as present documents; publication enforcement itself is
# exercised by real Neo4j fixtures with explicit publication flags.
if query == graph_search.DOCUMENT_SOURCES_QUERY and query not in self.responses:
urls = {}
for rows in self.responses.values():
for row in rows:
for prefix in ['', 'left_', 'right_']:
if row.get(prefix+'document_path'):
urls[row[prefix+'document_path']] = row.get(prefix+'source_url')
for node in row.get('nodes', []):
if node.get('path'):
urls[node['path']] = node.get('source_url')
return [{'document_path': path, 'source_url': urls.get(path)} for path in parameters.get('paths', [])]
return self.responses.get( return self.responses.get(
query, query,
[], [],

1
tools/graphrag/tests/test_graphrag.py
View File

@ -584,6 +584,7 @@ def test_topic_is_created_from_topic_document() -> None:
"Nástroje a zdroje pre " "Nástroje a zdroje pre "
"spracovanie prirodzeného jazyka" "spracovanie prirodzeného jazyka"
), ),
"source_document": path,
"graph_scope": "zpwiki", "graph_scope": "zpwiki",
} }
] ]

11
tools/publication.py Normal file
View File

@ -0,0 +1,11 @@
"""Shared default: only explicit published=false is excluded.
Missing/NULL publication metadata follows the wiki default and remains visible.
published_only=False is an explicit opt-out for authenticated/admin workflows.
"""
DEFAULT_PUBLISHED_ONLY = True
PUBLICATION_POLICY_VERSION = 'explicit-false-hidden-v1'
def effective_published(value: bool | None) -> bool:
return True if value is None else bool(value)

4
tools/rag/rag_document_expansion.py
View File

@ -361,7 +361,7 @@ def find_exact_student_document(
FROM documents FROM documents
WHERE ( WHERE (
? = 0 ? = 0
OR published = 1 OR COALESCE(published, 1) = 1
) )
ORDER BY ORDER BY
LENGTH(title) DESC, LENGTH(title) DESC,
@ -676,7 +676,7 @@ def load_best_document_section_chunk(
WHERE document_path = ? WHERE document_path = ?
AND ( AND (
? = 0 ? = 0
OR published = 1 OR COALESCE(published, 1) = 1
) )
ORDER BY ORDER BY
chunk_index ASC, chunk_index ASC,

10
tools/rag/rag_query_evidence.py
View File

@ -1,5 +1,7 @@
from __future__ import annotations from __future__ import annotations
from tools.publication import DEFAULT_PUBLISHED_ONLY
import json import json
import re import re
import sqlite3 import sqlite3
@ -704,7 +706,7 @@ def _load_document_chunks(
and "published" in columns and "published" in columns
): ):
where_parts.append( where_parts.append(
"published = 1" "COALESCE(published, 1) = 1"
) )
order_parts = [ order_parts = [
@ -766,7 +768,7 @@ def find_top_query_evidence_chunks(
document_path: str, document_path: str,
query: str, query: str,
*, *,
published_only: bool = False, published_only: bool = DEFAULT_PUBLISHED_ONLY,
top_k: int = DEFAULT_TOP_EVIDENCE_CHUNKS, top_k: int = DEFAULT_TOP_EVIDENCE_CHUNKS,
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
if top_k <= 0: if top_k <= 0:
@ -898,7 +900,7 @@ def find_best_query_evidence_chunk(
document_path: str, document_path: str,
query: str, query: str,
*, *,
published_only: bool = False, published_only: bool = DEFAULT_PUBLISHED_ONLY,
) -> dict[str, Any] | None: ) -> dict[str, Any] | None:
chunks = _load_document_chunks( chunks = _load_document_chunks(
conn, conn,
@ -970,7 +972,7 @@ def expand_results_with_query_evidence(
dict[str, Any] dict[str, Any]
], ],
*, *,
published_only: bool = False, published_only: bool = DEFAULT_PUBLISHED_ONLY,
top_k: int = DEFAULT_TOP_EVIDENCE_CHUNKS, top_k: int = DEFAULT_TOP_EVIDENCE_CHUNKS,
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
base_results = [ base_results = [

6
tools/rag/rag_supplemental_expansion.py
View File

@ -1,5 +1,7 @@
from __future__ import annotations from __future__ import annotations
from tools.publication import DEFAULT_PUBLISHED_ONLY
import json import json
import re import re
import sqlite3 import sqlite3
@ -280,7 +282,7 @@ def _load_chunks(
where = "" where = ""
if published_only and "published" in columns: if published_only and "published" in columns:
where = " WHERE published = 1" where = " WHERE COALESCE(published, 1) = 1"
order = " ORDER BY document_path ASC, chunk_index ASC" order = " ORDER BY document_path ASC, chunk_index ASC"
if "id" in columns: if "id" in columns:
@ -651,7 +653,7 @@ def expand_results_with_supplemental_sources(
query: str, query: str,
results: list[dict[str, Any]], results: list[dict[str, Any]],
*, *,
published_only: bool = False, published_only: bool = DEFAULT_PUBLISHED_ONLY,
limit: int = 5, limit: int = 5,
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
base_results = [dict(item) for item in results] base_results = [dict(item) for item in results]

6
tools/rag/rag_utils.py
View File

@ -1,5 +1,7 @@
from __future__ import annotations from __future__ import annotations
from tools.publication import DEFAULT_PUBLISHED_ONLY
import inspect import inspect
import json import json
import sqlite3 import sqlite3
@ -310,7 +312,7 @@ def load_section_lead_chunk(
FROM chunks FROM chunks
WHERE document_path = ? WHERE document_path = ?
AND chunk_index <= ? AND chunk_index <= ?
AND (? = 0 OR published = 1) AND (? = 0 OR COALESCE(published, 1) = 1)
ORDER BY chunk_index ASC, id ASC ORDER BY chunk_index ASC, id ASC
""", """,
( (
@ -698,7 +700,7 @@ def build_rag_context(
query: str, query: str,
*, *,
limit: int = 5, limit: int = 5,
published_only: bool = False, published_only: bool = DEFAULT_PUBLISHED_ONLY,
max_per_document: int = 1, max_per_document: int = 1,
) -> dict[str, Any]: ) -> dict[str, Any]:
response = search_database( response = search_database(

4
tools/rag/search_core.py
View File

@ -423,7 +423,7 @@ def run_fts_query(
'bm25({BM25_SQL})' 'bm25({BM25_SQL})'
AND ( AND (
? = 0 ? = 0
OR chunks.published = 1 OR COALESCE(chunks.published, 1) = 1
) )
ORDER BY ORDER BY
chunks_fts.rank ASC, chunks_fts.rank ASC,
@ -491,7 +491,7 @@ def run_vector_query(
= chunk_embeddings.chunk_id = chunk_embeddings.chunk_id
WHERE ( WHERE (
? = 0 ? = 0
OR chunks.published = 1 OR COALESCE(chunks.published, 1) = 1
) )
""", """,
( (

6
tools/rag/search_db.py
View File

@ -136,9 +136,9 @@ def main() -> None:
parser.add_argument( parser.add_argument(
"--published-only", "--published-only",
action="store_true", action="store_true",
default=True,
help=( help=(
"Vyhľadáva iba v dokumentoch " "Vylúči explicitné published=false (predvolené)."
"s published: true."
), ),
) )
@ -153,6 +153,8 @@ def main() -> None:
), ),
) )
parser.add_argument("--include-unpublished", action="store_false", dest="published_only",
help="Výslovne zahrnie aj nepublikované dokumenty.")
args = parser.parse_args() args = parser.parse_args()
query = " ".join(args.query) query = " ".join(args.query)

4
tools/rag/search_utils.py
View File

@ -1,5 +1,7 @@
from __future__ import annotations from __future__ import annotations
from tools.publication import DEFAULT_PUBLISHED_ONLY
import sqlite3 import sqlite3
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@ -49,7 +51,7 @@ def search_database(
db_file: Path, db_file: Path,
query: str, query: str,
limit: int = 10, limit: int = 10,
published_only: bool = False, published_only: bool = DEFAULT_PUBLISHED_ONLY,
max_per_document: int = 3, max_per_document: int = 3,
) -> dict[str, Any]: ) -> dict[str, Any]:
if not db_file.exists(): if not db_file.exists():

2
tools/rag/tests/test_rag.py
View File

@ -1156,7 +1156,7 @@ def test_rag_endpoint(
assert ( assert (
published_only published_only
is False is True
) )
assert ( assert (

2
tools/rag/tests/test_search.py
View File

@ -254,6 +254,7 @@ def test_prefix_fallback_handles_slovak_inflection(
db_file, db_file,
"hlboke ucenie", "hlboke ucenie",
limit=5, limit=5,
published_only=False,
) )
assert response["strategies"] == [ assert response["strategies"] == [
@ -309,6 +310,7 @@ def test_published_only_excludes_unpublished_chunks(
db_file, db_file,
"hlboke ucenie", "hlboke ucenie",
limit=5, limit=5,
published_only=False,
) )
public_results = search_database( public_results = search_database(

3
tools/sync_indexes.py
View File

@ -25,6 +25,7 @@ from tools.rag.scan_zpwiki import scan_pages
from tools.rag.build_chunks import build_chunks from tools.rag.build_chunks import build_chunks
from tools.rag.build_sqlite_index import build_database from tools.rag.build_sqlite_index import build_database
from tools.rag.embedding_utils import embedding_model_name from tools.rag.embedding_utils import embedding_model_name
from tools.publication import PUBLICATION_POLICY_VERSION
from tools.graphrag.graphrag import build_graph_payload from tools.graphrag.graphrag import build_graph_payload
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
from tools.graphrag.build_graphrag import (create_constraints, graph_version, from tools.graphrag.build_graphrag import (create_constraints, graph_version,
@ -68,7 +69,7 @@ def prepare_snapshot(wiki_root: Path, stage: Path) -> tuple[dict, dict, dict]:
'created_at_utc': datetime.now(timezone.utc).isoformat(), 'created_at_utc': datetime.now(timezone.utc).isoformat(),
'source_sha256': hashlib.sha256(json.dumps(sources, sort_keys=True).encode()).hexdigest(), 'source_sha256': hashlib.sha256(json.dumps(sources, sort_keys=True).encode()).hexdigest(),
'documents_sha256': digest(stage / 'documents.json'), 'chunks_sha256': digest(stage / 'chunks.json'), 'documents_sha256': digest(stage / 'documents.json'), 'chunks_sha256': digest(stage / 'chunks.json'),
'embedding_model': embedding_model_name()} 'embedding_model': embedding_model_name(), 'publication_policy': PUBLICATION_POLICY_VERSION}
with closing(sqlite3.connect(stage / 'zp_index.sqlite')) as conn, conn: with closing(sqlite3.connect(stage / 'zp_index.sqlite')) as conn, conn:
conn.execute('CREATE TABLE index_state (id INTEGER PRIMARY KEY CHECK(id=1), metadata_json TEXT NOT NULL)') conn.execute('CREATE TABLE index_state (id INTEGER PRIMARY KEY CHECK(id=1), metadata_json TEXT NOT NULL)')
conn.execute('INSERT INTO index_state VALUES (1, ?)', (json.dumps(version, sort_keys=True),)) conn.execute('INSERT INTO index_state VALUES (1, ?)', (json.dumps(version, sort_keys=True),))

89
tools/tool_health.py Normal file
View File

@ -0,0 +1,89 @@
"""Independent, sanitized readiness results for the two retrieval backends."""
from __future__ import annotations
import json
import sqlite3
from contextlib import closing
from pathlib import Path
from neo4j import Query
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
from tools.graphrag.graphrag import GRAPH_SCOPE
from tools.index_state import IndexUnavailableError, index_snapshot, sqlite_version
from tools.publication import PUBLICATION_POLICY_VERSION
from tools.rag.search_core import verify_search_schema, embedding_index_info
PROBE_TIMEOUT_SECONDS = 2.0
def readiness(ready: bool, reason: str | None = None, **details) -> dict:
return {'ready': ready, 'status': 'ready' if ready else 'unavailable', 'reason': reason, **details}
def rag_readiness(db_file: Path, version: dict | None, *, embedding_ready: bool) -> dict:
if not db_file.is_file():
return readiness(False, 'index_missing')
try:
if version is not None and sqlite_version(db_file) != version:
return readiness(False, 'version_mismatch')
with closing(sqlite3.connect(db_file.resolve().as_uri()+'?mode=ro', uri=True, timeout=1)) as conn:
conn.row_factory = sqlite3.Row
verify_search_schema(conn)
model, dimensions = embedding_index_info(conn)
conn.execute('SELECT rowid FROM chunks_fts LIMIT 1').fetchone()
if not embedding_ready:
return readiness(False, 'embedding_not_ready', embedding_model=model)
return readiness(True, embedding_model=model, dimensions=dimensions, index_version=version)
except (sqlite3.Error, RuntimeError, ValueError, OSError):
return readiness(False, 'index_invalid')
def neo4j_readiness(version: dict | None) -> tuple[dict, dict]:
try:
settings = get_neo4j_settings()
except RuntimeError:
return readiness(False, 'not_configured'), readiness(False, 'neo4j_not_configured')
try:
with create_neo4j_driver(settings, connection_timeout=PROBE_TIMEOUT_SECONDS,
connection_acquisition_timeout=PROBE_TIMEOUT_SECONDS,
connection_write_timeout=PROBE_TIMEOUT_SECONDS,
max_transaction_retry_time=0, max_connection_pool_size=1) as driver:
driver.verify_connectivity()
with driver.session(database=settings.database) as session:
record = session.run(Query('''MATCH (n) WHERE n.graph_scope=$scope
WITH count(n) AS nodes
OPTIONAL MATCH (state:IndexState {id:$scope})
RETURN nodes, state.metadata_json AS metadata''', timeout=PROBE_TIMEOUT_SECONDS),
scope=GRAPH_SCOPE).single()
except Exception:
# Credentials, connection strings and raw driver errors stay server-side.
return readiness(False, 'unreachable'), readiness(False, 'neo4j_unavailable')
neo4j = readiness(True)
if not record or record['nodes'] == 0:
return neo4j, readiness(False, 'graph_empty')
try:
actual = json.loads(record['metadata']) if record['metadata'] else None
except (ValueError, TypeError):
return neo4j, readiness(False, 'graph_version_invalid')
if version is not None and actual != version:
return neo4j, readiness(False, 'version_mismatch')
if version is None or version.get('publication_policy') != PUBLICATION_POLICY_VERSION:
return neo4j, readiness(False, 'publication_sync_required')
return neo4j, readiness(True, index_version=actual, nodes=record['nodes'])
def tools_readiness(db_file: Path, *, security_configured: bool, embedding_ready: bool) -> tuple[dict, dict | None]:
version = None
try:
with index_snapshot(db_file.parent) as version:
rag = rag_readiness(db_file, version, embedding_ready=embedding_ready)
neo4j, graph = neo4j_readiness(version)
except IndexUnavailableError:
neo4j, _ = neo4j_readiness(None)
rag = readiness(False, 'snapshot_unavailable')
graph = readiness(False, 'snapshot_unavailable')
if not security_configured:
rag = readiness(False, 'api_not_configured')
graph = readiness(False, 'api_not_configured')
return {'rag': rag, 'neo4j': neo4j, 'graphrag': graph}, version