Publikovanie a dostupnosť
This commit is contained in:
parent
86a155691d
commit
fee6ffc70f
16
README.md
16
README.md
@ -695,6 +695,18 @@ vyvolá rollback. Pri nedokončenom prepnutí po páde zostane vyhľadávanie
|
||||
nedostupné do úspešného opakovania synchronizácie. Podrobnosti a zotavenie:
|
||||
[Spoločná synchronizácia indexov](docs/synchronizacia_indexov.md).
|
||||
|
||||
## Publikovanie a dostupnosť nástrojov
|
||||
|
||||
Oba nástroje predvolene vylučujú iba explicitné `published=false`.
|
||||
Chýbajúca hodnota sa považuje za publikovanú. Parameter
|
||||
`published_only=false` výslovne zahrnie aj nepublikované dokumenty pri
|
||||
autorizovanom volaní; CLI používa `--include-unpublished`.
|
||||
|
||||
`/health` uvádza samostatne `tools.rag`, `tools.neo4j` a `tools.graphrag`.
|
||||
Globálne `ready` vyžaduje oba vyhľadávacie nástroje; pri výpadku Neo4j
|
||||
môže klient ďalej používať RAG podľa `tools.rag.ready`.
|
||||
[Politika, migrácia a príklady](docs/publikovanie_a_dostupnost.md).
|
||||
|
||||
## Hybridné vyhľadávanie
|
||||
|
||||
Vyhľadávanie kombinuje:
|
||||
@ -747,7 +759,7 @@ curl -X POST http://127.0.0.1:8000/search \
|
||||
-d '{
|
||||
"query": "strojový preklad",
|
||||
"limit": 3,
|
||||
"published_only": false,
|
||||
"published_only": true,
|
||||
"max_per_document": 1
|
||||
}'
|
||||
```
|
||||
@ -798,7 +810,7 @@ curl -X POST http://127.0.0.1:8000/rag \
|
||||
-d '{
|
||||
"query": "V akom roku robil Ján Holp diplomovú prácu?",
|
||||
"limit": 5,
|
||||
"published_only": false,
|
||||
"published_only": true,
|
||||
"max_per_document": 1
|
||||
}'
|
||||
```
|
||||
|
||||
@ -41,15 +41,17 @@ OPENWEBUI_ORIGIN = (
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(
|
||||
_: FastAPI,
|
||||
application: FastAPI,
|
||||
):
|
||||
validate_security_configuration()
|
||||
application.state.embedding_ready = False
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
embed_query,
|
||||
"warmup",
|
||||
)
|
||||
application.state.embedding_ready = True
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
|
||||
@ -46,7 +46,8 @@ from tools.rag.rebuild_index import (
|
||||
rebuild_index,
|
||||
)
|
||||
from tools.rag.search_utils import search_database
|
||||
from tools.index_state import index_snapshot, IndexUnavailableError
|
||||
from tools.publication import DEFAULT_PUBLISHED_ONLY
|
||||
from tools.tool_health import tools_readiness
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -104,7 +105,7 @@ class SearchRequest(
|
||||
le=50,
|
||||
)
|
||||
|
||||
published_only: bool = False
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY
|
||||
|
||||
max_per_document: int = Field(
|
||||
default=1,
|
||||
@ -137,10 +138,10 @@ class RagRequest(
|
||||
)
|
||||
|
||||
published_only: bool = Field(
|
||||
default=False,
|
||||
default=DEFAULT_PUBLISHED_ONLY,
|
||||
description=(
|
||||
"Ak je true, použijú sa iba "
|
||||
"publikované dokumenty."
|
||||
"dokumenty okrem explicitného published=false; chýbajúca hodnota sa zahrnie."
|
||||
),
|
||||
)
|
||||
|
||||
@ -184,6 +185,7 @@ class HealthResponse(BaseModel):
|
||||
|
||||
security_configured: bool
|
||||
index_version: dict[str, Any] | None = None
|
||||
tools: dict[str, Any]
|
||||
|
||||
|
||||
class RagSource(BaseModel):
|
||||
@ -515,7 +517,7 @@ async def read_webhook_body(
|
||||
include_in_schema=False,
|
||||
response_model=HealthResponse,
|
||||
)
|
||||
def health() -> dict[str, Any]:
|
||||
def health(request: Request) -> dict[str, Any]:
|
||||
database_exists = (
|
||||
DB_FILE.exists()
|
||||
)
|
||||
@ -532,22 +534,15 @@ def health() -> dict[str, Any]:
|
||||
except RuntimeError:
|
||||
security_configured = False
|
||||
|
||||
ready = (
|
||||
database_exists
|
||||
and zpwiki_exists
|
||||
and security_configured
|
||||
)
|
||||
version = None
|
||||
try:
|
||||
with index_snapshot(DB_FILE.parent, db_file=DB_FILE) as version:
|
||||
pass
|
||||
except (IndexUnavailableError, sqlite3.Error):
|
||||
ready = False
|
||||
tools, version = tools_readiness(DB_FILE, security_configured=security_configured,
|
||||
embedding_ready=getattr(request.app.state, 'embedding_ready', False))
|
||||
ready = tools['rag']['ready'] and tools['graphrag']['ready']
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"ready": ready,
|
||||
"index_version": version,
|
||||
"tools": tools,
|
||||
"database_exists": (
|
||||
database_exists
|
||||
),
|
||||
@ -623,9 +618,10 @@ def health() -> dict[str, Any]:
|
||||
)
|
||||
def rag(
|
||||
request: RagRequest,
|
||||
http_request: Request,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return build_rag_context(
|
||||
result = build_rag_context(
|
||||
DB_FILE,
|
||||
request.query,
|
||||
limit=request.limit,
|
||||
@ -636,6 +632,8 @@ def rag(
|
||||
request.max_per_document
|
||||
),
|
||||
)
|
||||
http_request.app.state.embedding_ready = True
|
||||
return result
|
||||
|
||||
except Exception as error:
|
||||
raise_retrieval_error(
|
||||
@ -655,6 +653,7 @@ def rag(
|
||||
)
|
||||
def search(
|
||||
request: SearchRequest,
|
||||
http_request: Request,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
response = search_database(
|
||||
@ -668,6 +667,7 @@ def search(
|
||||
request.max_per_document
|
||||
),
|
||||
)
|
||||
http_request.app.state.embedding_ready = True
|
||||
|
||||
except Exception as error:
|
||||
raise_retrieval_error(
|
||||
@ -690,6 +690,7 @@ def search(
|
||||
results
|
||||
),
|
||||
"results": results,
|
||||
**({'index_version': response['index_version']} if response.get('index_version') else {}),
|
||||
}
|
||||
|
||||
|
||||
@ -705,6 +706,7 @@ def search(
|
||||
)
|
||||
def sync(
|
||||
request: SyncRequest,
|
||||
http_request: Request,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
result = rebuild_index(
|
||||
@ -712,6 +714,7 @@ def sync(
|
||||
request.pull_git
|
||||
)
|
||||
)
|
||||
http_request.app.state.embedding_ready = True
|
||||
|
||||
except Exception as error:
|
||||
raise_reindex_error(
|
||||
@ -890,6 +893,7 @@ async def gitea_webhook(
|
||||
webhook_should_pull_git()
|
||||
),
|
||||
)
|
||||
request.app.state.embedding_ready = True
|
||||
|
||||
except Exception as error:
|
||||
raise_reindex_error(
|
||||
|
||||
102
docs/publikovanie_a_dostupnost.md
Normal file
102
docs/publikovanie_a_dostupnost.md
Normal file
@ -0,0 +1,102 @@
|
||||
# Publikovanie dokumentov a dostupnosť nástrojov
|
||||
|
||||
## Jednotná politika
|
||||
|
||||
Používateľ zvolil zahrnutie dokumentov bez `published`. RAG aj GraphRAG preto
|
||||
predvolene používajú `published_only=true` s týmto významom:
|
||||
|
||||
| Hodnota zdroja | Predvolené vyhľadávanie |
|
||||
|---|---|
|
||||
| `published: true` | Zahrnúť |
|
||||
| Chýbajúce `published` alebo normalizované `null` | Zahrnúť |
|
||||
| `published: false` | Vylúčiť |
|
||||
|
||||
Chýbajúca hodnota zodpovedá predvolenému významu príznaku v
|
||||
[Grav](https://learn.getgrav.org/2/content/headers). Táto politika určuje
|
||||
výber podľa `published`; API naďalej používa svoju existujúcu autentifikáciu.
|
||||
V aktuálnom korpuse nie sú nastavené `publish_date` ani `unpublish_date`.
|
||||
|
||||
SQL používa `COALESCE(published, 1) = 1`, graf efektívnu publikačnú hodnotu.
|
||||
Zdrojový JSON a SQLite zachovávajú pôvodné `null`; grafový builder ho
|
||||
normalizuje na efektívne `true`. Všetkých 16 pôvodne neoznačených dokumentov
|
||||
je teda dostupných v oboch vetvách. Jeden explicitne nepublikovaný dokument
|
||||
sa predvolene vylúči.
|
||||
|
||||
Pri RAG sa filter aplikuje na FTS, embeddingy aj všetky rozšírenia kontextu.
|
||||
Pri grafe platí pre katalóg entít, fakty, spoločné vlastnosti, počítanie,
|
||||
prechádzanie grafu a zoznam zdrojov. Entity doložené iba skrytými dokumentmi
|
||||
sa nezaradia do verejného katalógu. Pôvod názvu témy a canonical profilu
|
||||
osoby sa kontroluje podľa príslušného zdrojového dokumentu.
|
||||
|
||||
Výslovné zahrnutie nepublikovaných dokumentov pre autorizované volanie:
|
||||
|
||||
```json
|
||||
{"query": "vyhľadávací dotaz", "published_only": false}
|
||||
```
|
||||
|
||||
Python funkcie používajú rovnaký parameter `published_only=False`.
|
||||
Oba vyhľadávacie CLI podporujú `--include-unpublished`. Pred pripojením
|
||||
nástrojov do OpenWebUI sa používa aktuálna OpenAPI schéma s novým defaultom.
|
||||
|
||||
## Migrácia uloženého grafu
|
||||
|
||||
Po nasadení sa vykoná spoločná synchronizácia. Tá opraví staré zamenenie
|
||||
chýbajúcej hodnoty s `false` a doplní pôvod názvu témy. Verzia indexov obsahuje
|
||||
`publication_policy: explicit-false-hidden-v1`. Pred migráciou predvolené
|
||||
grafové vyhľadávanie vyžaduje synchronizáciu a health uvádza
|
||||
`publication_sync_required`. Existujúce historické merania sa neprepisujú;
|
||||
časť klasického baseline používala aj nepublikované dokumenty.
|
||||
|
||||
## `/health`
|
||||
|
||||
HTTP 200 a `status=ok` označujú živý API proces. `ready` znamená pripravenosť
|
||||
oboch vyhľadávacích nástrojov. Klient/agenta pri výbere konkrétneho nástroja
|
||||
zaujímajú samostatné položky `tools`:
|
||||
|
||||
- `tools.rag`: čitateľný SQLite/FTS5/embedding index, zhoda verzie a známa
|
||||
úspešná inicializácia embeddingového modelu. Úspešné vyhľadanie alebo sync
|
||||
obnoví stav modelu po prípadnom zlyhaní inicializácie.
|
||||
- `tools.neo4j`: dostupné spojenie a vykonanie read-only sondy v databáze.
|
||||
Sonda používa krátke časové limity a nevykonáva automatické opakovania transakcií.
|
||||
- `tools.graphrag`: dostupné Neo4j, neprázdny graf, zhodná generácia dát
|
||||
a migrovaná publikačná politika.
|
||||
|
||||
Výpadok Neo4j nespôsobí nedostupnosť RAG. Neplatný SQLite index môže vyradiť
|
||||
RAG, pričom graf ostane dostupný. Počas nejasného prepnutia generácie sa
|
||||
vyhľadávacie nástroje označia ako nedostupné; stav samotného Neo4j sa uvádza
|
||||
samostatne. Odpovede neobsahujú prihlasovacie údaje ani surové chyby drivera.
|
||||
|
||||
Príklad výpadku Neo4j, skrátený na rozhodujúce polia:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"ready": false,
|
||||
"tools": {
|
||||
"rag": {"ready": true, "status": "ready", "reason": null},
|
||||
"neo4j": {"ready": false, "status": "unavailable", "reason": "unreachable"},
|
||||
"graphrag": {"ready": false, "status": "unavailable", "reason": "neo4j_unavailable"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Ide o dostupnosť retrieval nástrojov. Health neposiela otázky školskému LLM
|
||||
a nemeria dostupnosť externého generatívneho modelu ani presnosť odpovedí.
|
||||
|
||||
## Overenie nasadenia 25. 10. 2026
|
||||
|
||||
- Kompletná regresná sada: **485 úspešných testov, 4 preskočené**. Zahŕňa
|
||||
skutočné Cypher testy filtrovania, agregácie a grafových prechodov nad
|
||||
izolovaným Neo4j grafom s rollbackom.
|
||||
- Docker image bol znovu zostavený a API nasadené. Autentifikovaný `/sync`
|
||||
bez Git pullu obnovil 114 dokumentov, 887 chunkov a graf 447 uzlov/879 vzťahov.
|
||||
- Reálne `/search`, `/rag` a `search_graph()` overili predvolené vylúčenie
|
||||
skrytého dokumentu aj jeho zahrnutie pri `published_only=false`.
|
||||
- HTTP `/health` hlási všetky tri stavy pripravené; SQLite, súborová verzia
|
||||
a Neo4j používajú rovnakú novú generáciu dát.
|
||||
- Sonda s nefunkčnou adresou Neo4j potvrdila dostupnosť RAG pri nedostupnom
|
||||
grafe. Sonda s chýbajúcim SQLite indexom potvrdila dostupnosť grafu pri
|
||||
nedostupnom RAG. Produkčné služby sa pri týchto sondách nevypínali.
|
||||
|
||||
Strojovo čitateľný výsledok a identifikácia dát:
|
||||
[publikovanie_a_dostupnost_overenie.json](publikovanie_a_dostupnost_overenie.json).
|
||||
162
docs/publikovanie_a_dostupnost_overenie.json
Normal file
162
docs/publikovanie_a_dostupnost_overenie.json
Normal file
@ -0,0 +1,162 @@
|
||||
{
|
||||
"status": "ok",
|
||||
"index_version": {
|
||||
"version": "ff682d98134d406081040902f234c44f",
|
||||
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
||||
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
|
||||
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
||||
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
||||
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"publication_policy": "explicit-false-hidden-v1"
|
||||
},
|
||||
"health": {
|
||||
"status": "ok",
|
||||
"ready": true,
|
||||
"database_exists": true,
|
||||
"search_engine": "hybrid_fts5_embeddings",
|
||||
"rag_enabled": true,
|
||||
"zpwiki_exists": true,
|
||||
"security_configured": true,
|
||||
"index_version": {
|
||||
"version": "ff682d98134d406081040902f234c44f",
|
||||
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
||||
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
|
||||
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
||||
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
||||
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"publication_policy": "explicit-false-hidden-v1"
|
||||
},
|
||||
"tools": {
|
||||
"rag": {
|
||||
"ready": true,
|
||||
"status": "ready",
|
||||
"reason": null,
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"dimensions": 384,
|
||||
"index_version": {
|
||||
"version": "ff682d98134d406081040902f234c44f",
|
||||
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
||||
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
|
||||
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
||||
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
||||
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"publication_policy": "explicit-false-hidden-v1"
|
||||
}
|
||||
},
|
||||
"neo4j": {
|
||||
"ready": true,
|
||||
"status": "ready",
|
||||
"reason": null
|
||||
},
|
||||
"graphrag": {
|
||||
"ready": true,
|
||||
"status": "ready",
|
||||
"reason": null,
|
||||
"index_version": {
|
||||
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
||||
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
|
||||
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
||||
"publication_policy": "explicit-false-hidden-v1",
|
||||
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
||||
"version": "ff682d98134d406081040902f234c44f"
|
||||
},
|
||||
"nodes": 447
|
||||
}
|
||||
}
|
||||
},
|
||||
"publication_counts": {
|
||||
"published_true": 97,
|
||||
"missing": 16,
|
||||
"published_false": 1,
|
||||
"default_visible": 113
|
||||
},
|
||||
"graph_document_counts": {
|
||||
"True": 113,
|
||||
"False": 1
|
||||
},
|
||||
"live_search_checks": {
|
||||
"/search": {
|
||||
"default_hides_unpublished": true,
|
||||
"explicit_opt_out_includes_it": true
|
||||
},
|
||||
"/rag": {
|
||||
"default_hides_unpublished": true,
|
||||
"explicit_opt_out_includes_it": true
|
||||
},
|
||||
"graphrag": {
|
||||
"default_hides_unpublished": true,
|
||||
"explicit_opt_out_includes_it": true,
|
||||
"default_status": "ok"
|
||||
}
|
||||
},
|
||||
"neo4j_outage_probe": {
|
||||
"rag": {
|
||||
"ready": true,
|
||||
"status": "ready",
|
||||
"reason": null,
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"dimensions": 384,
|
||||
"index_version": {
|
||||
"version": "ff682d98134d406081040902f234c44f",
|
||||
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
||||
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
|
||||
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
||||
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
||||
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"publication_policy": "explicit-false-hidden-v1"
|
||||
}
|
||||
},
|
||||
"neo4j": {
|
||||
"ready": false,
|
||||
"status": "unavailable",
|
||||
"reason": "unreachable"
|
||||
},
|
||||
"graphrag": {
|
||||
"ready": false,
|
||||
"status": "unavailable",
|
||||
"reason": "neo4j_unavailable"
|
||||
}
|
||||
},
|
||||
"rag_index_outage_probe": {
|
||||
"rag": {
|
||||
"ready": false,
|
||||
"status": "unavailable",
|
||||
"reason": "index_missing"
|
||||
},
|
||||
"neo4j": {
|
||||
"ready": true,
|
||||
"status": "ready",
|
||||
"reason": null
|
||||
},
|
||||
"graphrag": {
|
||||
"ready": true,
|
||||
"status": "ready",
|
||||
"reason": null,
|
||||
"index_version": {
|
||||
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
||||
"created_at_utc": "2026-10-25T15:06:03.279424+00:00",
|
||||
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
||||
"publication_policy": "explicit-false-hidden-v1",
|
||||
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
||||
"version": "ff682d98134d406081040902f234c44f"
|
||||
},
|
||||
"nodes": 447
|
||||
}
|
||||
},
|
||||
"checked_at_utc": "2026-10-25T15:09:42.459584+00:00",
|
||||
"regression": {
|
||||
"passed": 485,
|
||||
"skipped": 4,
|
||||
"failed": 0,
|
||||
"includes_real_neo4j_publication_tests": true
|
||||
},
|
||||
"scope": "Publikačný filter a dostupnosť retrieval nástrojov; nejde o nové meranie presnosti LLM."
|
||||
}
|
||||
166
test/test_publication_health.py
Normal file
166
test/test_publication_health.py
Normal file
@ -0,0 +1,166 @@
|
||||
import json
|
||||
import sqlite3
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.main as main
|
||||
import app.routes as routes
|
||||
import tools.tool_health as health
|
||||
import tools.rag.build_sqlite_index as indexer
|
||||
import tools.rag.search_core as search_core
|
||||
from tools.index_state import durable_json, PENDING_FILE, VERSION_FILE
|
||||
from tools.publication import PUBLICATION_POLICY_VERSION, effective_published
|
||||
from tools.rag.search_utils import search_database
|
||||
from tools.rag.rag_utils import build_rag_context
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def corpus(tmp_path, monkeypatch):
|
||||
docs = []; chunks = []
|
||||
paths = ['pages/students/2020/eva/README.md', 'pages/topics/missing/README.md',
|
||||
'pages/topics/hidden/README.md', 'pages/students/2020/eva/private/README.md']
|
||||
for index, (path, published) in enumerate(zip(paths, [True, None, False, False])):
|
||||
title = ['Verejný profil', 'Chýbajúca značka', 'Skrytý dokument', 'Skrytá poznámka'][index]
|
||||
text = 'spoločný overovací výraz ' + ('SKRYTY_OVEROVACI_OBSAH' if published is False else title)
|
||||
docs.append({'path': path, 'title': title, 'published': published})
|
||||
chunks.append({'chunk_id': path+'::0', 'document_path': path, 'title': title,
|
||||
'published': published, 'chunk_index': 0, 'heading_paths': [['Úvod']],
|
||||
'text': text, 'text_length': len(text), 'token_count': 10,
|
||||
'content_hash': str(index), 'tags': [], 'categories': []})
|
||||
data = tmp_path / 'data'; data.mkdir()
|
||||
for name, rows in [('documents.json', docs), ('chunks.json', chunks)]:
|
||||
(data/name).write_text(json.dumps(rows))
|
||||
monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.ones((len(texts), 4), dtype=np.float32))
|
||||
monkeypatch.setattr(search_core, 'embed_query', lambda *args, **kwargs: np.ones(4, dtype=np.float32))
|
||||
db = data / 'zp_index.sqlite'
|
||||
indexer.build_database(documents_file=data/'documents.json', chunks_file=data/'chunks.json', db_file=db)
|
||||
version = {'version': 'test-generation', 'publication_policy': PUBLICATION_POLICY_VERSION}
|
||||
with sqlite3.connect(db) as conn:
|
||||
conn.execute('CREATE TABLE index_state (id INTEGER PRIMARY KEY, metadata_json TEXT)')
|
||||
conn.execute('INSERT INTO index_state VALUES (1,?)', (json.dumps(version),))
|
||||
durable_json(data/VERSION_FILE, version)
|
||||
monkeypatch.setattr(routes, 'DB_FILE', db)
|
||||
monkeypatch.setattr(main, 'embed_query', lambda *args: None)
|
||||
return db, version, paths
|
||||
|
||||
|
||||
@pytest.mark.parametrize('value,visible', [(True, True), (None, True), (False, False)])
|
||||
def test_publication_policy(value, visible):
|
||||
assert effective_published(value) is visible
|
||||
|
||||
|
||||
def test_rag_default_filters_both_fts_and_vectors_but_keeps_null(corpus):
|
||||
db, _, paths = corpus
|
||||
result = search_database(db, 'spoločný overovací výraz', limit=10)
|
||||
assert {r['document_path'] for r in result['results']} == set(paths[:2])
|
||||
assert any(r['published'] is None for r in result['results'])
|
||||
all_rows = search_database(db, 'spoločný overovací výraz', limit=10, published_only=False)
|
||||
assert {r['document_path'] for r in all_rows['results']} == set(paths)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('query', ['spoločný overovací výraz', 'Skrytý dokument', 'Skrytá poznámka'])
|
||||
def test_rag_expansions_do_not_reintroduce_hidden_text(corpus, query):
|
||||
db, _, paths = corpus
|
||||
result = build_rag_context(db, query, limit=10)
|
||||
assert all(s['document_path'] not in paths[2:] for s in result['sources'])
|
||||
assert 'SKRYTY_OVEROVACI_OBSAH' not in result['context']
|
||||
all_rows = build_rag_context(db, query, limit=10, published_only=False)
|
||||
assert 'SKRYTY_OVEROVACI_OBSAH' in all_rows['context']
|
||||
|
||||
|
||||
def healthy_graph(version):
|
||||
return health.readiness(True), health.readiness(True, index_version=version, nodes=2)
|
||||
|
||||
|
||||
def test_neo4j_outage_does_not_disable_rag(corpus, monkeypatch):
|
||||
_, _, paths = corpus
|
||||
monkeypatch.setattr(health, 'neo4j_readiness', lambda v: (
|
||||
health.readiness(False, 'unreachable'), health.readiness(False, 'neo4j_unavailable')))
|
||||
with TestClient(main.app) as client:
|
||||
payload = client.get('/health').json()
|
||||
assert payload['tools']['rag']['ready'] is True
|
||||
assert payload['tools']['neo4j']['ready'] is False
|
||||
assert payload['tools']['graphrag']['ready'] is False
|
||||
assert payload['ready'] is False
|
||||
response = client.post('/rag', json={'query': 'spoločný overovací výraz'},
|
||||
headers={'Authorization': 'Bearer '+'a'*64})
|
||||
assert response.status_code == 200
|
||||
assert {s['document_path'] for s in response.json()['sources']} == set(paths[:2])
|
||||
|
||||
|
||||
def test_rag_index_outage_does_not_disable_graph(corpus, monkeypatch):
|
||||
db, version, _ = corpus
|
||||
monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph)
|
||||
db.unlink()
|
||||
with TestClient(main.app) as client:
|
||||
payload = client.get('/health').json()
|
||||
assert payload['tools']['rag']['ready'] is False
|
||||
assert payload['tools']['neo4j']['ready'] is True
|
||||
assert payload['tools']['graphrag']['ready'] is True
|
||||
assert payload['ready'] is False
|
||||
|
||||
|
||||
def test_pending_sync_blocks_both_tools_but_reports_neo4j_service(corpus, monkeypatch):
|
||||
db, _, _ = corpus
|
||||
durable_json(db.parent/PENDING_FILE, {'target_version': 'next'})
|
||||
monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph)
|
||||
with TestClient(main.app) as client:
|
||||
tools = client.get('/health').json()['tools']
|
||||
assert tools['rag']['reason'] == tools['graphrag']['reason'] == 'snapshot_unavailable'
|
||||
assert tools['neo4j']['ready'] is True
|
||||
|
||||
|
||||
def test_embedding_startup_failure_and_successful_query_recovery(corpus, monkeypatch):
|
||||
monkeypatch.setattr(health, 'neo4j_readiness', healthy_graph)
|
||||
def broken(*args):
|
||||
raise RuntimeError('transient warmup failure')
|
||||
monkeypatch.setattr(main, 'embed_query', broken)
|
||||
with TestClient(main.app) as client:
|
||||
tools = client.get('/health').json()['tools']
|
||||
assert tools['rag']['reason'] == 'embedding_not_ready'
|
||||
assert tools['graphrag']['ready'] is True
|
||||
response = client.post('/rag', json={'query': 'spoločný overovací výraz'},
|
||||
headers={'Authorization': 'Bearer '+'a'*64})
|
||||
assert response.status_code == 200
|
||||
assert client.get('/health').json()['tools']['rag']['ready'] is True
|
||||
|
||||
|
||||
def test_neo4j_probe_is_bounded_and_does_not_expose_errors(monkeypatch):
|
||||
monkeypatch.setattr(health, 'get_neo4j_settings', lambda: SimpleNamespace(database='neo4j'))
|
||||
captured = {}
|
||||
def fail(settings, **options):
|
||||
captured.update(options)
|
||||
raise RuntimeError('password=never-return-this')
|
||||
monkeypatch.setattr(health, 'create_neo4j_driver', fail)
|
||||
result = health.neo4j_readiness(None)
|
||||
assert not result[0]['ready'] and not result[1]['ready']
|
||||
assert captured['connection_timeout'] == captured['connection_acquisition_timeout'] == 2.0
|
||||
assert captured['max_transaction_retry_time'] == 0
|
||||
assert 'never-return-this' not in json.dumps(result)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('actual,nodes,reason', [
|
||||
({'version': 'different'}, 2, 'version_mismatch'),
|
||||
(None, 0, 'graph_empty'),
|
||||
({'version': 'old'}, 2, 'publication_sync_required'),
|
||||
])
|
||||
def test_connection_readiness_is_distinct_from_graph_readiness(monkeypatch, actual, nodes, reason):
|
||||
class Result:
|
||||
def single(self):
|
||||
return {'nodes': nodes, 'metadata': json.dumps(actual) if actual else None}
|
||||
class Connection:
|
||||
def __enter__(self): return self
|
||||
def __exit__(self, *args): pass
|
||||
def verify_connectivity(self): pass
|
||||
def session(self, **kwargs): return self
|
||||
def run(self, query, **params):
|
||||
assert query.timeout == 2.0
|
||||
return Result()
|
||||
monkeypatch.setattr(health, 'get_neo4j_settings', lambda: SimpleNamespace(database='neo4j'))
|
||||
monkeypatch.setattr(health, 'create_neo4j_driver', lambda *args, **kwargs: Connection())
|
||||
expected = {'version': 'expected'} if reason=='version_mismatch' else actual
|
||||
neo, graph = health.neo4j_readiness(expected)
|
||||
assert neo['ready'] is True and graph['ready'] is False and graph['reason']==reason
|
||||
@ -39,6 +39,7 @@ def get_neo4j_settings() -> Neo4jSettings:
|
||||
|
||||
def create_neo4j_driver(
|
||||
settings: Neo4jSettings | None = None,
|
||||
**driver_options,
|
||||
):
|
||||
resolved = settings or get_neo4j_settings()
|
||||
|
||||
@ -48,6 +49,7 @@ def create_neo4j_driver(
|
||||
resolved.user,
|
||||
resolved.password,
|
||||
),
|
||||
**driver_options,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@ -5,6 +5,8 @@ import json
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Any, Iterable
|
||||
from tools.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION
|
||||
from .publication import public_query
|
||||
|
||||
|
||||
GRAPH_SCOPE = "zpwiki"
|
||||
@ -2286,6 +2288,23 @@ RETURN document.path AS document_path, document.source_url AS source_url
|
||||
ORDER BY document_path
|
||||
"""
|
||||
|
||||
# Restrict the complete retrieval pipeline, including vocabulary and traversal.
|
||||
CATALOG_QUERY = public_query(CATALOG_QUERY, nodes=('n',))
|
||||
NEIGHBORHOOD_QUERY = public_query(NEIGHBORHOOD_QUERY, path=True)
|
||||
PAIR_PATH_QUERY = public_query(PAIR_PATH_QUERY, path=True)
|
||||
THESIS_FACTS_QUERY = public_query(THESIS_FACTS_QUERY, nodes=('person',), documents=('document',))
|
||||
WORK_FACTS_QUERY = public_query(WORK_FACTS_QUERY, nodes=('person',), documents=('document',))
|
||||
THESIS_PATHS_QUERY = public_query(THESIS_PATHS_QUERY, nodes=('person',), documents=('document',))
|
||||
WORK_PATHS_QUERY = public_query(WORK_PATHS_QUERY, nodes=('person',), documents=('document',))
|
||||
START_YEAR_FACTS_QUERY = public_query(START_YEAR_FACTS_QUERY, nodes=('person',))
|
||||
PERSON_KEYWORD_FACTS_QUERY = public_query(PERSON_KEYWORD_FACTS_QUERY, nodes=('person',), documents=('document',))
|
||||
PERSON_CATEGORY_FACTS_QUERY = public_query(PERSON_CATEGORY_FACTS_QUERY, nodes=('person',), documents=('document',))
|
||||
PERSON_DOCUMENT_AUTHOR_FACTS_QUERY = public_query(PERSON_DOCUMENT_AUTHOR_FACTS_QUERY, nodes=('person',), documents=('document',))
|
||||
SHARED_KEYWORD_FACTS_QUERY = public_query(SHARED_KEYWORD_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document'))
|
||||
SHARED_CATEGORY_FACTS_QUERY = public_query(SHARED_CATEGORY_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document'))
|
||||
SHARED_DOCUMENT_AUTHOR_FACTS_QUERY = public_query(SHARED_DOCUMENT_AUTHOR_FACTS_QUERY, nodes=('left','right'), documents=('left_document','right_document'))
|
||||
DOCUMENT_SOURCES_QUERY = public_query(DOCUMENT_SOURCES_QUERY, documents=('document',))
|
||||
|
||||
|
||||
def search_graph_with_session(
|
||||
session: Any,
|
||||
@ -2293,6 +2312,7 @@ def search_graph_with_session(
|
||||
*,
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
max_entities: int = DEFAULT_MAX_ENTITIES,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
) -> dict[str, Any]:
|
||||
clean_query = str(query or "").strip()
|
||||
if not clean_query:
|
||||
@ -2303,7 +2323,8 @@ def search_graph_with_session(
|
||||
raise ValueError("max_entities musí byť aspoň 1")
|
||||
|
||||
constraints = extract_query_constraints(clean_query)
|
||||
catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE)
|
||||
catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE, published_only=published_only)
|
||||
constraints['published_only'] = published_only
|
||||
# Explicit tag filters are resolved separately. Incidental names such as the
|
||||
# tag 'zp' in 'ZP Wiki' must never become additional query constraints.
|
||||
phrase = constraints["keyword_phrase"]
|
||||
@ -2425,7 +2446,8 @@ def search_graph_with_session(
|
||||
# the presentation limit. The catalog bounds the number of academic nodes;
|
||||
# relational lookups use a conservative bound covering the managed graph.
|
||||
fetch_limit = max(limit, len(catalog) ** 2)
|
||||
common = {"scope": GRAPH_SCOPE, "person_ids": person_ids, "limit": fetch_limit}
|
||||
common = {"scope": GRAPH_SCOPE, "person_ids": person_ids, "limit": fetch_limit,
|
||||
"published_only": published_only}
|
||||
blocked_reasons = []
|
||||
if constraints["unresolved_keyword"]:
|
||||
blocked_reasons.append("unknown_keyword")
|
||||
@ -2507,7 +2529,8 @@ def search_graph_with_session(
|
||||
))
|
||||
if not structured and resolved_entities and not blocked:
|
||||
start_ids = [str(row["element_id"]) for row in resolved_entities]
|
||||
params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit}
|
||||
params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit,
|
||||
"published_only": published_only}
|
||||
if constraints["mentions_shared"] and len(start_ids) >= 2:
|
||||
paths.extend(run_rows(session, PAIR_PATH_QUERY, **params))
|
||||
else:
|
||||
@ -2543,8 +2566,11 @@ def search_graph_with_session(
|
||||
# actual stored URLs rather than manufacturing URLs from expected answers.
|
||||
documents = collect_documents(resolved_entities, paths, facts)
|
||||
sources = run_rows(session, DOCUMENT_SOURCES_QUERY, scope=GRAPH_SCOPE,
|
||||
paths=[row["document_path"] for row in documents]) if documents else []
|
||||
paths=[row["document_path"] for row in documents],
|
||||
published_only=published_only) if documents else []
|
||||
by_path = {row["document_path"]: row.get("source_url") for row in sources}
|
||||
if published_only:
|
||||
documents = [row for row in documents if row['document_path'] in by_path]
|
||||
for document in documents:
|
||||
if not document.get("source_url"):
|
||||
document["source_url"] = by_path.get(document["document_path"])
|
||||
@ -2594,6 +2620,7 @@ def search_graph(
|
||||
max_entities: int = (
|
||||
DEFAULT_MAX_ENTITIES
|
||||
),
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
) -> dict[str, Any]:
|
||||
from .graph_db import (
|
||||
create_neo4j_driver,
|
||||
@ -2607,12 +2634,15 @@ def search_graph(
|
||||
)
|
||||
|
||||
with index_snapshot(DATA_DIR) as version:
|
||||
if published_only and (version is None or version.get('publication_policy') != PUBLICATION_POLICY_VERSION):
|
||||
raise IndexUnavailableError('Graf vyžaduje synchronizáciu pravidiel publikovania.')
|
||||
with create_neo4j_driver(settings) as driver:
|
||||
driver.verify_connectivity()
|
||||
with driver.session(database=settings.database) as session:
|
||||
if version is not None and graph_version(session) != version:
|
||||
raise IndexUnavailableError('Neo4j graf nezodpovedá aktívnej verzii dát.')
|
||||
result = search_graph_with_session(session, query, limit=limit, max_entities=max_entities)
|
||||
result = search_graph_with_session(session, query, limit=limit, max_entities=max_entities,
|
||||
published_only=published_only)
|
||||
if version is not None:
|
||||
result['index_version'] = version
|
||||
return result
|
||||
@ -2641,6 +2671,8 @@ def main() -> None:
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument('--include-unpublished', action='store_true',
|
||||
help='Výslovne zahrnie aj nepublikované dokumenty.')
|
||||
args = parser.parse_args()
|
||||
|
||||
result = search_graph(
|
||||
@ -2649,6 +2681,7 @@ def main() -> None:
|
||||
max_entities=(
|
||||
args.max_entities
|
||||
),
|
||||
published_only=not args.include_unpublished,
|
||||
)
|
||||
|
||||
print(
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.publication import effective_published
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import defaultdict
|
||||
@ -1498,12 +1500,7 @@ def build_graph_payload(
|
||||
)
|
||||
),
|
||||
"document_kind": kind,
|
||||
"published": bool(
|
||||
document.get(
|
||||
"published",
|
||||
False,
|
||||
)
|
||||
),
|
||||
"published": effective_published(document.get("published")),
|
||||
"categories": (
|
||||
doc_categories
|
||||
),
|
||||
@ -1616,6 +1613,7 @@ def build_graph_payload(
|
||||
] = {
|
||||
"id": described_topic,
|
||||
"name": title,
|
||||
"source_document": path,
|
||||
"graph_scope": (
|
||||
GRAPH_SCOPE
|
||||
),
|
||||
|
||||
44
tools/graphrag/publication.py
Normal file
44
tools/graphrag/publication.py
Normal file
@ -0,0 +1,44 @@
|
||||
"""Publication predicates applied before entity resolution, joins and LIMIT.
|
||||
|
||||
The query wrappers fail at import if their insertion point changes. Filtering
|
||||
only final citations would still expose hidden facts and incorrect counts.
|
||||
"""
|
||||
|
||||
|
||||
def visible_node(node: str) -> str:
|
||||
# Identifiers come only from the fixed query definitions, never user input.
|
||||
return f'''CASE
|
||||
WHEN {node}:Document THEN coalesce({node}.published, true)
|
||||
WHEN {node}:Person OR ({node}:Topic AND {node}.source_document IS NOT NULL) THEN EXISTS {{
|
||||
MATCH (public_source:Document)
|
||||
WHERE public_source.graph_scope=$scope AND public_source.path={node}.source_document
|
||||
AND coalesce(public_source.published, true)
|
||||
}}
|
||||
WHEN {node}:Thesis OR {node}:Work THEN EXISTS {{
|
||||
MATCH ({node})-[:EVIDENCED_BY]->(public_source:Document)
|
||||
WHERE public_source.graph_scope=$scope AND coalesce(public_source.published, true)
|
||||
}} AND ({node}.source_document IS NULL OR EXISTS {{
|
||||
MATCH (origin:Document) WHERE origin.graph_scope=$scope
|
||||
AND origin.path={node}.source_document AND coalesce(origin.published, true)
|
||||
}})
|
||||
ELSE EXISTS {{
|
||||
MATCH ({node})--(public_source:Document)
|
||||
WHERE public_source.graph_scope=$scope AND coalesce(public_source.published, true)
|
||||
}} END'''
|
||||
|
||||
|
||||
def public_query(query: str, *, nodes: tuple[str, ...] = (), documents: tuple[str, ...] = (),
|
||||
path: bool = False) -> str:
|
||||
if path:
|
||||
anchor = 'WHERE all(node IN nodes(p) WHERE node.graph_scope = $scope)'
|
||||
predicate = f'all(public_node IN nodes(p) WHERE {visible_node("public_node")})'
|
||||
if query.count(anchor) != 1:
|
||||
raise ValueError('Missing unique path publication guard position')
|
||||
return query.replace(anchor, anchor + f' AND (NOT $published_only OR ({predicate}))', 1)
|
||||
predicates = [visible_node(node) for node in nodes]
|
||||
predicates.extend(f'coalesce({node}.published, true)' for node in documents)
|
||||
anchor = '\nWHERE '
|
||||
if anchor not in query or not predicates:
|
||||
raise ValueError('Missing publication guard position')
|
||||
condition = ' AND '.join(f'({p})' for p in predicates)
|
||||
return query.replace(anchor, f'\nWHERE (NOT $published_only OR ({condition})) AND ', 1)
|
||||
101
tools/graphrag/tests/test_graph_publication.py
Normal file
101
tools/graphrag/tests/test_graph_publication.py
Normal file
@ -0,0 +1,101 @@
|
||||
"""Publication is enforced by actual Cypher, before aggregation/traversal."""
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.graphrag import graph_search as gs
|
||||
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
|
||||
|
||||
pytestmark = pytest.mark.skipif(os.getenv('RUN_NEO4J_TESTS') != '1', reason='Requires local Neo4j.')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def graph(monkeypatch):
|
||||
scope = 'publication-' + uuid.uuid4().hex
|
||||
monkeypatch.setattr(gs, 'GRAPH_SCOPE', scope)
|
||||
settings = get_neo4j_settings()
|
||||
with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session:
|
||||
tx = session.begin_transaction()
|
||||
try:
|
||||
for key, name, published in [('a','Alena Nová',True),('b','Boris Iný',None),('c','Cyril Malý',False)]:
|
||||
tx.run('''CREATE (p:Person {id:$id, name:$name, graph_scope:$scope, source_document:$path})
|
||||
CREATE (d:Document {path:$path, published:$published, graph_scope:$scope, source_url:$url})
|
||||
CREATE (t:Thesis {id:$thesis, title:$title, year:2024, thesis_type:'bachelor_thesis',
|
||||
source_document:$path, graph_scope:$scope})
|
||||
MERGE (k:Keyword {id:$tag, name:'sharedcheck', graph_scope:$scope})
|
||||
CREATE (p)-[:HAS_DOCUMENT]->(d)-[:HAS_KEYWORD]->(k)
|
||||
CREATE (p)-[:AUTHORED]->(t)-[:EVIDENCED_BY]->(d)''',
|
||||
id=scope+key, name=name, scope=scope, path=scope+'/'+key+'.md', published=published,
|
||||
url='https://example.test/'+key, thesis=scope+'th'+key, title='Výskum '+key,tag=scope+'shared').consume()
|
||||
tx.run('''MATCH (p:Person {id:$person}), (public:Document {path:$public})
|
||||
CREATE (d:Document {path:$path, published:false, source_url:'https://example.test/private', graph_scope:$scope})
|
||||
CREATE (k:Keyword {id:$keyword, name:'hiddentag', graph_scope:$scope})
|
||||
CREATE (c:Category {name:$category, graph_scope:$scope})
|
||||
CREATE (a:DocumentAuthor {id:$author, name:'Hidden Author', graph_scope:$scope})
|
||||
CREATE (topic:Topic {id:$topic, name:'Hidden Topic', source_document:$path, graph_scope:$scope})
|
||||
CREATE (p)-[:HAS_DOCUMENT]->(d)-[:HAS_KEYWORD]->(k)
|
||||
CREATE (d)-[:IN_CATEGORY]->(c) CREATE (a)-[:AUTHORED_DOCUMENT]->(d)
|
||||
CREATE (d)-[:DESCRIBES]->(topic) CREATE (public)-[:DESCRIBES]->(topic)''',
|
||||
person=scope+'a',public=scope+'/a.md', path=scope+'/private.md',scope=scope,
|
||||
keyword=scope+'hidden',category=scope+'hidden-category',author=scope+'author',topic=scope+'topic').consume()
|
||||
yield tx, scope
|
||||
finally:
|
||||
tx.rollback()
|
||||
|
||||
|
||||
def test_catalog_hides_private_people_and_private_only_vocabulary(graph):
|
||||
tx, _ = graph
|
||||
public = gs.run_rows(tx, gs.CATALOG_QUERY, scope=gs.GRAPH_SCOPE, published_only=True)
|
||||
names = {r.get('name') for r in public}
|
||||
assert 'Alena Nová' in names and 'Boris Iný' in names
|
||||
assert not names & {'Cyril Malý','hiddentag','Hidden Author','Hidden Topic'}
|
||||
all_rows = gs.run_rows(tx, gs.CATALOG_QUERY, scope=gs.GRAPH_SCOPE, published_only=False)
|
||||
assert {'Cyril Malý','hiddentag','Hidden Author','Hidden Topic'} <= {r.get('name') for r in all_rows}
|
||||
|
||||
|
||||
def test_traversal_and_citations_do_not_expose_hidden_documents(graph):
|
||||
tx, scope = graph
|
||||
result = gs.search_graph_with_session(tx, 'Alena Nová')
|
||||
serialized = json.dumps(result, ensure_ascii=False)
|
||||
assert scope+'/private.md' not in serialized
|
||||
assert 'Hidden Topic' not in serialized and 'hiddentag' not in serialized
|
||||
all_rows = gs.search_graph_with_session(tx, 'Alena Nová', published_only=False)
|
||||
assert scope+'/private.md' in json.dumps(all_rows)
|
||||
|
||||
|
||||
def test_global_count_is_filtered_before_counting(graph):
|
||||
tx, _ = graph
|
||||
public = gs.search_graph_with_session(tx, 'Koľko osôb má prácu v roku 2024?')
|
||||
all_rows = gs.search_graph_with_session(tx, 'Koľko osôb má prácu v roku 2024?', published_only=False)
|
||||
assert public['aggregation']['count'] == 2
|
||||
assert set(public['aggregation']['people']) == {'Alena Nová','Boris Iný'}
|
||||
assert all_rows['aggregation']['count'] == 3
|
||||
|
||||
|
||||
def test_private_person_is_not_resolved_as_a_public_fact(graph):
|
||||
tx, scope = graph
|
||||
result = gs.search_graph_with_session(tx, 'Aký názov práce má osoba Cyril Malý v roku 2024?')
|
||||
assert result['status']=='no_answer' and result['facts']==[] and result['documents']==[]
|
||||
all_rows = gs.search_graph_with_session(tx, 'Aký názov práce má osoba Cyril Malý v roku 2024?', published_only=False)
|
||||
assert all_rows['status']=='ok' and any(f.get('person')=='Cyril Malý' for f in all_rows['facts'])
|
||||
|
||||
|
||||
def test_private_child_tag_is_not_used_in_conjunction_or_metadata(graph):
|
||||
tx, _ = graph
|
||||
result = gs.search_graph_with_session(tx,
|
||||
'Aký názov práce má osoba Alena Nová v roku 2024 s kľúčovým slovom hiddentag?')
|
||||
assert result['status']=='no_answer' and not result['facts']
|
||||
all_rows = gs.search_graph_with_session(tx,
|
||||
'Aký názov práce má osoba Alena Nová v roku 2024 s kľúčovým slovom hiddentag?', published_only=False)
|
||||
assert all_rows['status']=='ok' and any(f.get('keyword')=='hiddentag' for f in all_rows['facts'])
|
||||
|
||||
|
||||
def test_shared_keyword_does_not_include_private_person(graph):
|
||||
tx, _ = graph
|
||||
question = 'Majú osoby Alena Nová a Cyril Malý spoločné kľúčové slovo sharedcheck?'
|
||||
public = gs.search_graph_with_session(tx, question)
|
||||
assert public['status']=='no_answer' and not public['facts']
|
||||
all_rows = gs.search_graph_with_session(tx, question, published_only=False)
|
||||
assert any(f['fact_type']=='shared_keyword' for f in all_rows['facts'])
|
||||
@ -32,6 +32,21 @@ class FakeSession:
|
||||
)
|
||||
)
|
||||
|
||||
# Existing unit fixtures omit the separate stored-source lookup. Model
|
||||
# that lookup as present documents; publication enforcement itself is
|
||||
# exercised by real Neo4j fixtures with explicit publication flags.
|
||||
if query == graph_search.DOCUMENT_SOURCES_QUERY and query not in self.responses:
|
||||
urls = {}
|
||||
for rows in self.responses.values():
|
||||
for row in rows:
|
||||
for prefix in ['', 'left_', 'right_']:
|
||||
if row.get(prefix+'document_path'):
|
||||
urls[row[prefix+'document_path']] = row.get(prefix+'source_url')
|
||||
for node in row.get('nodes', []):
|
||||
if node.get('path'):
|
||||
urls[node['path']] = node.get('source_url')
|
||||
return [{'document_path': path, 'source_url': urls.get(path)} for path in parameters.get('paths', [])]
|
||||
|
||||
return self.responses.get(
|
||||
query,
|
||||
[],
|
||||
|
||||
@ -584,6 +584,7 @@ def test_topic_is_created_from_topic_document() -> None:
|
||||
"Nástroje a zdroje pre "
|
||||
"spracovanie prirodzeného jazyka"
|
||||
),
|
||||
"source_document": path,
|
||||
"graph_scope": "zpwiki",
|
||||
}
|
||||
]
|
||||
|
||||
11
tools/publication.py
Normal file
11
tools/publication.py
Normal file
@ -0,0 +1,11 @@
|
||||
"""Shared default: only explicit published=false is excluded.
|
||||
|
||||
Missing/NULL publication metadata follows the wiki default and remains visible.
|
||||
published_only=False is an explicit opt-out for authenticated/admin workflows.
|
||||
"""
|
||||
DEFAULT_PUBLISHED_ONLY = True
|
||||
PUBLICATION_POLICY_VERSION = 'explicit-false-hidden-v1'
|
||||
|
||||
|
||||
def effective_published(value: bool | None) -> bool:
|
||||
return True if value is None else bool(value)
|
||||
@ -361,7 +361,7 @@ def find_exact_student_document(
|
||||
FROM documents
|
||||
WHERE (
|
||||
? = 0
|
||||
OR published = 1
|
||||
OR COALESCE(published, 1) = 1
|
||||
)
|
||||
ORDER BY
|
||||
LENGTH(title) DESC,
|
||||
@ -676,7 +676,7 @@ def load_best_document_section_chunk(
|
||||
WHERE document_path = ?
|
||||
AND (
|
||||
? = 0
|
||||
OR published = 1
|
||||
OR COALESCE(published, 1) = 1
|
||||
)
|
||||
ORDER BY
|
||||
chunk_index ASC,
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.publication import DEFAULT_PUBLISHED_ONLY
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
@ -704,7 +706,7 @@ def _load_document_chunks(
|
||||
and "published" in columns
|
||||
):
|
||||
where_parts.append(
|
||||
"published = 1"
|
||||
"COALESCE(published, 1) = 1"
|
||||
)
|
||||
|
||||
order_parts = [
|
||||
@ -766,7 +768,7 @@ def find_top_query_evidence_chunks(
|
||||
document_path: str,
|
||||
query: str,
|
||||
*,
|
||||
published_only: bool = False,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
top_k: int = DEFAULT_TOP_EVIDENCE_CHUNKS,
|
||||
) -> list[dict[str, Any]]:
|
||||
if top_k <= 0:
|
||||
@ -898,7 +900,7 @@ def find_best_query_evidence_chunk(
|
||||
document_path: str,
|
||||
query: str,
|
||||
*,
|
||||
published_only: bool = False,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
) -> dict[str, Any] | None:
|
||||
chunks = _load_document_chunks(
|
||||
conn,
|
||||
@ -970,7 +972,7 @@ def expand_results_with_query_evidence(
|
||||
dict[str, Any]
|
||||
],
|
||||
*,
|
||||
published_only: bool = False,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
top_k: int = DEFAULT_TOP_EVIDENCE_CHUNKS,
|
||||
) -> list[dict[str, Any]]:
|
||||
base_results = [
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.publication import DEFAULT_PUBLISHED_ONLY
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
@ -280,7 +282,7 @@ def _load_chunks(
|
||||
|
||||
where = ""
|
||||
if published_only and "published" in columns:
|
||||
where = " WHERE published = 1"
|
||||
where = " WHERE COALESCE(published, 1) = 1"
|
||||
|
||||
order = " ORDER BY document_path ASC, chunk_index ASC"
|
||||
if "id" in columns:
|
||||
@ -651,7 +653,7 @@ def expand_results_with_supplemental_sources(
|
||||
query: str,
|
||||
results: list[dict[str, Any]],
|
||||
*,
|
||||
published_only: bool = False,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
limit: int = 5,
|
||||
) -> list[dict[str, Any]]:
|
||||
base_results = [dict(item) for item in results]
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.publication import DEFAULT_PUBLISHED_ONLY
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import sqlite3
|
||||
@ -310,7 +312,7 @@ def load_section_lead_chunk(
|
||||
FROM chunks
|
||||
WHERE document_path = ?
|
||||
AND chunk_index <= ?
|
||||
AND (? = 0 OR published = 1)
|
||||
AND (? = 0 OR COALESCE(published, 1) = 1)
|
||||
ORDER BY chunk_index ASC, id ASC
|
||||
""",
|
||||
(
|
||||
@ -698,7 +700,7 @@ def build_rag_context(
|
||||
query: str,
|
||||
*,
|
||||
limit: int = 5,
|
||||
published_only: bool = False,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
max_per_document: int = 1,
|
||||
) -> dict[str, Any]:
|
||||
response = search_database(
|
||||
|
||||
@ -423,7 +423,7 @@ def run_fts_query(
|
||||
'bm25({BM25_SQL})'
|
||||
AND (
|
||||
? = 0
|
||||
OR chunks.published = 1
|
||||
OR COALESCE(chunks.published, 1) = 1
|
||||
)
|
||||
ORDER BY
|
||||
chunks_fts.rank ASC,
|
||||
@ -491,7 +491,7 @@ def run_vector_query(
|
||||
= chunk_embeddings.chunk_id
|
||||
WHERE (
|
||||
? = 0
|
||||
OR chunks.published = 1
|
||||
OR COALESCE(chunks.published, 1) = 1
|
||||
)
|
||||
""",
|
||||
(
|
||||
|
||||
@ -136,9 +136,9 @@ def main() -> None:
|
||||
parser.add_argument(
|
||||
"--published-only",
|
||||
action="store_true",
|
||||
default=True,
|
||||
help=(
|
||||
"Vyhľadáva iba v dokumentoch "
|
||||
"s published: true."
|
||||
"Vylúči explicitné published=false (predvolené)."
|
||||
),
|
||||
)
|
||||
|
||||
@ -153,6 +153,8 @@ def main() -> None:
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument("--include-unpublished", action="store_false", dest="published_only",
|
||||
help="Výslovne zahrnie aj nepublikované dokumenty.")
|
||||
args = parser.parse_args()
|
||||
query = " ".join(args.query)
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.publication import DEFAULT_PUBLISHED_ONLY
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@ -49,7 +51,7 @@ def search_database(
|
||||
db_file: Path,
|
||||
query: str,
|
||||
limit: int = 10,
|
||||
published_only: bool = False,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
max_per_document: int = 3,
|
||||
) -> dict[str, Any]:
|
||||
if not db_file.exists():
|
||||
|
||||
@ -1156,7 +1156,7 @@ def test_rag_endpoint(
|
||||
|
||||
assert (
|
||||
published_only
|
||||
is False
|
||||
is True
|
||||
)
|
||||
|
||||
assert (
|
||||
|
||||
@ -254,6 +254,7 @@ def test_prefix_fallback_handles_slovak_inflection(
|
||||
db_file,
|
||||
"hlboke ucenie",
|
||||
limit=5,
|
||||
published_only=False,
|
||||
)
|
||||
|
||||
assert response["strategies"] == [
|
||||
@ -309,6 +310,7 @@ def test_published_only_excludes_unpublished_chunks(
|
||||
db_file,
|
||||
"hlboke ucenie",
|
||||
limit=5,
|
||||
published_only=False,
|
||||
)
|
||||
|
||||
public_results = search_database(
|
||||
|
||||
@ -25,6 +25,7 @@ from tools.rag.scan_zpwiki import scan_pages
|
||||
from tools.rag.build_chunks import build_chunks
|
||||
from tools.rag.build_sqlite_index import build_database
|
||||
from tools.rag.embedding_utils import embedding_model_name
|
||||
from tools.publication import PUBLICATION_POLICY_VERSION
|
||||
from tools.graphrag.graphrag import build_graph_payload
|
||||
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
|
||||
from tools.graphrag.build_graphrag import (create_constraints, graph_version,
|
||||
@ -68,7 +69,7 @@ def prepare_snapshot(wiki_root: Path, stage: Path) -> tuple[dict, dict, dict]:
|
||||
'created_at_utc': datetime.now(timezone.utc).isoformat(),
|
||||
'source_sha256': hashlib.sha256(json.dumps(sources, sort_keys=True).encode()).hexdigest(),
|
||||
'documents_sha256': digest(stage / 'documents.json'), 'chunks_sha256': digest(stage / 'chunks.json'),
|
||||
'embedding_model': embedding_model_name()}
|
||||
'embedding_model': embedding_model_name(), 'publication_policy': PUBLICATION_POLICY_VERSION}
|
||||
with closing(sqlite3.connect(stage / 'zp_index.sqlite')) as conn, conn:
|
||||
conn.execute('CREATE TABLE index_state (id INTEGER PRIMARY KEY CHECK(id=1), metadata_json TEXT NOT NULL)')
|
||||
conn.execute('INSERT INTO index_state VALUES (1, ?)', (json.dumps(version, sort_keys=True),))
|
||||
|
||||
89
tools/tool_health.py
Normal file
89
tools/tool_health.py
Normal file
@ -0,0 +1,89 @@
|
||||
"""Independent, sanitized readiness results for the two retrieval backends."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
from neo4j import Query
|
||||
|
||||
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
|
||||
from tools.graphrag.graphrag import GRAPH_SCOPE
|
||||
from tools.index_state import IndexUnavailableError, index_snapshot, sqlite_version
|
||||
from tools.publication import PUBLICATION_POLICY_VERSION
|
||||
from tools.rag.search_core import verify_search_schema, embedding_index_info
|
||||
|
||||
PROBE_TIMEOUT_SECONDS = 2.0
|
||||
|
||||
|
||||
def readiness(ready: bool, reason: str | None = None, **details) -> dict:
|
||||
return {'ready': ready, 'status': 'ready' if ready else 'unavailable', 'reason': reason, **details}
|
||||
|
||||
|
||||
def rag_readiness(db_file: Path, version: dict | None, *, embedding_ready: bool) -> dict:
|
||||
if not db_file.is_file():
|
||||
return readiness(False, 'index_missing')
|
||||
try:
|
||||
if version is not None and sqlite_version(db_file) != version:
|
||||
return readiness(False, 'version_mismatch')
|
||||
with closing(sqlite3.connect(db_file.resolve().as_uri()+'?mode=ro', uri=True, timeout=1)) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
verify_search_schema(conn)
|
||||
model, dimensions = embedding_index_info(conn)
|
||||
conn.execute('SELECT rowid FROM chunks_fts LIMIT 1').fetchone()
|
||||
if not embedding_ready:
|
||||
return readiness(False, 'embedding_not_ready', embedding_model=model)
|
||||
return readiness(True, embedding_model=model, dimensions=dimensions, index_version=version)
|
||||
except (sqlite3.Error, RuntimeError, ValueError, OSError):
|
||||
return readiness(False, 'index_invalid')
|
||||
|
||||
|
||||
def neo4j_readiness(version: dict | None) -> tuple[dict, dict]:
|
||||
try:
|
||||
settings = get_neo4j_settings()
|
||||
except RuntimeError:
|
||||
return readiness(False, 'not_configured'), readiness(False, 'neo4j_not_configured')
|
||||
try:
|
||||
with create_neo4j_driver(settings, connection_timeout=PROBE_TIMEOUT_SECONDS,
|
||||
connection_acquisition_timeout=PROBE_TIMEOUT_SECONDS,
|
||||
connection_write_timeout=PROBE_TIMEOUT_SECONDS,
|
||||
max_transaction_retry_time=0, max_connection_pool_size=1) as driver:
|
||||
driver.verify_connectivity()
|
||||
with driver.session(database=settings.database) as session:
|
||||
record = session.run(Query('''MATCH (n) WHERE n.graph_scope=$scope
|
||||
WITH count(n) AS nodes
|
||||
OPTIONAL MATCH (state:IndexState {id:$scope})
|
||||
RETURN nodes, state.metadata_json AS metadata''', timeout=PROBE_TIMEOUT_SECONDS),
|
||||
scope=GRAPH_SCOPE).single()
|
||||
except Exception:
|
||||
# Credentials, connection strings and raw driver errors stay server-side.
|
||||
return readiness(False, 'unreachable'), readiness(False, 'neo4j_unavailable')
|
||||
neo4j = readiness(True)
|
||||
if not record or record['nodes'] == 0:
|
||||
return neo4j, readiness(False, 'graph_empty')
|
||||
try:
|
||||
actual = json.loads(record['metadata']) if record['metadata'] else None
|
||||
except (ValueError, TypeError):
|
||||
return neo4j, readiness(False, 'graph_version_invalid')
|
||||
if version is not None and actual != version:
|
||||
return neo4j, readiness(False, 'version_mismatch')
|
||||
if version is None or version.get('publication_policy') != PUBLICATION_POLICY_VERSION:
|
||||
return neo4j, readiness(False, 'publication_sync_required')
|
||||
return neo4j, readiness(True, index_version=actual, nodes=record['nodes'])
|
||||
|
||||
|
||||
def tools_readiness(db_file: Path, *, security_configured: bool, embedding_ready: bool) -> tuple[dict, dict | None]:
|
||||
version = None
|
||||
try:
|
||||
with index_snapshot(db_file.parent) as version:
|
||||
rag = rag_readiness(db_file, version, embedding_ready=embedding_ready)
|
||||
neo4j, graph = neo4j_readiness(version)
|
||||
except IndexUnavailableError:
|
||||
neo4j, _ = neo4j_readiness(None)
|
||||
rag = readiness(False, 'snapshot_unavailable')
|
||||
graph = readiness(False, 'snapshot_unavailable')
|
||||
if not security_configured:
|
||||
rag = readiness(False, 'api_not_configured')
|
||||
graph = readiness(False, 'api_not_configured')
|
||||
return {'rag': rag, 'neo4j': neo4j, 'graphrag': graph}, version
|
||||
Loading…
Reference in New Issue
Block a user