diff --git a/.dockerignore b/.dockerignore index bb7c091..2c991f0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,11 @@ .venv/ .git/ __pycache__/ -*.pyc +*.py[cod] *.log - -data/*.sqlite -data/*.db -data/*.json +.env +.env.* +.pytest_cache/ +.coverage +htmlcov/ +data/ diff --git a/.gitignore b/.gitignore index 9ac4e06..2d63932 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,9 @@ .venv/ __pycache__/ -*.pyc - +*.py[cod] .env -.env.* *.log - -data/*.sqlite -data/*.db -data/*.json -data/*.json +.pytest_cache/ +.coverage +htmlcov/ +data/ diff --git a/README.md b/README.md index a2f51d9..22828aa 100644 --- a/README.md +++ b/README.md @@ -1,59 +1,32 @@ -# dp-zp-agent +# ZP Agent -Agent pre manažment záverečných prác nad repozitárom `zpwiki`. +Backend pre indexovanie a vyhľadávanie v repozitári záverečných prác `zpwiki`. -Projekt rieši základnú časť systému pre vyhľadávanie v Markdown súboroch zo školského repozitára záverečných prác. Cieľom je vytvoriť samostatnú službu, ktorá vie indexovať obsah `zpwiki`, vyhľadávať v ňom a neskôr sa napojí na OpenWebUI, RAG, znalostný graf a GraphRAG. +Projekt načítava Markdown dokumenty, spracuje YAML metadata, rozdelí obsah na tokenové chunky a vytvorí SQLite FTS5 index. Vyhľadávanie je dostupné cez FastAPI a systém podporuje manuálnu aj webhookovú synchronizáciu. -Aktuálne je implementovaný prototyp, ktorý vie načítať Markdown dokumenty, spracovať ich metadata, rozdeliť ich na menšie časti, uložiť ich do SQLite databázy a sprístupniť vyhľadávanie cez FastAPI. +## Implementované -## Aktuálny stav +- načítanie Markdown súborov a YAML front matter, +- normalizácia názvov, autorov, tagov, kategórií a `published`, +- tokenové chunkovanie pomocou `tiktoken`, +- zachovanie názvu dokumentu a hierarchie nadpisov v chunku, +- SQLite databáza a FTS5 fulltextový index, +- BM25 vyhľadávanie s podporou diakritiky a prefixových výrazov, +- filtrovanie publikovaných dokumentov, +- FastAPI endpointy `/health`, `/search`, `/sync` a `/webhook/gitea`, +- autorizácia `/sync` pomocou API kľúča, +- Gitea webhook s HMAC-SHA256 podpisom a kontrolou udalosti a repozitára, +- zámok proti súbežnému reindexovaniu, +- atomická výmena databázy po úspešnom reindexovaní, +- automatizované a integračné testy nad reálnymi dátami. -Zatiaľ je implementované: - -1. načítanie Markdown súborov z repozitára `zpwiki`, -2. extrakcia metadát z YAML front matter, -3. spracovanie položiek `taxonomy`, hlavne kategórie, tagy a autor, -4. rozdelenie dokumentov na menšie textové chunky, -5. vytvorenie SQLite indexu, -6. jednoduché skórovacie fulltextové vyhľadávanie nad chunkmi, -7. rozlíšenie režimu vyhľadávania: - 1. `person` pre mená osôb, napríklad `jan ptak`, - 2. `topic` pre tematické dopyty, napríklad `rag agent` alebo `knowledge graph`, -8. FastAPI backend, -9. endpoint `GET /health`, -10. endpoint `POST /search`, -11. endpoint `POST /sync` pre manuálne spustenie reindexovania, -12. endpoint `POST /webhook/gitea` pre prijatie webhooku z Gitea, -13. overenie webhooku pomocou jednoduchého tokenu alebo HMAC podpisu, -14. automatická Swagger dokumentácia API, -15. Dockerfile a `docker-compose.yml`, -16. spustenie celého riešenia cez Docker, -17. volume mount pre priečinok `data`, -18. volume mount pre repozitár `zpwiki`. - -## Overený stav testovania - -Pri testovaní cez Docker bolo overené: - -1. FastAPI kontajner sa spustí, -2. endpoint `/health` vracia `200 OK`, -3. endpoint `/search` vracia `200 OK`, -4. endpoint `/sync` spustí reindexovanie a vracia `200 OK`, -5. endpoint `/webhook/gitea` prijme platný webhook a spustí reindexovanie, -6. Docker kontajner vidí repozitár `zpwiki` cez cestu `/zpwiki`, -7. systém načítal 114 dokumentov, -8. systém vytvoril 955 chunkov, -9. SQLite index bol vytvorený v `/app/data/zp_index.sqlite`. - -## Štruktúra projektu +## Štruktúra ```text -dp-zp-agent/ +zp-agent/ ├── app/ -│ ├── __init__.py │ └── main.py ├── scripts/ -│ ├── __init__.py │ ├── common.py │ ├── scan_zpwiki.py │ ├── build_chunks.py @@ -61,93 +34,16 @@ dp-zp-agent/ │ ├── rebuild_index.py │ ├── search_db.py │ └── search_utils.py +├── test/ ├── data/ ├── Dockerfile ├── docker-compose.yml ├── requirements.txt -├── .gitignore +├── requirements-dev.txt └── README.md ``` -Súbor `scripts/search_chunks.py` bol odstránený, pretože jeho funkcionalita bola duplicitná voči súboru `scripts/build_chunks.py`. - -## Popis hlavných súborov - -### `app/main.py` - -Obsahuje FastAPI aplikáciu a API endpointy: - -1. `GET /health`, -2. `POST /search`, -3. `POST /sync`, -4. `POST /webhook/gitea`. - -### `scripts/common.py` - -Obsahuje spoločné konštanty a pomocné funkcie: - -1. cesty k projektu, -2. cesta k `zpwiki`, -3. cesta k dátovým súborom, -4. čítanie a zápis JSON, -5. spracovanie YAML metadát, -6. normalizácia tagov a kategórií. - -### `scripts/scan_zpwiki.py` - -Prejde Markdown súbory v `zpwiki`, načíta metadata a uloží základné informácie do súboru: - -```text -data/documents.json -``` - -### `scripts/build_chunks.py` - -Rozdelí obsah Markdown dokumentov na menšie textové chunky a uloží ich do súboru: - -```text -data/chunks.json -``` - -### `scripts/build_sqlite_index.py` - -Vytvorí SQLite databázu: - -```text -data/zp_index.sqlite -``` - -Do databázy uloží dokumenty, chunky, tagy a kategórie. - -### `scripts/rebuild_index.py` - -Spustí celý proces naraz: - -1. načítanie dokumentov, -2. vytvorenie chunkov, -3. vytvorenie SQLite indexu. - -Voliteľne vie pred reindexovaním spustiť aj `git pull`. - -### `scripts/search_utils.py` - -Obsahuje spoločnú logiku vyhľadávania: - -1. normalizácia textu, -2. tokenizácia, -3. rozlíšenie režimu `person` a `topic`, -4. skórovanie výsledkov, -5. vyhľadávanie v SQLite databáze. - -### `scripts/search_db.py` - -Slúži na testovanie vyhľadávania z terminálu. - -## Príprava prostredia - -Projekt očakáva, že vedľa neho existuje naklonovaný repozitár `zpwiki`. - -Odporúčaná štruktúra: +Projekt očakáva repozitáre v tejto štruktúre: ```text ~/DP/ @@ -155,396 +51,113 @@ Odporúčaná štruktúra: └── zp-agent/ ``` -## Lokálne spustenie bez Dockeru +## Konfigurácia -Vytvorenie a aktivácia Python prostredia: +V koreňovom priečinku vytvor `.env`: -```bash -python3 -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt +```dotenv +WEBHOOK_SECRET= +SYNC_API_KEY= +EXPECTED_GITEA_REPOSITORY=KEMT/zpwiki +WEBHOOK_PULL_GIT=false ``` -Vygenerovanie dát a indexu: +Tajomstvá je možné vygenerovať príkazom: ```bash -python scripts/rebuild_index.py +openssl rand -hex 32 ``` -Alternatívne sa dá proces spustiť po krokoch: - -```bash -python scripts/scan_zpwiki.py -python scripts/build_chunks.py -python scripts/build_sqlite_index.py -``` - -Testovanie vyhľadávania v termináli: - -```bash -python scripts/search_db.py "jan ptak" -python scripts/search_db.py "rag agent" -python scripts/search_db.py "knowledge graph" -``` - -Spustenie API lokálne: - -```bash -uvicorn app.main:app --reload -``` - -Health check: - -```bash -curl http://127.0.0.1:8000/health -``` - -Vyhľadávanie cez API: - -```bash -curl -X POST http://127.0.0.1:8000/search \ - -H "Content-Type: application/json" \ - -d '{"query":"jan ptak","limit":5}' -``` +Súbor `.env` sa nesmie commitovať. ## Spustenie cez Docker -Projekt je možné spustiť cez Docker Compose. Kontajner používa volume mount pre priečinok `data` a pre repozitár `zpwiki`. - -Build Docker image: - ```bash docker compose build --no-cache -``` - -Spustenie kontajnera: - -```bash docker compose up -d ``` -Zobrazenie logov: - -```bash -docker compose logs -f zp-agent-api -``` - -Zastavenie kontajnera: - -```bash -docker compose down -``` - -## Reindexovanie cez Docker - -Celý proces indexovania je možné spustiť priamo v Docker kontajneri: - -```bash -docker compose run --rm zp-agent-api python scripts/rebuild_index.py -``` - -Tento príkaz vykoná: - -1. načítanie Markdown dokumentov, -2. extrakciu metadát, -3. rozdelenie dokumentov na chunky, -4. vytvorenie SQLite indexu. - -Po úspešnom behu vzniknú v priečinku `data` súbory: - -```text -documents.json -chunks.json -zp_index.sqlite -``` - -Kontrola dát: - -```bash -ls -lh data -``` - -## Testovanie vyhľadávania cez Docker - -```bash -docker compose run --rm zp-agent-api python scripts/search_db.py "rag agent" -``` - -```bash -docker compose run --rm zp-agent-api python scripts/search_db.py "jan ptak" -``` - -## Testovanie API cez Docker - -Health check: +Kontrola služby: ```bash curl http://127.0.0.1:8000/health ``` -Vyhľadávanie: - -```bash -curl -X POST http://127.0.0.1:8000/search \ - -H "Content-Type: application/json" \ - -d '{"query":"rag agent","limit":5}' -``` - -Manuálne reindexovanie cez API: - -```bash -curl -X POST http://127.0.0.1:8000/sync \ - -H "Content-Type: application/json" \ - -d '{"pull_git":false}' -``` - -## Swagger UI - -FastAPI automaticky generuje Swagger dokumentáciu API. - -Po spustení servera je dostupná na adrese: +Swagger UI: ```text http://127.0.0.1:8000/docs ``` -V Swagger UI je možné testovať endpointy: - -1. `/health`, -2. `/search`, -3. `/sync`, -4. `/webhook/gitea`. - -## Webhook pre Gitea - -Aplikácia obsahuje endpoint: - -```text -POST /webhook/gitea -``` - -Webhook slúži na spustenie reindexovania po zmene v repozitári. - -Endpoint podporuje dva spôsoby overenia: - -1. jednoduchý token cez header `X-Gitea-Token`, -2. HMAC podpis cez header `X-Gitea-Signature`. - -Hodnota tajného kľúča sa nastavuje cez environment premennú: - -```text -WEBHOOK_SECRET -``` - -V `docker-compose.yml` je počas vývoja nastavené: - -```text -WEBHOOK_SECRET=dev-secret -``` - -### Test webhooku cez token +Zastavenie: ```bash -curl -X POST http://127.0.0.1:8000/webhook/gitea \ - -H "Content-Type: application/json" \ - -H "X-Gitea-Event: push" \ - -H "X-Gitea-Token: dev-secret" \ - -d '{"repository":{"full_name":"KEMT/zpwiki"}}' -``` - -### Test webhooku cez HMAC podpis - -```bash -BODY='{"repository":{"full_name":"KEMT/zpwiki"}}' - -SIG=$(printf '%s' "$BODY" | openssl dgst -sha256 -hmac "dev-secret" -hex | sed 's/^.* //') - -curl -X POST http://127.0.0.1:8000/webhook/gitea \ - -H "Content-Type: application/json" \ - -H "X-Gitea-Event: push" \ - -H "X-Gitea-Signature: sha256=$SIG" \ - --data-raw "$BODY" -``` - -### Test neplatného tokenu - -Pri neplatnom tokene má endpoint vrátiť `401 Unauthorized`. - -```bash -curl -i -X POST http://127.0.0.1:8000/webhook/gitea \ - -H "Content-Type: application/json" \ - -H "X-Gitea-Event: push" \ - -H "X-Gitea-Token: zly-token" \ - -d '{"repository":{"full_name":"KEMT/zpwiki"}}' -``` - -## Kompletný test cez Docker - -```bash -cd ~/DP/zp-agent - docker compose down -docker compose build --no-cache - -docker compose run --rm zp-agent-api ls /zpwiki/pages | head - -docker compose run --rm zp-agent-api python scripts/rebuild_index.py - -ls -lh data - -docker compose run --rm zp-agent-api python scripts/search_db.py "rag agent" - -docker compose up -d - -curl http://127.0.0.1:8000/health - -curl -X POST http://127.0.0.1:8000/search \ - -H "Content-Type: application/json" \ - -d '{"query":"rag agent","limit":5}' - -curl -X POST http://127.0.0.1:8000/sync \ - -H "Content-Type: application/json" \ - -d '{"pull_git":false}' ``` -## Čo ešte treba dorobiť +## Reindexovanie -### 1. OpenWebUI integrácia +Celý proces načíta dokumenty, vytvorí chunky a obnoví SQLite FTS5 index: -Treba napojiť API na OpenWebUI. +```bash +docker compose run --rm zp-agent-api python scripts/rebuild_index.py +``` -Možné riešenia: +Vzniknú súbory: -1. OpenAPI tool server, -2. OpenWebUI tool, -3. OpenWebUI pipeline, -4. vlastný agent, ktorý bude volať endpoint `/search`. +```text +data/documents.json +data/chunks.json +data/zp_index.sqlite +``` -Cieľ je, aby používateľ mohol v OpenWebUI položiť otázku a agent použil vyhľadávanie nad `zpwiki`. +## Vyhľadávanie -### 2. Embeddingy a vektorové vyhľadávanie +Test z terminálu: -Aktuálne vyhľadávanie je fulltextové a skórovacie. Ďalší krok je pridať embeddingy. +```bash +docker compose run --rm zp-agent-api python scripts/search_db.py "rag agent" --limit 5 +``` -Treba dorobiť: +Vyhľadávanie cez API: -1. výber embedding modelu, -2. generovanie embeddingov pre chunky, -3. uloženie embeddingov, -4. vektorové vyhľadávanie, -5. porovnanie fulltextového a vektorového vyhľadávania. +```bash +curl -X POST http://127.0.0.1:8000/search -H "Content-Type: application/json" -d '{ + "query": "rag agent", + "limit": 5, + "published_only": false, + "max_per_document": 3 + }' +``` -Možné databázy: +Manuálne reindexovanie cez zabezpečený endpoint: -1. PostgreSQL plus pgvector, -2. Qdrant, -3. ChromaDB, -4. FAISS ako jednoduchý lokálny prototyp. +```bash +curl -X POST http://127.0.0.1:8000/sync -H "Content-Type: application/json" -H "X-API-Key: $SYNC_API_KEY" -d '{"pull_git": false}' +``` -### 3. RAG odpovede s citáciami +## Testy -Treba doplniť generovanie odpovede pomocou jazykového modelu. +Inštalácia testovacích závislostí: -Postup: +```bash +pip install -r requirements-dev.txt +``` -1. používateľ položí otázku, -2. systém nájde relevantné chunky, -3. chunkom priradí zdrojové URL, -4. jazykový model vytvorí odpoveď iba z nájdeného kontextu, -5. odpoveď obsahuje odkazy na zdrojové stránky. +Bežné automatizované testy: -Cieľ je, aby agent nehalucinoval a vedel ukázať, z ktorých dokumentov odpovedal. +```bash +pytest -q test +``` -### 4. Znalostný graf +Testy vrátane kontroly reálne vygenerovaných dát a databázy: -Treba vytvoriť štruktúrovaný graf nad dátami zo `zpwiki`. +```bash +RUN_LIVE_TESTS=1 pytest -q test +``` -Základné entity: +Aktuálna implementácia prešla všetkými 65 testami vrátane live testov. -1. `Student`, -2. `Thesis`, -3. `Tag`, -4. `Category`, -5. `Author`, -6. `Year`. +## Ďalší krok -Základné vzťahy: - -1. študent má prácu, -2. práca má tag, -3. práca patrí do kategórie, -4. autor vedie alebo spravuje prácu, -5. práca je podobná inej práci, -6. práca patrí do roka alebo obdobia. - -### 5. GraphRAG - -Treba prepojiť RAG a znalostný graf. - -GraphRAG časť má umožniť: - -1. vyhľadávanie podľa vzťahov, -2. vysvetlenie, prečo sa našli konkrétne práce, -3. odporúčanie podobných tém, -4. analýzu tém podľa tagov, rokov a kategórií, -5. kombináciu textového, vektorového a grafového vyhľadávania. - -### 6. Čiastočné reindexovanie - -Aktuálne endpoint `/sync` a webhook spúšťajú celé reindexovanie. Neskôr treba doplniť efektívnejší spôsob synchronizácie. - -Plánované časti: - -1. zistenie aktuálneho commitu, -2. detekcia zmenených Markdown súborov, -3. reindexovanie iba zmenených dokumentov, -4. uloženie stavu synchronizácie do databázy, -5. logovanie výsledku synchronizácie. - -### 7. Vyhodnotenie systému - -Treba pripraviť testovaciu sadu otázok a porovnať viacero prístupov. - -Porovnať treba minimálne: - -1. jednoduché fulltextové vyhľadávanie, -2. vektorové vyhľadávanie, -3. RAG, -4. GraphRAG. - -Príklady testovacích otázok: - -1. `Nájdi práce o RAG.` -2. `Nájdi práce podobné téme Agent pre manažment záverečných prác.` -3. `Ktoré práce používajú znalostný graf?` -4. `Kto riešil chatbot alebo agenta?` -5. `Aké témy patria do kategórie dp2027?` -6. `Zhrň práce súvisiace s NLP.` - -Sledované vlastnosti: - -1. relevantnosť výsledkov, -2. správnosť odpovede, -3. správnosť citácií, -4. počet halucinácií, -5. čas odpovede, -6. čas reindexovania po zmene v Gite. - -### 8. Dokumentácia do diplomovej práce - -Treba priebežne písať: - -1. čo je RAG, -2. čo je generatívny model, -3. čo je znalostný graf, -4. čo je GraphRAG, -5. ako funguje `zpwiki`, -6. návrh architektúry systému, -7. návrh databázy a indexu, -8. návrh webhook synchronizácie, -9. návrh integrácie s OpenWebUI, -10. popis experimentov a vyhodnotenia. - -## Najbližší praktický krok - -Najbližšie treba pokračovať integráciou s OpenWebUI a prípravou RAG odpovedí s citáciami. Potom bude možné porovnať jednoduché fulltextové vyhľadávanie s RAG a neskôr s GraphRAG. +Najbližšia etapa je integrácia s OpenWebUI a vytvorenie agentového rozhrania. Následne sa doplnia embeddingy, hybridné vyhľadávanie a RAG odpovede s citáciami. diff --git a/app/main.py b/app/main.py index d60b52a..21e1f01 100644 --- a/app/main.py +++ b/app/main.py @@ -1,13 +1,18 @@ from __future__ import annotations +import asyncio import hashlib import hmac import json import os import sys +from contextlib import asynccontextmanager from pathlib import Path +from typing import Any -from fastapi import FastAPI, Header, HTTPException, Request +from fastapi import Depends, FastAPI, Header, HTTPException, Request, Security, status +from fastapi.responses import JSONResponse +from fastapi.security import APIKeyHeader from pydantic import BaseModel, Field @@ -18,94 +23,253 @@ if str(PROJECT_ROOT) not in sys.path: from scripts.common import DB_FILE, ZPWIKI_ROOT -from scripts.rebuild_index import rebuild_index +from scripts.rebuild_index import ReindexInProgressError, rebuild_index from scripts.search_utils import search_database -WEBHOOK_SECRET = os.getenv("WEBHOOK_SECRET", "dev-secret") +MIN_SECRET_LENGTH = 32 +SYNC_API_KEY_HEADER = "X-API-Key" - -app = FastAPI( - title="ZP Agent API", - description="API pre vyhľadávanie v repozitári záverečných prác zpwiki.", - version="0.4.0", +sync_api_key_scheme = APIKeyHeader( + name=SYNC_API_KEY_HEADER, + auto_error=False, + description="API kľúč pre manuálne spustenie reindexovania.", ) class SearchRequest(BaseModel): - query: str = Field(..., min_length=1) + query: str = Field(..., min_length=1, max_length=500) limit: int = Field(default=10, ge=1, le=50) + published_only: bool = False + max_per_document: int = Field(default=3, ge=0, le=10) class SyncRequest(BaseModel): pull_git: bool = Field( default=False, - description="Ak je true, pred reindexovaním sa vykoná git pull v repozitári zpwiki.", + description="Pred reindexovaním vykoná git pull --ff-only.", ) -def verify_gitea_signature(raw_body: bytes, signature: str | None) -> bool: +def required_environment_value(name: str) -> str: + value = os.getenv(name, "").strip() + + if not value: + raise RuntimeError(f"Chýba povinná environment premenná {name}") + + return value + + +def validate_secret(name: str) -> str: + value = required_environment_value(name) + + if len(value) < MIN_SECRET_LENGTH: + raise RuntimeError( + f"{name} musí mať aspoň {MIN_SECRET_LENGTH} znakov" + ) + + return value + + +def expected_gitea_repository() -> str: + value = required_environment_value("EXPECTED_GITEA_REPOSITORY") + + if "/" not in value: + raise RuntimeError( + "EXPECTED_GITEA_REPOSITORY musí mať tvar vlastník/repozitár" + ) + + return value + + +def webhook_should_pull_git() -> bool: + value = os.getenv("WEBHOOK_PULL_GIT", "false").strip().casefold() + + return value in {"1", "true", "yes", "on"} + + +def validate_security_configuration() -> None: + validate_secret("WEBHOOK_SECRET") + validate_secret("SYNC_API_KEY") + expected_gitea_repository() + + +@asynccontextmanager +async def lifespan(_: FastAPI): + # Aplikácia sa nespustí s chýbajúcim alebo slabým tajomstvom. + validate_security_configuration() + yield + + +app = FastAPI( + title="ZP Agent API", + description="API pre vyhľadávanie v repozitári záverečných prác zpwiki.", + version="0.6.0", + lifespan=lifespan, +) + + +def require_sync_api_key( + api_key: str | None = Security(sync_api_key_scheme), +) -> None: + expected = validate_secret("SYNC_API_KEY") + + if not api_key or not hmac.compare_digest(api_key, expected): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Neplatný alebo chýbajúci API kľúč", + headers={"WWW-Authenticate": "ApiKey"}, + ) + + +def verify_gitea_signature( + raw_body: bytes, + signature: str | None, + secret: str, +) -> bool: if not signature: return False + supplied = signature.strip().casefold() + + # X-Gitea-Signature je čistý hex digest. Prefix prijímame iba + # kvôli kompatibilite s X-Hub-Signature-256. + if supplied.startswith("sha256="): + supplied = supplied.removeprefix("sha256=") + + if len(supplied) != 64: + return False + + try: + int(supplied, 16) + except ValueError: + return False + expected = hmac.new( - WEBHOOK_SECRET.encode("utf-8"), + secret.encode("utf-8"), raw_body, hashlib.sha256, ).hexdigest() - signature = signature.strip() - - if signature.startswith("sha256="): - signature = signature.replace("sha256=", "", 1) - - return hmac.compare_digest(expected, signature) + return hmac.compare_digest(expected, supplied) -def verify_simple_token(token: str | None) -> bool: - if not token: - return False +def repository_name_from_payload( + payload: dict[str, Any], +) -> str | None: + repository = payload.get("repository") - return hmac.compare_digest(token, WEBHOOK_SECRET) + if not isinstance(repository, dict): + return None + + value = ( + repository.get("full_name") + or repository.get("name") + ) + + if not isinstance(value, str): + return None + + value = value.strip() + + return value or None + + +def same_repository( + actual: str, + expected: str, +) -> bool: + return hmac.compare_digest( + actual.casefold(), + expected.casefold(), + ) @app.get("/health") -def health() -> dict: +def health() -> dict[str, Any]: return { "status": "ok", "database_exists": DB_FILE.exists(), "database_path": str(DB_FILE), + "search_engine": "sqlite_fts5", "zpwiki_root": str(ZPWIKI_ROOT), "zpwiki_exists": ZPWIKI_ROOT.exists(), - "webhook_secret_configured": bool(WEBHOOK_SECRET), + "security_configured": all( + bool(os.getenv(name, "").strip()) + for name in ( + "WEBHOOK_SECRET", + "SYNC_API_KEY", + "EXPECTED_GITEA_REPOSITORY", + ) + ), } @app.post("/search") -def search(request: SearchRequest) -> dict: +def search( + request: SearchRequest, +) -> dict[str, Any]: try: - mode, results = search_database( + response = search_database( DB_FILE, request.query, request.limit, + published_only=request.published_only, + max_per_document=request.max_per_document, ) + except FileNotFoundError as error: - raise HTTPException(status_code=500, detail=str(error)) from error + raise HTTPException( + status_code=500, + detail=str(error), + ) from error + + except ValueError as error: + raise HTTPException( + status_code=400, + detail=str(error), + ) from error + + except RuntimeError as error: + raise HTTPException( + status_code=500, + detail=str(error), + ) from error + + results = response["results"] return { "query": request.query, - "mode": mode, + "engine": response["engine"], + "strategies": response["strategies"], "count": len(results), "results": results, } -@app.post("/sync") -def sync(request: SyncRequest) -> dict: +@app.post( + "/sync", + dependencies=[Depends(require_sync_api_key)], +) +def sync( + request: SyncRequest, +) -> dict[str, Any]: try: - result = rebuild_index(pull_git=request.pull_git) + result = rebuild_index( + pull_git=request.pull_git + ) + + except ReindexInProgressError as error: + raise HTTPException( + status_code=409, + detail=str(error), + ) from error + except RuntimeError as error: - raise HTTPException(status_code=500, detail=str(error)) from error + raise HTTPException( + status_code=500, + detail=str(error), + ) from error return { "status": "ok", @@ -115,42 +279,120 @@ def sync(request: SyncRequest) -> dict: } -@app.post("/webhook/gitea") +@app.post( + "/webhook/gitea", + response_model = None, +) async def gitea_webhook( request: Request, - x_gitea_event: str | None = Header(default=None, alias="X-Gitea-Event"), - x_gitea_signature: str | None = Header(default=None, alias="X-Gitea-Signature"), - x_gitea_token: str | None = Header(default=None, alias="X-Gitea-Token"), -) -> dict: + x_gitea_event: str | None = Header( + default=None, + alias="X-Gitea-Event", + ), + x_gitea_signature: str | None = Header( + default=None, + alias="X-Gitea-Signature", + ), +) -> dict[str, Any] | JSONResponse: raw_body = await request.body() + secret = validate_secret("WEBHOOK_SECRET") - signature_ok = verify_gitea_signature(raw_body, x_gitea_signature) - token_ok = verify_simple_token(x_gitea_token) - - if not signature_ok and not token_ok: + if not verify_gitea_signature( + raw_body, + x_gitea_signature, + secret, + ): raise HTTPException( - status_code=401, - detail="Invalid webhook signature or token", + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Neplatný webhook podpis", ) try: - payload = json.loads(raw_body.decode("utf-8")) if raw_body else {} - except json.JSONDecodeError: - payload = {} + payload = json.loads( + raw_body.decode("utf-8") + ) - repository = payload.get("repository", {}) - repository_name = repository.get("full_name") or repository.get("name") + except ( + UnicodeDecodeError, + json.JSONDecodeError, + ) as error: + raise HTTPException( + status_code=400, + detail="Webhook payload nie je platný JSON", + ) from error + + if not isinstance(payload, dict): + raise HTTPException( + status_code=400, + detail="Webhook payload musí byť JSON objekt", + ) + + if not x_gitea_event: + raise HTTPException( + status_code=400, + detail="Chýba hlavička X-Gitea-Event", + ) + + if x_gitea_event.casefold() != "push": + return JSONResponse( + status_code=status.HTTP_202_ACCEPTED, + content={ + "status": "ignored", + "reason": "unsupported_event", + "event": x_gitea_event, + }, + ) + + repository_name = repository_name_from_payload( + payload + ) + + if repository_name is None: + raise HTTPException( + status_code=400, + detail=( + "Webhook payload neobsahuje " + "repository.full_name" + ), + ) + + expected_repository = expected_gitea_repository() + + if not same_repository( + repository_name, + expected_repository, + ): + raise HTTPException( + status_code=403, + detail=( + "Webhook patrí neočakávanému " + "repozitáru" + ), + ) try: - result = rebuild_index(pull_git=False) + result = await asyncio.to_thread( + rebuild_index, + pull_git=webhook_should_pull_git(), + ) + + except ReindexInProgressError as error: + raise HTTPException( + status_code=409, + detail=str(error), + ) from error + except RuntimeError as error: - raise HTTPException(status_code=500, detail=str(error)) from error + raise HTTPException( + status_code=500, + detail=str(error), + ) from error return { "status": "ok", - "event": x_gitea_event or "unknown", + "event": x_gitea_event, "repository": repository_name, - "verified_by": "signature" if signature_ok else "token", + "verified_by": "hmac_sha256", "duration_seconds": result["duration_seconds"], "counts": result["counts"], } diff --git a/docker-compose.yml b/docker-compose.yml index a83890f..83a2249 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,10 +4,19 @@ services: container_name: zp-agent-api ports: - "8000:8000" + + env_file: + - .env + environment: - - ZPWIKI_ROOT=/zpwiki - - WEBHOOK_SECRET=dev-secret + ZPWIKI_ROOT: /zpwiki + CHUNK_MAX_TOKENS: "450" + CHUNK_OVERLAP_TOKENS: "70" + CHUNK_MIN_TOKENS: "80" + CHUNK_TOKEN_ENCODING: cl100k_base + volumes: - ./data:/app/data - ../zpwiki:/zpwiki + restart: unless-stopped diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..d75e054 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt +httpx>=0.27,<1 +pytest>=8,<10 +pytest-cov>=5,<8 diff --git a/requirements.txt b/requirements.txt index 6f1caf5..5d4ad16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,6 @@ -annotated-doc==0.0.4 -annotated-types==0.7.0 -anyio==4.13.0 -click==8.4.1 -exceptiongroup==1.3.1 fastapi==0.136.3 -gitdb==4.0.12 -GitPython==3.1.50 -h11==0.16.0 -idna==3.18 -markdown-it-py==4.2.0 -mdurl==0.1.2 pydantic==2.13.4 -pydantic_core==2.46.4 -Pygments==2.20.0 python-frontmatter==1.3.0 -PyYAML==6.0.3 rich==15.0.0 -smmap==5.0.3 -starlette==1.2.1 -typing-inspection==0.4.2 -typing_extensions==4.15.0 -uvicorn==0.48.0 +tiktoken>=0.8,<1 +uvicorn[standard]==0.48.0 diff --git a/scripts/build_chunks.py b/scripts/build_chunks.py index cd01286..1bc845d 100644 --- a/scripts/build_chunks.py +++ b/scripts/build_chunks.py @@ -1,7 +1,10 @@ from __future__ import annotations +import hashlib +import os import re import sys +from dataclasses import dataclass from pathlib import Path from rich import print @@ -13,107 +16,939 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from scripts.common import CHUNKS_FILE, PAGES_ROOT, ZPWIKI_ROOT, load_zpwiki_page, write_json +from scripts.common import ( + CHUNKS_FILE, + PAGES_ROOT, + ZPWIKI_ROOT, + load_zpwiki_page, + write_json, +) -MAX_CHARS = 1200 -OVERLAP_CHARS = 200 +MAX_TOKENS = int(os.getenv("CHUNK_MAX_TOKENS", "450")) +OVERLAP_TOKENS = int(os.getenv("CHUNK_OVERLAP_TOKENS", "70")) +MIN_CHUNK_TOKENS = int(os.getenv("CHUNK_MIN_TOKENS", "80")) +TOKEN_ENCODING = os.getenv("CHUNK_TOKEN_ENCODING", "cl100k_base") + +HEADING_RE = re.compile(r"^(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$") +FENCE_RE = re.compile(r"^[ \t]*(```+|~~~+)") +LIST_ITEM_RE = re.compile(r"^[ \t]*(?:[-+*]|\d+[.)])[ \t]+") +SENTENCE_SPLIT_RE = re.compile( + r"(?<=[.!?…])(?:[\"'”’»\)\]]*)\s+" + r"(?=[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ0-9])" +) +FALLBACK_TOKEN_RE = re.compile(r"\w+|[^\w\s]", re.UNICODE) + + +@dataclass(frozen=True) +class Section: + heading_path: tuple[str, ...] + body: str + + +@dataclass +class ChunkDraft: + context_lines: list[str] + units: list[str] + heading_paths: list[list[str]] + + @property + def text(self) -> str: + context = "\n".join(self.context_lines).strip() + body = join_units(self.units) + + if context and body: + return f"{context}\n\n{body}" + + return context or body + + +class TokenCounter: + """Použije tiktoken, prípadne jednoduchý lokálny odhad tokenov.""" + + def __init__(self, encoding_name: str) -> None: + self.encoding_name = encoding_name + self.encoding = None + + try: + import tiktoken + + self.encoding = tiktoken.get_encoding(encoding_name) + except (ImportError, ValueError): + self.encoding = None + + @property + def name(self) -> str: + if self.encoding is None: + return "regex fallback" + + return f"tiktoken/{self.encoding_name}" + + def count(self, text: str) -> int: + if not text: + return 0 + + if self.encoding is not None: + return len( + self.encoding.encode( + text, + disallowed_special=(), + ) + ) + + return len(FALLBACK_TOKEN_RE.findall(text)) + + +TOKEN_COUNTER = TokenCounter(TOKEN_ENCODING) def clean_markdown(text: str) -> str: - text = text.replace("\r\n", "\n") + text = text.replace("\r\n", "\n").replace("\r", "\n") + text = re.sub(r"[ \t]+\n", "\n", text) text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() -def split_by_headings(text: str) -> list[str]: - parts = re.split(r"(?m)(?=^#{1,6}\s+)", text) - return [part.strip() for part in parts if part.strip()] +def normalize_label(text: str) -> str: + text = text.casefold().strip() + text = re.sub(r"\s+", " ", text) + + return text -def find_split_position(text: str, max_chars: int) -> int: - """Nájde lepšie miesto delenia, aby chunk nekončil úplne náhodne.""" - if len(text) <= max_chars: - return len(text) +def parse_sections(text: str) -> list[Section]: + """Rozdelí Markdown podľa nadpisov a zachová ich hierarchiu.""" + lines = clean_markdown(text).splitlines() - search_area = text[:max_chars] - min_position = int(max_chars * 0.6) + sections: list[Section] = [] + heading_stack: list[str] = [] + current_path: tuple[str, ...] = () + body_lines: list[str] = [] + active_fence: str | None = None - for separator in ("\n\n", "\n", ". ", " "): - position = search_area.rfind(separator) + def flush() -> None: + body = "\n".join(body_lines).strip() - if position >= min_position: - return position + len(separator) + # Prázdny rodičovský nadpis netvorí samostatný chunk. + # Jeho názov zostane súčasťou heading_path podsekcií. + if body: + sections.append( + Section( + heading_path=current_path, + body=body, + ) + ) - return max_chars + body_lines.clear() + + for line in lines: + fence_match = FENCE_RE.match(line) + + if fence_match: + marker = fence_match.group(1)[0] + + if active_fence == marker: + active_fence = None + else: + active_fence = marker + + body_lines.append(line) + continue + + heading_match = None if active_fence else HEADING_RE.match(line) + + if not heading_match: + body_lines.append(line) + continue + + flush() + + level = len(heading_match.group(1)) + heading = heading_match.group(2).strip() + + heading_stack[:] = heading_stack[: level - 1] + heading_stack.append(heading) + + current_path = tuple(heading_stack) + + flush() + + return sections -def split_long_text( - text: str, - max_chars: int = MAX_CHARS, - overlap: int = OVERLAP_CHARS, -) -> list[str]: - if max_chars <= overlap: - raise ValueError("max_chars musí byť väčšie ako overlap") +def split_markdown_blocks(text: str) -> list[str]: + """Rozdelí text na odseky bez rozbitia fenced code blocku.""" + blocks: list[str] = [] + current: list[str] = [] + active_fence: str | None = None - if len(text) <= max_chars: + def flush() -> None: + block = "\n".join(current).strip() + + if block: + blocks.append(block) + + current.clear() + + for line in text.splitlines(): + fence_match = FENCE_RE.match(line) + + if active_fence is not None: + current.append(line) + + if line.lstrip().startswith(active_fence): + active_fence = None + flush() + + continue + + if fence_match: + flush() + active_fence = fence_match.group(1) + current.append(line) + continue + + if not line.strip(): + flush() + else: + current.append(line) + + flush() + + return blocks + + +def is_code_block(text: str) -> bool: + stripped = text.lstrip() + + return stripped.startswith("```") or stripped.startswith("~~~") + + +def is_table(text: str) -> bool: + lines = [ + line.strip() + for line in text.splitlines() + if line.strip() + ] + + if len(lines) < 2 or "|" not in lines[0]: + return False + + return bool( + re.fullmatch( + r"\|?[ :|-]+\|?", + lines[1], + ) + ) + + +def split_sentences(text: str) -> list[str]: + lines = [ + line.strip() + for line in text.splitlines() + if line.strip() + ] + + if lines and all(LIST_ITEM_RE.match(line) for line in lines): + return lines + + normalized = " ".join(lines) + + if not normalized: + return [] + + return [ + part.strip() + for part in SENTENCE_SPLIT_RE.split(normalized) + if part.strip() + ] + + +def split_by_words(text: str, max_tokens: int) -> list[str]: + """Posledná poistka: text rozdelí iba medzi slovami.""" + words = re.findall(r"\S+", text) + + pieces: list[str] = [] + current: list[str] = [] + + for word in words: + candidate = " ".join([*current, word]) + + if current and TOKEN_COUNTER.count(candidate) > max_tokens: + pieces.append(" ".join(current)) + current = [word] + else: + current.append(word) + + if current: + pieces.append(" ".join(current)) + + return pieces + + +def split_code_block(text: str, max_tokens: int) -> list[str]: + lines = text.splitlines() + + if len(lines) < 3: + return split_by_words(text, max_tokens) + + opening = lines[0] + has_closing = bool(FENCE_RE.match(lines[-1])) + closing = lines[-1] if has_closing else "```" + content = lines[1:-1] if has_closing else lines[1:] + + pieces: list[str] = [] + current: list[str] = [] + + for line in content: + candidate = "\n".join( + [ + opening, + *current, + line, + closing, + ] + ) + + if current and TOKEN_COUNTER.count(candidate) > max_tokens: + pieces.append( + "\n".join( + [ + opening, + *current, + closing, + ] + ) + ) + current = [line] + else: + current.append(line) + + if current: + pieces.append( + "\n".join( + [ + opening, + *current, + closing, + ] + ) + ) + + return pieces or [text] + + +def split_table(text: str, max_tokens: int) -> list[str]: + lines = [ + line + for line in text.splitlines() + if line.strip() + ] + + if len(lines) < 3: return [text] - chunks = [] - start = 0 + header = lines[:2] + rows = lines[2:] - while start < len(text): - remaining = text[start:] + pieces: list[str] = [] + current_rows: list[str] = [] - if len(remaining) <= max_chars: - chunk = remaining.strip() + for row in rows: + candidate = "\n".join( + [ + *header, + *current_rows, + row, + ] + ) - if chunk: - chunks.append(chunk) + if current_rows and TOKEN_COUNTER.count(candidate) > max_tokens: + pieces.append( + "\n".join( + [ + *header, + *current_rows, + ] + ) + ) + current_rows = [row] + else: + current_rows.append(row) + if current_rows: + pieces.append( + "\n".join( + [ + *header, + *current_rows, + ] + ) + ) + + return pieces or [text] + + +def block_to_units(block: str, max_tokens: int) -> list[str]: + """Vytvorí jednotky delené iba na prirodzených hraniciach.""" + if is_code_block(block): + if TOKEN_COUNTER.count(block) <= max_tokens: + return [block] + + return split_code_block(block, max_tokens) + + if is_table(block): + if TOKEN_COUNTER.count(block) <= max_tokens: + return [block] + + return split_table(block, max_tokens) + + units: list[str] = [] + + for sentence in split_sentences(block): + if TOKEN_COUNTER.count(sentence) <= max_tokens: + units.append(sentence) + else: + units.extend( + split_by_words( + sentence, + max_tokens, + ) + ) + + return units + + +def join_units(units: list[str]) -> str: + return "\n\n".join( + unit.strip() + for unit in units + if unit.strip() + ).strip() + + +def unique_strings(values: list[str]) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + + for value in values: + key = normalize_label(value) + + if key and key not in seen: + result.append(value) + seen.add(key) + + return result + + +def unique_paths(paths: list[list[str]]) -> list[list[str]]: + result: list[list[str]] = [] + seen: set[tuple[str, ...]] = set() + + for path in paths: + if not path: + continue + + key = tuple( + normalize_label(item) + for item in path + ) + + if key not in seen: + result.append(path) + seen.add(key) + + return result + + +def get_overlap( + units: list[str], + target_tokens: int, +) -> list[str]: + """Vráti koncové celé vety alebo bloky pre ďalší chunk.""" + if target_tokens <= 0 or len(units) <= 1: + return [] + + selected: list[str] = [] + + for unit in reversed(units): + candidate = [unit, *selected] + + if ( + selected + and TOKEN_COUNTER.count(join_units(candidate)) > target_tokens + ): break - split_at = find_split_position(remaining, max_chars) - chunk = remaining[:split_at].strip() + selected = candidate - if chunk: - chunks.append(chunk) + if TOKEN_COUNTER.count(join_units(selected)) >= target_tokens: + break - start += max(1, split_at - overlap) + # Celý chunk sa nesmie zopakovať ako overlap. + if len(selected) == len(units): + selected = selected[1:] - return chunks + return selected -def chunk_markdown(text: str) -> list[str]: - """Rozdelí Markdown najprv podľa nadpisov a potom podľa dĺžky.""" +def pack_units( + units: list[str], + max_tokens: int, + overlap_tokens: int, +) -> list[list[str]]: + packed: list[list[str]] = [] + current: list[str] = [] + + for unit in units: + candidate = join_units([*current, unit]) + + if current and TOKEN_COUNTER.count(candidate) > max_tokens: + packed.append(current.copy()) + + current = get_overlap( + current, + overlap_tokens, + ) + + while ( + current + and TOKEN_COUNTER.count( + join_units([*current, unit]) + ) + > max_tokens + ): + current.pop(0) + + current.append(unit) + + if current: + packed.append(current.copy()) + + return packed + + +def build_context_lines( + document_title: str | None, + heading_path: tuple[str, ...], +) -> list[str]: + """Pridá názov dokumentu a podľa potreby aj cestu sekcie.""" + title = (document_title or "").strip() + + path = [ + item.strip() + for item in heading_path + if item.strip() + ] + + lines: list[str] = [] + + if title: + lines.append(f"Dokument: {title}") + + if path: + visible_path = path + + # Ak je prvý Markdown nadpis rovnaký ako názov dokumentu, + # v riadku Sekcia ho neopakujeme. + if ( + title + and normalize_label(path[0]) == normalize_label(title) + ): + visible_path = path[1:] + + if visible_path: + lines.append( + f"Sekcia: {' > '.join(visible_path)}" + ) + + return lines + + +def heading_path_list( + heading_path: tuple[str, ...], +) -> list[list[str]]: + if not heading_path: + return [] + + return [list(heading_path)] + + +def chunk_section( + section: Section, + document_title: str | None, +) -> list[ChunkDraft]: + context_lines = build_context_lines( + document_title, + section.heading_path, + ) + + context_tokens = TOKEN_COUNTER.count( + "\n".join(context_lines) + ) + + body_limit = max( + 1, + MAX_TOKENS - context_tokens - 4, + ) + + units: list[str] = [] + + for block in split_markdown_blocks(section.body): + units.extend( + block_to_units( + block, + body_limit, + ) + ) + + if not units: + return [] + + packed_groups = pack_units( + units, + body_limit, + OVERLAP_TOKENS, + ) + + return [ + ChunkDraft( + context_lines=context_lines.copy(), + units=packed_units, + heading_paths=heading_path_list( + section.heading_path + ), + ) + for packed_units in packed_groups + ] + + +def merge_drafts( + first: ChunkDraft, + second: ChunkDraft, +) -> ChunkDraft: + return ChunkDraft( + context_lines=unique_strings( + [ + *first.context_lines, + *second.context_lines, + ] + ), + units=[ + *first.units, + *second.units, + ], + heading_paths=unique_paths( + [ + *first.heading_paths, + *second.heading_paths, + ] + ), + ) + + +def token_count(draft: ChunkDraft) -> int: + return TOKEN_COUNTER.count(draft.text) + + +def try_merge_short_chunks( + drafts: list[ChunkDraft], +) -> tuple[list[ChunkDraft], bool]: + """Najprv skúsi krátky chunk úplne zlúčiť so susedom.""" + result: list[ChunkDraft] = [] + + changed = False + index = 0 + + while index < len(drafts): + current = drafts[index] + + if ( + token_count(current) < MIN_CHUNK_TOKENS + and index + 1 < len(drafts) + ): + combined = merge_drafts( + current, + drafts[index + 1], + ) + + if token_count(combined) <= MAX_TOKENS: + result.append(combined) + index += 2 + changed = True + continue + + if ( + result + and token_count(current) < MIN_CHUNK_TOKENS + ): + combined = merge_drafts( + result[-1], + current, + ) + + if token_count(combined) <= MAX_TOKENS: + result[-1] = combined + index += 1 + changed = True + continue + + result.append(current) + index += 1 + + return result, changed + + +def try_balance_with_next( + current: ChunkDraft, + following: ChunkDraft, +) -> bool: + """Presunie celé úvodné jednotky z ďalšieho chunku do krátkeho.""" + changed = False + + while ( + token_count(current) < MIN_CHUNK_TOKENS + and len(following.units) > 1 + ): + moved_unit = following.units[0] + + candidate = ChunkDraft( + context_lines=unique_strings( + [ + *current.context_lines, + *following.context_lines, + ] + ), + units=[ + *current.units, + moved_unit, + ], + heading_paths=unique_paths( + [ + *current.heading_paths, + *following.heading_paths, + ] + ), + ) + + remaining = ChunkDraft( + context_lines=following.context_lines.copy(), + units=following.units[1:], + heading_paths=[ + path.copy() + for path in following.heading_paths + ], + ) + + if token_count(candidate) > MAX_TOKENS: + break + + # Nevytvoríme nový krátky sused, + # pokiaľ tým reálne nič nezlepšíme. + if token_count(remaining) < MIN_CHUNK_TOKENS: + break + + current.context_lines = candidate.context_lines + current.units = candidate.units + current.heading_paths = candidate.heading_paths + + following.units = remaining.units + + changed = True + + return changed + + +def try_balance_with_previous( + previous: ChunkDraft, + current: ChunkDraft, +) -> bool: + """Presunie celé koncové jednotky z predošlého chunku do krátkeho.""" + changed = False + + while ( + token_count(current) < MIN_CHUNK_TOKENS + and len(previous.units) > 1 + ): + moved_unit = previous.units[-1] + + candidate = ChunkDraft( + context_lines=unique_strings( + [ + *previous.context_lines, + *current.context_lines, + ] + ), + units=[ + moved_unit, + *current.units, + ], + heading_paths=unique_paths( + [ + *previous.heading_paths, + *current.heading_paths, + ] + ), + ) + + remaining = ChunkDraft( + context_lines=previous.context_lines.copy(), + units=previous.units[:-1], + heading_paths=[ + path.copy() + for path in previous.heading_paths + ], + ) + + if token_count(candidate) > MAX_TOKENS: + break + + if token_count(remaining) < MIN_CHUNK_TOKENS: + break + + previous.units = remaining.units + + current.context_lines = candidate.context_lines + current.units = candidate.units + current.heading_paths = candidate.heading_paths + + changed = True + + return changed + + +def balance_short_chunks( + drafts: list[ChunkDraft], +) -> list[ChunkDraft]: + """Zlúči alebo vyváži krátke chunky bez rezania viet a blokov.""" + if len(drafts) < 2 or MIN_CHUNK_TOKENS <= 0: + return drafts + + # Viac prechodov pomôže po zlúčení + # susedných krátkych sekcií. + for _ in range(3): + drafts, merged = try_merge_short_chunks(drafts) + balanced = False + + for index, current in enumerate(drafts): + if token_count(current) >= MIN_CHUNK_TOKENS: + continue + + if index + 1 < len(drafts): + balanced = ( + try_balance_with_next( + current, + drafts[index + 1], + ) + or balanced + ) + + if ( + token_count(current) < MIN_CHUNK_TOKENS + and index > 0 + ): + balanced = ( + try_balance_with_previous( + drafts[index - 1], + current, + ) + or balanced + ) + + if not merged and not balanced: + break + + return drafts + + +def validate_settings() -> None: + if MAX_TOKENS <= 0: + raise ValueError( + "CHUNK_MAX_TOKENS musí byť väčšie ako 0" + ) + + if ( + OVERLAP_TOKENS < 0 + or OVERLAP_TOKENS >= MAX_TOKENS + ): + raise ValueError( + "CHUNK_OVERLAP_TOKENS musí byť od 0 " + "po MAX_TOKENS - 1" + ) + + if ( + MIN_CHUNK_TOKENS < 0 + or MIN_CHUNK_TOKENS >= MAX_TOKENS + ): + raise ValueError( + "CHUNK_MIN_TOKENS musí byť od 0 " + "po MAX_TOKENS - 1" + ) + + +def chunk_markdown( + text: str, + document_title: str | None = None, +) -> list[ChunkDraft]: + validate_settings() + text = clean_markdown(text) if not text: return [] - chunks = [] + drafts: list[ChunkDraft] = [] - for part in split_by_headings(text): - chunks.extend(split_long_text(part)) + for section in parse_sections(text): + drafts.extend( + chunk_section( + section, + document_title, + ) + ) - return chunks + return balance_short_chunks(drafts) + + +def content_hash( + document_path: str, + text: str, +) -> str: + value = f"{document_path}\n{text}".encode("utf-8") + + return hashlib.sha256(value).hexdigest()[:16] def build_chunks() -> list[dict]: if not PAGES_ROOT.exists(): - raise SystemExit(f"Neexistuje priečinok: {PAGES_ROOT}") + raise SystemExit( + f"Neexistuje priečinok: {PAGES_ROOT}" + ) - all_chunks = [] + all_chunks: list[dict] = [] document_count = 0 - for file_path in sorted(PAGES_ROOT.glob("**/README.md")): + for file_path in sorted( + PAGES_ROOT.glob("**/README.md") + ): document = load_zpwiki_page(file_path) document_count += 1 - for index, text in enumerate(chunk_markdown(document["content"])): + drafts = chunk_markdown( + document["content"], + document_title=document.get("title"), + ) + + for index, draft in enumerate(drafts): + text = draft.text + token_total = TOKEN_COUNTER.count(text) + all_chunks.append( { - "chunk_id": f"{document['path']}::chunk-{index}", + "chunk_id": ( + f"{document['path']}::chunk-{index}" + ), "document_path": document["path"], "title": document["title"], "categories": document["categories"], @@ -121,20 +956,65 @@ def build_chunks() -> list[dict]: "author": document["author"], "published": document["published"], "chunk_index": index, + "heading_paths": draft.heading_paths, "text": text, "text_length": len(text), + "token_count": token_total, + "content_hash": content_hash( + document["path"], + text, + ), } ) - write_json(CHUNKS_FILE, all_chunks) + write_json( + CHUNKS_FILE, + all_chunks, + ) - print(f"[green]ZPWIKI_ROOT:[/green] {ZPWIKI_ROOT}") - print(f"[green]Dokumentov:[/green] {document_count}") - print(f"[green]Chunkov:[/green] {len(all_chunks)}") - print(f"[green]Výstup uložený do:[/green] {CHUNKS_FILE}") + short_count = sum( + chunk["token_count"] < MIN_CHUNK_TOKENS + for chunk in all_chunks + ) + + oversized_count = sum( + chunk["token_count"] > MAX_TOKENS + for chunk in all_chunks + ) + + print( + f"[green]ZPWIKI_ROOT:[/green] " + f"{ZPWIKI_ROOT}" + ) + print( + f"[green]Tokenizer:[/green] " + f"{TOKEN_COUNTER.name}" + ) + print( + f"[green]Dokumentov:[/green] " + f"{document_count}" + ) + print( + f"[green]Chunkov:[/green] " + f"{len(all_chunks)}" + ) + print( + f"[green]Krátkych chunkov:[/green] " + f"{short_count}" + ) + print( + f"[green]Príliš veľkých chunkov:[/green] " + f"{oversized_count}" + ) + print( + f"[green]Výstup uložený do:[/green] " + f"{CHUNKS_FILE}" + ) if all_chunks: - print("\n[bold]Ukážka prvého chunku:[/bold]") + print( + "\n[bold]Ukážka prvého chunku:[/bold]" + ) print(all_chunks[0]) return all_chunks diff --git a/scripts/build_sqlite_index.py b/scripts/build_sqlite_index.py index b39fba5..0ac0bb0 100644 --- a/scripts/build_sqlite_index.py +++ b/scripts/build_sqlite_index.py @@ -1,9 +1,11 @@ from __future__ import annotations import json +import os import sqlite3 import sys from pathlib import Path +from typing import Any from rich import print @@ -17,76 +19,147 @@ if str(PROJECT_ROOT) not in sys.path: from scripts.common import CHUNKS_FILE, DB_FILE, DOCUMENTS_FILE, read_json -def create_tables(conn: sqlite3.Connection) -> None: - cursor = conn.cursor() +FTS_TOKENIZER = "unicode61 remove_diacritics 2" +FTS_PREFIXES = "3 4 5" - cursor.executescript( - """ + +def published_to_db(value: Any) -> int | None: + if value is True: + return 1 + + if value is False: + return 0 + + return None + + +def verify_fts5(conn: sqlite3.Connection) -> None: + """Overí, či aktuálna SQLite knižnica podporuje FTS5.""" + try: + conn.execute( + "CREATE VIRTUAL TABLE temp.fts5_check USING fts5(value)" + ) + conn.execute("DROP TABLE temp.fts5_check") + except sqlite3.OperationalError as error: + raise RuntimeError( + "Táto inštalácia SQLite nemá dostupné FTS5. " + "Použi Python/SQLite zostavenie s podporou SQLITE_ENABLE_FTS5." + ) from error + + +def create_tables(conn: sqlite3.Connection) -> None: + conn.executescript( + f""" PRAGMA foreign_keys = ON; - DROP TABLE IF EXISTS chunk_tags; - DROP TABLE IF EXISTS chunk_categories; - DROP TABLE IF EXISTS chunks; - DROP TABLE IF EXISTS documents; - CREATE TABLE documents ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, path TEXT UNIQUE NOT NULL, title TEXT, author TEXT, - published INTEGER, - content_length INTEGER, - metadata_json TEXT + published INTEGER + CHECK (published IN (0, 1) OR published IS NULL), + content_length INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{{}}' ); CREATE TABLE chunks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, chunk_id TEXT UNIQUE NOT NULL, document_path TEXT NOT NULL, title TEXT, author TEXT, - chunk_index INTEGER, + published INTEGER + CHECK (published IN (0, 1) OR published IS NULL), + chunk_index INTEGER NOT NULL, + heading_paths_json TEXT NOT NULL DEFAULT '[]', text TEXT NOT NULL, - text_length INTEGER, - FOREIGN KEY(document_path) REFERENCES documents(path) + text_length INTEGER NOT NULL DEFAULT 0, + token_count INTEGER, + content_hash TEXT, + FOREIGN KEY(document_path) + REFERENCES documents(path) + ON UPDATE CASCADE + ON DELETE CASCADE ); CREATE TABLE chunk_tags ( chunk_id TEXT NOT NULL, tag TEXT NOT NULL, - UNIQUE(chunk_id, tag), - FOREIGN KEY(chunk_id) REFERENCES chunks(chunk_id) + PRIMARY KEY(chunk_id, tag), + FOREIGN KEY(chunk_id) + REFERENCES chunks(chunk_id) + ON UPDATE CASCADE + ON DELETE CASCADE ); CREATE TABLE chunk_categories ( chunk_id TEXT NOT NULL, category TEXT NOT NULL, - UNIQUE(chunk_id, category), - FOREIGN KEY(chunk_id) REFERENCES chunks(chunk_id) + PRIMARY KEY(chunk_id, category), + FOREIGN KEY(chunk_id) + REFERENCES chunks(chunk_id) + ON UPDATE CASCADE + ON DELETE CASCADE ); - CREATE INDEX idx_documents_path ON documents(path); - CREATE INDEX idx_chunks_document_path ON chunks(document_path); - CREATE INDEX idx_chunks_title ON chunks(title); - CREATE INDEX idx_chunk_tags_tag ON chunk_tags(tag); - CREATE INDEX idx_chunk_categories_category ON chunk_categories(category); + CREATE INDEX idx_documents_path + ON documents(path); + + CREATE INDEX idx_documents_published + ON documents(published); + + CREATE INDEX idx_chunks_document_path + ON chunks(document_path); + + CREATE INDEX idx_chunks_title + ON chunks(title); + + CREATE INDEX idx_chunks_author + ON chunks(author); + + CREATE INDEX idx_chunks_published + ON chunks(published); + + CREATE INDEX idx_chunk_tags_tag + ON chunk_tags(tag); + + CREATE INDEX idx_chunk_categories_category + ON chunk_categories(category); + + CREATE VIRTUAL TABLE chunks_fts USING fts5( + chunk_id UNINDEXED, + title, + author, + document_path, + tags, + categories, + text, + tokenize='{FTS_TOKENIZER}', + prefix='{FTS_PREFIXES}' + ); """ ) - conn.commit() - -def insert_documents(conn: sqlite3.Connection, documents: list[dict]) -> None: +def insert_documents( + conn: sqlite3.Connection, + documents: list[dict], +) -> None: rows = [ ( - doc.get("path"), - doc.get("title"), - doc.get("author"), - 1 if doc.get("published") else 0, - doc.get("content_length"), - json.dumps(doc.get("metadata") or {}, ensure_ascii=False), + document.get("path"), + document.get("title"), + document.get("author"), + published_to_db(document.get("published")), + int(document.get("content_length") or 0), + json.dumps( + document.get("metadata") or {}, + ensure_ascii=False, + sort_keys=True, + ), ) - for doc in documents + for document in documents ] conn.executemany( @@ -104,16 +177,24 @@ def insert_documents(conn: sqlite3.Connection, documents: list[dict]) -> None: rows, ) - conn.commit() - -def insert_chunks(conn: sqlite3.Connection, chunks: list[dict]) -> None: - chunk_rows = [] - tag_rows = [] - category_rows = [] +def insert_chunks( + conn: sqlite3.Connection, + chunks: list[dict], +) -> None: + chunk_rows: list[tuple] = [] + tag_rows: list[tuple[str, str]] = [] + category_rows: list[tuple[str, str]] = [] for chunk in chunks: - chunk_id = chunk.get("chunk_id") + chunk_id = str(chunk.get("chunk_id") or "").strip() + + if not chunk_id: + raise ValueError( + "Chunk bez chunk_id nie je možné indexovať" + ) + + text = chunk.get("text") or "" chunk_rows.append( ( @@ -121,17 +202,30 @@ def insert_chunks(conn: sqlite3.Connection, chunks: list[dict]) -> None: chunk.get("document_path"), chunk.get("title"), chunk.get("author"), - chunk.get("chunk_index"), - chunk.get("text"), - chunk.get("text_length"), + published_to_db(chunk.get("published")), + int(chunk.get("chunk_index") or 0), + json.dumps( + chunk.get("heading_paths") or [], + ensure_ascii=False, + ), + text, + int(chunk.get("text_length") or len(text)), + chunk.get("token_count"), + chunk.get("content_hash"), ) ) for tag in chunk.get("tags") or []: - tag_rows.append((chunk_id, tag)) + value = str(tag).strip() + + if value: + tag_rows.append((chunk_id, value)) for category in chunk.get("categories") or []: - category_rows.append((chunk_id, category)) + value = str(category).strip() + + if value: + category_rows.append((chunk_id, value)) conn.executemany( """ @@ -140,18 +234,25 @@ def insert_chunks(conn: sqlite3.Connection, chunks: list[dict]) -> None: document_path, title, author, + published, chunk_index, + heading_paths_json, text, - text_length + text_length, + token_count, + content_hash ) - VALUES (?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, chunk_rows, ) conn.executemany( """ - INSERT OR IGNORE INTO chunk_tags (chunk_id, tag) + INSERT OR IGNORE INTO chunk_tags ( + chunk_id, + tag + ) VALUES (?, ?) """, tag_rows, @@ -159,44 +260,195 @@ def insert_chunks(conn: sqlite3.Connection, chunks: list[dict]) -> None: conn.executemany( """ - INSERT OR IGNORE INTO chunk_categories (chunk_id, category) + INSERT OR IGNORE INTO chunk_categories ( + chunk_id, + category + ) VALUES (?, ?) """, category_rows, ) - conn.commit() + +def build_fts_index(conn: sqlite3.Connection) -> None: + """Vytvorí FTS5 index nad chunkmi a ich metadátami.""" + conn.execute( + """ + INSERT INTO chunks_fts ( + rowid, + chunk_id, + title, + author, + document_path, + tags, + categories, + text + ) + SELECT + chunks.id, + chunks.chunk_id, + COALESCE(chunks.title, ''), + COALESCE(chunks.author, ''), + chunks.document_path, + COALESCE(tags.values_text, ''), + COALESCE(categories.values_text, ''), + chunks.text + FROM chunks + LEFT JOIN ( + SELECT + chunk_id, + GROUP_CONCAT(tag, ' ') AS values_text + FROM chunk_tags + GROUP BY chunk_id + ) AS tags + ON tags.chunk_id = chunks.chunk_id + LEFT JOIN ( + SELECT + chunk_id, + GROUP_CONCAT(category, ' ') AS values_text + FROM chunk_categories + GROUP BY chunk_id + ) AS categories + ON categories.chunk_id = chunks.chunk_id + ORDER BY chunks.id + """ + ) + + conn.execute( + "INSERT INTO chunks_fts(chunks_fts) VALUES('optimize')" + ) -def get_counts(conn: sqlite3.Connection) -> dict[str, int]: - cursor = conn.cursor() +def validate_database(conn: sqlite3.Connection) -> None: + integrity = conn.execute( + "PRAGMA integrity_check" + ).fetchone()[0] + if integrity != "ok": + raise RuntimeError( + f"SQLite integrity check zlyhal: {integrity}" + ) + + foreign_key_errors = conn.execute( + "PRAGMA foreign_key_check" + ).fetchall() + + if foreign_key_errors: + raise RuntimeError( + "Databáza obsahuje chyby cudzích kľúčov: " + f"{foreign_key_errors[:5]}" + ) + + conn.execute( + "INSERT INTO chunks_fts(chunks_fts) " + "VALUES('integrity-check')" + ) + + chunk_count = conn.execute( + "SELECT COUNT(*) FROM chunks" + ).fetchone()[0] + + fts_count = conn.execute( + "SELECT COUNT(*) FROM chunks_fts" + ).fetchone()[0] + + if chunk_count != fts_count: + raise RuntimeError( + "Počet záznamov v chunks a chunks_fts sa nezhoduje: " + f"{chunk_count} != {fts_count}" + ) + + +def get_counts( + conn: sqlite3.Connection, +) -> dict[str, int]: return { - "documents": cursor.execute("SELECT COUNT(*) FROM documents").fetchone()[0], - "chunks": cursor.execute("SELECT COUNT(*) FROM chunks").fetchone()[0], - "tags": cursor.execute("SELECT COUNT(*) FROM chunk_tags").fetchone()[0], - "categories": cursor.execute("SELECT COUNT(*) FROM chunk_categories").fetchone()[0], + "documents": conn.execute( + "SELECT COUNT(*) FROM documents" + ).fetchone()[0], + "chunks": conn.execute( + "SELECT COUNT(*) FROM chunks" + ).fetchone()[0], + "fts_chunks": conn.execute( + "SELECT COUNT(*) FROM chunks_fts" + ).fetchone()[0], + "tags": conn.execute( + "SELECT COUNT(*) FROM chunk_tags" + ).fetchone()[0], + "categories": conn.execute( + "SELECT COUNT(*) FROM chunk_categories" + ).fetchone()[0], } +def temporary_database_path(db_file: Path) -> Path: + return db_file.with_name( + f".{db_file.name}.tmp" + ) + + def build_database() -> dict[str, int]: documents = read_json(DOCUMENTS_FILE) chunks = read_json(CHUNKS_FILE) - DB_FILE.parent.mkdir(parents=True, exist_ok=True) + DB_FILE.parent.mkdir( + parents=True, + exist_ok=True, + ) - with sqlite3.connect(DB_FILE) as conn: - conn.execute("PRAGMA foreign_keys = ON") - create_tables(conn) - insert_documents(conn, documents) - insert_chunks(conn, chunks) - counts = get_counts(conn) + temporary_file = temporary_database_path(DB_FILE) - print(f"[green]SQLite index vytvorený:[/green] {DB_FILE}") - print(f"Dokumentov: {counts['documents']}") - print(f"Chunkov: {counts['chunks']}") - print(f"Tag záznamov: {counts['tags']}") - print(f"Kategória záznamov: {counts['categories']}") + if temporary_file.exists(): + temporary_file.unlink() + + try: + with sqlite3.connect(temporary_file) as conn: + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("PRAGMA temp_store = MEMORY") + + verify_fts5(conn) + + with conn: + create_tables(conn) + insert_documents(conn, documents) + insert_chunks(conn, chunks) + build_fts_index(conn) + + validate_database(conn) + counts = get_counts(conn) + + # Nová databáza nahradí starú až po úspešnom vytvorení. + os.replace( + temporary_file, + DB_FILE, + ) + + except Exception: + if temporary_file.exists(): + temporary_file.unlink() + + raise + + print( + f"[green]SQLite index vytvorený:[/green] " + f"{DB_FILE}" + ) + print( + f"Dokumentov: {counts['documents']}" + ) + print( + f"Chunkov: {counts['chunks']}" + ) + print( + f"FTS5 chunkov: {counts['fts_chunks']}" + ) + print( + f"Tag záznamov: {counts['tags']}" + ) + print( + f"Kategória záznamov: " + f"{counts['categories']}" + ) return counts diff --git a/scripts/common.py b/scripts/common.py index 14cc838..c6fe09c 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1,7 +1,9 @@ from __future__ import annotations +import html import json import os +import re from pathlib import Path from typing import Any @@ -9,7 +11,14 @@ import frontmatter PROJECT_ROOT = Path(__file__).resolve().parents[1] -ZPWIKI_ROOT = Path(os.getenv("ZPWIKI_ROOT", str(PROJECT_ROOT.parent / "zpwiki"))).resolve() + +ZPWIKI_ROOT = Path( + os.getenv( + "ZPWIKI_ROOT", + str(PROJECT_ROOT.parent / "zpwiki"), + ) +).resolve() + PAGES_ROOT = ZPWIKI_ROOT / "pages" DATA_DIR = PROJECT_ROOT / "data" @@ -18,16 +27,68 @@ CHUNKS_FILE = DATA_DIR / "chunks.json" DB_FILE = DATA_DIR / "zp_index.sqlite" +HEADING_RE = re.compile( + r"^[ \t]{0,3}#{1,6}[ \t]+(.+?)[ \t]*#*[ \t]*$" +) + +FENCE_RE = re.compile( + r"^[ \t]{0,3}(```+|~~~+)" +) + +MARKDOWN_IMAGE_RE = re.compile( + r"!\[([^\]]*)\]\([^)]*\)" +) + +MARKDOWN_LINK_RE = re.compile( + r"\[([^\]]+)\]\([^)]*\)" +) + +HTML_TAG_RE = re.compile( + r"<[^>]+>" +) + +WHITESPACE_RE = re.compile( + r"\s+" +) + + +TRUE_VALUES = { + "1", + "true", + "yes", + "on", + "ano", + "áno", +} + +FALSE_VALUES = { + "0", + "false", + "no", + "off", + "nie", +} + + def json_safe(value: Any) -> Any: """Prevedie metadata do formátu vhodného pre JSON.""" - if value is None or isinstance(value, (str, int, float, bool)): + if value is None or isinstance( + value, + (str, int, float, bool), + ): return value if isinstance(value, list): - return [json_safe(item) for item in value] + return [ + json_safe(item) + for item in value + ] if isinstance(value, dict): - return {str(key): json_safe(item) for key, item in value.items()} + return { + str(key): json_safe(item) + for key, item in value.items() + } return str(value) @@ -38,14 +99,24 @@ def normalize_list(value: Any) -> list[str]: return [] if isinstance(value, list): - raw_items = [str(item).strip() for item in value] - elif isinstance(value, str): - raw_items = [item.strip() for item in value.split(",")] - else: - raw_items = [str(value).strip()] + raw_items = [ + str(item).strip() + for item in value + ] - items = [] - seen = set() + elif isinstance(value, str): + raw_items = [ + item.strip() + for item in value.split(",") + ] + + else: + raw_items = [ + str(value).strip() + ] + + items: list[str] = [] + seen: set[str] = set() for item in raw_items: if item and item not in seen: @@ -55,22 +126,209 @@ def normalize_list(value: Any) -> list[str]: return items +def normalize_optional_bool( + value: Any, +) -> bool | None: + """Normalizuje bežné YAML reprezentácie true/false.""" + if value is None: + return None + + if isinstance(value, bool): + return value + + if isinstance(value, int) and value in {0, 1}: + return bool(value) + + if isinstance(value, str): + normalized = value.strip().casefold() + + if normalized in TRUE_VALUES: + return True + + if normalized in FALSE_VALUES: + return False + + return None + + def read_json(path: Path) -> Any: if not path.exists(): - raise FileNotFoundError(f"Súbor neexistuje: {path}") + raise FileNotFoundError( + f"Súbor neexistuje: {path}" + ) - with path.open("r", encoding="utf-8") as file: + with path.open( + "r", + encoding="utf-8", + ) as file: return json.load(file) -def write_json(path: Path, data: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) +def write_json( + path: Path, + data: Any, +) -> None: + path.parent.mkdir( + parents=True, + exist_ok=True, + ) - with path.open("w", encoding="utf-8") as file: - json.dump(data, file, ensure_ascii=False, indent=2) + with path.open( + "w", + encoding="utf-8", + ) as file: + json.dump( + data, + file, + ensure_ascii=False, + indent=2, + ) -def load_zpwiki_page(file_path: Path) -> dict[str, Any]: +def clean_heading_text( + value: str, +) -> str: + """Odstráni základné Markdown značky z nadpisu.""" + value = html.unescape(value) + + value = MARKDOWN_IMAGE_RE.sub( + r"\1", + value, + ) + + value = MARKDOWN_LINK_RE.sub( + r"\1", + value, + ) + + value = HTML_TAG_RE.sub( + "", + value, + ) + + value = value.replace("`", "") + value = value.replace("*", "") + value = value.replace("_", " ") + value = value.replace("~", "") + + return WHITESPACE_RE.sub( + " ", + value, + ).strip() + + +def first_markdown_heading( + content: str, +) -> str | None: + """ + Nájde prvý Markdown nadpis mimo fenced code blockov. + """ + active_fence: str | None = None + + for line in content.splitlines(): + fence_match = FENCE_RE.match(line) + + if fence_match: + marker = fence_match.group(1)[0] + + if active_fence == marker: + active_fence = None + + elif active_fence is None: + active_fence = marker + + continue + + if active_fence is not None: + continue + + heading_match = HEADING_RE.match(line) + + if not heading_match: + continue + + heading = clean_heading_text( + heading_match.group(1) + ) + + if heading: + return heading + + return None + + +def parent_directory_title( + file_path: Path, +) -> str | None: + """Vytvorí názov z rodičovského priečinka.""" + directory_name = file_path.parent.name.strip() + + if not directory_name: + return None + + readable = re.sub( + r"[_-]+", + " ", + directory_name, + ) + + readable = WHITESPACE_RE.sub( + " ", + readable, + ).strip() + + return readable.title() or None + + +def resolve_page_title( + file_path: Path, + metadata: dict[str, Any], + content: str, +) -> str: + """ + Určí názov dokumentu v tomto poradí: + + 1. YAML title, + 2. prvý Markdown nadpis, + 3. rodičovský priečinok, + 4. cesta k súboru. + """ + metadata_title = metadata.get("title") + + if metadata_title is not None: + title = str(metadata_title).strip() + + if title: + return title + + heading_title = first_markdown_heading( + content + ) + + if heading_title: + return heading_title + + directory_title = parent_directory_title( + file_path + ) + + if directory_title: + return directory_title + + try: + relative_path = file_path.relative_to( + ZPWIKI_ROOT + ) + + except ValueError: + relative_path = file_path + + return str(relative_path) + + +def load_zpwiki_page( + file_path: Path, +) -> dict[str, Any]: post = frontmatter.load(file_path) metadata = { @@ -78,7 +336,13 @@ def load_zpwiki_page(file_path: Path) -> dict[str, Any]: for key, value in post.metadata.items() } - taxonomy = metadata.get("taxonomy") or {} + raw_taxonomy = metadata.get("taxonomy") + + taxonomy = ( + raw_taxonomy + if isinstance(raw_taxonomy, dict) + else {} + ) categories = normalize_list( metadata.get("category") @@ -92,14 +356,29 @@ def load_zpwiki_page(file_path: Path) -> dict[str, Any]: or taxonomy.get("tags") ) + content = post.content.strip() + + title = resolve_page_title( + file_path, + metadata, + content, + ) + return { - "path": str(file_path.relative_to(ZPWIKI_ROOT)), - "title": metadata.get("title"), + "path": str( + file_path.relative_to(ZPWIKI_ROOT) + ), + "title": title, "categories": categories, "tags": tags, - "published": metadata.get("published"), - "author": metadata.get("author") or taxonomy.get("author"), + "published": normalize_optional_bool( + metadata.get("published") + ), + "author": ( + metadata.get("author") + or taxonomy.get("author") + ), "taxonomy": taxonomy, "metadata": metadata, - "content": post.content.strip(), + "content": content, } diff --git a/scripts/rebuild_index.py b/scripts/rebuild_index.py index a6f0fec..369fd66 100644 --- a/scripts/rebuild_index.py +++ b/scripts/rebuild_index.py @@ -1,10 +1,16 @@ from __future__ import annotations import argparse +import errno +import fcntl +import json +import os import subprocess import sys import time +from contextlib import contextmanager from pathlib import Path +from typing import Iterator, TextIO from rich import print @@ -17,23 +23,110 @@ if str(PROJECT_ROOT) not in sys.path: from scripts.build_chunks import build_chunks from scripts.build_sqlite_index import build_database -from scripts.common import DB_FILE, ZPWIKI_ROOT +from scripts.common import DATA_DIR, DB_FILE, ZPWIKI_ROOT from scripts.scan_zpwiki import scan_pages -def git_pull(repo_path: Path = ZPWIKI_ROOT) -> None: +REINDEX_LOCK_FILE = DATA_DIR / ".reindex.lock" +GIT_PULL_TIMEOUT_SECONDS = 120 + + +class ReindexInProgressError(RuntimeError): + """Iný proces už drží zámok reindexovania.""" + + +@contextmanager +def acquire_reindex_lock( + lock_file: Path = REINDEX_LOCK_FILE, +) -> Iterator[TextIO]: + """Získa neblokujúci procesový zámok nad spoločným data volume.""" + lock_file.parent.mkdir( + parents=True, + exist_ok=True, + ) + + handle = lock_file.open( + "a+", + encoding="utf-8", + ) + + try: + try: + fcntl.flock( + handle.fileno(), + fcntl.LOCK_EX | fcntl.LOCK_NB, + ) + + except OSError as error: + if error.errno in { + errno.EACCES, + errno.EAGAIN, + }: + raise ReindexInProgressError( + "Reindexovanie už prebieha" + ) from error + + raise + + handle.seek(0) + handle.truncate() + + json.dump( + { + "pid": os.getpid(), + "started_at_unix": time.time(), + }, + handle, + ) + + handle.flush() + + yield handle + + finally: + try: + fcntl.flock( + handle.fileno(), + fcntl.LOCK_UN, + ) + except OSError: + pass + + handle.close() + + +def git_pull( + repo_path: Path = ZPWIKI_ROOT, +) -> None: if not repo_path.exists(): - raise RuntimeError(f"ZPWIKI_ROOT neexistuje: {repo_path}") + raise RuntimeError( + f"ZPWIKI_ROOT neexistuje: {repo_path}" + ) if not (repo_path / ".git").exists(): - raise RuntimeError(f"Nie je to git repozitár: {repo_path}") + raise RuntimeError( + f"Nie je to git repozitár: {repo_path}" + ) - result = subprocess.run( - ["git", "pull"], - cwd=repo_path, - text=True, - capture_output=True, - ) + try: + result = subprocess.run( + [ + "git", + "pull", + "--ff-only", + ], + cwd=repo_path, + text=True, + capture_output=True, + timeout=GIT_PULL_TIMEOUT_SECONDS, + check=False, + ) + + except subprocess.TimeoutExpired as error: + raise RuntimeError( + "Git pull prekročil limit " + f"{GIT_PULL_TIMEOUT_SECONDS} sekúnd" + ) from error if result.stdout: print(result.stdout.strip()) @@ -42,53 +135,105 @@ def git_pull(repo_path: Path = ZPWIKI_ROOT) -> None: print(result.stderr.strip()) if result.returncode != 0: - raise RuntimeError("Git pull zlyhal") + raise RuntimeError( + "Git pull zlyhal s návratovým kódom " + f"{result.returncode}" + ) -def rebuild_index(pull_git: bool = False) -> dict: - start = time.time() +def rebuild_index( + pull_git: bool = False, + *, + lock_file: Path = REINDEX_LOCK_FILE, +) -> dict: + with acquire_reindex_lock(lock_file): + start = time.monotonic() - print(f"[green]ZPWIKI_ROOT:[/green] {ZPWIKI_ROOT}") + print( + f"[green]ZPWIKI_ROOT:[/green] " + f"{ZPWIKI_ROOT}" + ) - if pull_git: - git_pull() + if pull_git: + git_pull() - documents = scan_pages() - chunks = build_chunks() - counts = build_database() + documents = scan_pages() + chunks = build_chunks() + counts = build_database() - duration = round(time.time() - start, 2) + duration = round( + time.monotonic() - start, + 2, + ) - return { - "duration_seconds": duration, - "documents_scanned": len(documents), - "chunks_created": len(chunks), - "counts": counts, - "database_path": str(DB_FILE), - } + return { + "duration_seconds": duration, + "documents_scanned": len(documents), + "chunks_created": len(chunks), + "counts": counts, + "database_path": str(DB_FILE), + } def main() -> None: parser = argparse.ArgumentParser( - description="Obnoví JSON súbory a SQLite index." + description=( + "Obnoví JSON súbory " + "a SQLite FTS5 index." + ) ) parser.add_argument( "--pull", action="store_true", - help="Pred reindexovaním spustí git pull v zpwiki repozitári.", + help=( + "Pred reindexovaním spustí " + "git pull --ff-only." + ), ) args = parser.parse_args() - result = rebuild_index(pull_git=args.pull) + + try: + result = rebuild_index( + pull_git=args.pull + ) + + except ReindexInProgressError as error: + raise SystemExit( + str(error) + ) from error + counts = result["counts"] print("[green]Reindex hotový.[/green]") - print(f"Trvanie: {result['duration_seconds']} s") - print(f"Dokumentov: {counts['documents']}") - print(f"Chunkov: {counts['chunks']}") - print(f"Tag záznamov: {counts['tags']}") - print(f"Kategória záznamov: {counts['categories']}") + print( + f"Trvanie: " + f"{result['duration_seconds']} s" + ) + print( + f"Dokumentov: " + f"{counts['documents']}" + ) + print( + f"Chunkov: " + f"{counts['chunks']}" + ) + + if "fts_chunks" in counts: + print( + f"FTS5 chunkov: " + f"{counts['fts_chunks']}" + ) + + print( + f"Tag záznamov: " + f"{counts['tags']}" + ) + print( + f"Kategória záznamov: " + f"{counts['categories']}" + ) if __name__ == "__main__": diff --git a/scripts/search_db.py b/scripts/search_db.py index a82d439..2732b0c 100644 --- a/scripts/search_db.py +++ b/scripts/search_db.py @@ -17,29 +17,107 @@ from scripts.common import DB_FILE from scripts.search_utils import search_database -def print_results(query: str, mode: str, results: list[dict]) -> None: - print(f"[bold]Dopyt:[/bold] {query}") - print(f"[bold]Režim:[/bold] {mode}") - print(f"[bold]Počet výsledkov:[/bold] {len(results)}") - print("\n[bold]Top výsledky:[/bold]\n") +def print_results( + query: str, + response: dict, +) -> None: + results = response["results"] + + print( + f"[bold]Dopyt:[/bold] " + f"{query}" + ) + + print( + f"[bold]Vyhľadávač:[/bold] " + f"{response['engine']}" + ) + + print( + f"[bold]Stratégie:[/bold] " + f"{', '.join(response['strategies']) or 'žiadna'}" + ) + + print( + f"[bold]Počet výsledkov:[/bold] " + f"{len(results)}" + ) + + print( + "\n[bold]Top výsledky:[/bold]\n" + ) + + for rank, item in enumerate( + results, + start=1, + ): + print( + f"[cyan]{rank}. " + f"Skóre: {item['score']} " + f"(BM25: {item['bm25_score']})" + f"[/cyan]" + ) + + print( + f"[bold]Názov:[/bold] " + f"{item['title']}" + ) + + print( + f"[bold]Cesta:[/bold] " + f"{item['document_path']}" + ) + + print( + f"[bold]URL:[/bold] " + f"{item['source_url']}" + ) + + print( + f"[bold]Chunk:[/bold] " + f"{item['chunk_index']}" + ) + + print( + f"[bold]Zhoda:[/bold] " + f"{item['match_strategy']}" + ) + + print( + f"[bold]Kategórie:[/bold] " + f"{item['categories']}" + ) + + print( + f"[bold]Tagy:[/bold] " + f"{item['tags']}" + ) + + print( + f"[bold]Autor:[/bold] " + f"{item['author']}" + ) + + print( + f"[bold]Ukážka:[/bold] " + f"{item['snippet']}" + ) - for rank, item in enumerate(results, start=1): - print(f"[cyan]{rank}. Skóre: {item['score']}[/cyan]") - print(f"[bold]Názov:[/bold] {item['title']}") - print(f"[bold]Cesta:[/bold] {item['document_path']}") - print(f"[bold]URL:[/bold] {item['source_url']}") - print(f"[bold]Chunk:[/bold] {item['chunk_index']}") - print(f"[bold]Kategórie:[/bold] {item['categories']}") - print(f"[bold]Tagy:[/bold] {item['tags']}") - print(f"[bold]Autor:[/bold] {item['author']}") print("[bold]Text:[/bold]") - print((item["text"] or "")[:700]) + + print( + (item["text"] or "")[:700] + ) + print("-" * 80) def main() -> None: parser = argparse.ArgumentParser( - description="Vyhľadávanie v SQLite indexe zpwiki." + description=( + "FTS5 vyhľadávanie " + "v SQLite indexe zpwiki." + ) ) parser.add_argument( @@ -55,15 +133,50 @@ def main() -> None: help="Počet výsledkov.", ) + parser.add_argument( + "--published-only", + action="store_true", + help=( + "Vyhľadáva iba v dokumentoch " + "s published: true." + ), + ) + + parser.add_argument( + "--max-per-document", + type=int, + default=3, + help=( + "Maximálny počet chunkov " + "z jedného dokumentu. " + "Hodnota 0 vypne limit." + ), + ) + args = parser.parse_args() query = " ".join(args.query) try: - mode, results = search_database(DB_FILE, query, args.limit) - except FileNotFoundError as error: - raise SystemExit(str(error)) from error + response = search_database( + DB_FILE, + query, + args.limit, + published_only=args.published_only, + max_per_document=args.max_per_document, + ) - print_results(query, mode, results) + except ( + FileNotFoundError, + RuntimeError, + ) as error: + raise SystemExit( + str(error) + ) from error + + print_results( + query, + response, + ) if __name__ == "__main__": diff --git a/scripts/search_utils.py b/scripts/search_utils.py index 6c4e600..426d272 100644 --- a/scripts/search_utils.py +++ b/scripts/search_utils.py @@ -1,239 +1,643 @@ from __future__ import annotations +import json import re import sqlite3 import unicodedata -from collections import Counter, defaultdict +from collections import defaultdict from pathlib import Path from typing import Any -TECHNICAL_TERMS = { - "rag", - "agent", - "graph", - "knowledge", - "chatbot", - "nlp", - "llm", - "lm", - "openwebui", - "docker", - "webhook", - "database", - "db", - "neo4j", - "python", - "search", - "retrieval", - "generation", - "embedding", - "vector", - "vectors", - "langchain", - "graphrag", - "qa", - "question", - "answer", - "cloud", - "api", +WORD_RE = re.compile( + r"[^\W_]+", + re.UNICODE, +) + + +# Poradie zodpovedá stĺpcom v chunks_fts: +# chunk_id, title, author, document_path, +# tags, categories, text +BM25_WEIGHTS = ( + 0.0, + 10.0, + 7.0, + 5.0, + 8.0, + 4.0, + 1.0, +) + +BM25_SQL = ", ".join( + str(value) + for value in BM25_WEIGHTS +) + + +DEFAULT_CANDIDATE_MULTIPLIER = 8 +MIN_CANDIDATES = 50 +MIN_STEM_PREFIX_LENGTH = 5 + + +STRATEGY_PRIORITY = { + "all_terms": 3, + "prefix_terms": 2, + "any_term": 1, } -def normalize_text(text: str) -> str: - text = text.lower() - text = text.replace("_", " ") - text = text.replace("/", " ") - text = text.replace("-", " ") +def normalize_for_compare( + text: str, +) -> str: + """Normalizácia pre pomocné bonusové skóre.""" + text = unicodedata.normalize( + "NFKD", + text.casefold(), + ) - text = unicodedata.normalize("NFKD", text) - text = "".join(ch for ch in text if not unicodedata.combining(ch)) + text = "".join( + character + for character in text + if not unicodedata.combining(character) + ) - return re.sub(r"[^a-z0-9]+", " ", text).strip() - - -def tokenize(text: str) -> list[str]: - return [ - word - for word in normalize_text(text).split() - if len(word) >= 2 - ] - - -def detect_search_mode(tokens: list[str]) -> str: - """Jednoduchý odhad, či ide o meno osoby alebo odbornú tému.""" - if not tokens: - return "topic" - - has_technical_term = any(token in TECHNICAL_TERMS for token in tokens) - - if len(tokens) == 2 and not has_technical_term: - return "person" - - return "topic" - - -def contains_all(query_tokens: list[str], field_tokens: list[str]) -> bool: - return all(token in field_tokens for token in query_tokens) - - -def score_tokens( - query_tokens: list[str], - field_tokens: list[str], - weight: int, -) -> int: - counts = Counter(field_tokens) - - return sum( - counts.get(token, 0) * weight - for token in query_tokens + return " ".join( + WORD_RE.findall(text) ) -def make_source_url(document_path: str) -> str: - clean_path = document_path.replace("pages/", "").replace("/README.md", "") - return f"https://zp.kemt.fei.tuke.sk/{clean_path}" +def query_tokens( + query: str, +) -> list[str]: + """Vytvorí bezpečné tokeny pre FTS5.""" + tokens: list[str] = [] + seen: set[str] = set() + + for token in WORD_RE.findall(query): + normalized = normalize_for_compare( + token + ) + + if not normalized: + continue + + if normalized in seen: + continue + + tokens.append(token) + seen.add(normalized) + + return tokens + + +def quote_fts_token( + token: str, + *, + use_prefix: bool = True, + shorten: bool = False, +) -> str: + value = token + + if ( + shorten + and len(value) > MIN_STEM_PREFIX_LENGTH + ): + value = value[:MIN_STEM_PREFIX_LENGTH] + + escaped = value.replace( + '"', + '""', + ) + + suffix = ( + "*" + if use_prefix and len(value) >= 4 + else "" + ) + + return f'"{escaped}"{suffix}' + + +def build_match_queries( + query: str, +) -> list[tuple[str, str]]: + """ + Vráti stratégie od najpresnejšej: + + all_terms -> prefix_terms -> any_term + """ + tokens = query_tokens(query) + + if not tokens: + return [] + + full_terms = [ + quote_fts_token(token) + for token in tokens + ] + + all_terms_query = " AND ".join( + full_terms + ) + + queries = [ + ( + "all_terms", + all_terms_query, + ) + ] + + shortened_terms = [ + quote_fts_token( + token, + shorten=True, + ) + for token in tokens + ] + + shortened_query = " AND ".join( + shortened_terms + ) + + if shortened_query != all_terms_query: + queries.append( + ( + "prefix_terms", + shortened_query, + ) + ) + + if len(full_terms) > 1: + queries.append( + ( + "any_term", + " OR ".join(full_terms), + ) + ) + + return queries + + +def verify_search_schema( + conn: sqlite3.Connection, +) -> None: + row = conn.execute( + """ + SELECT 1 + FROM sqlite_master + WHERE type = 'table' + AND name = 'chunks_fts' + """ + ).fetchone() + + if row is None: + raise RuntimeError( + "FTS5 index v databáze chýba. " + "Spusti python scripts/rebuild_index.py." + ) + + +def make_source_url( + document_path: str, +) -> str: + clean_path = document_path + + if clean_path.startswith("pages/"): + clean_path = clean_path[ + len("pages/"): + ] + + if clean_path.endswith("/README.md"): + clean_path = clean_path[ + :-len("/README.md") + ] + + return ( + "https://zp.kemt.fei.tuke.sk/" + f"{clean_path}" + ) def load_labels( conn: sqlite3.Connection, table: str, column: str, + chunk_ids: list[str], ) -> dict[str, list[str]]: - rows = conn.execute(f"SELECT chunk_id, {column} FROM {table}").fetchall() - labels: dict[str, list[str]] = defaultdict(list) + if not chunk_ids: + return {} - for chunk_id, value in rows: - labels[chunk_id].append(value) - - return labels - - -def person_matches(query_tokens: list[str], item: dict[str, Any]) -> bool: - fields = [ - item.get("title") or "", - item.get("document_path") or "", - item.get("author") or "", - item.get("text") or "", - ] - - return any( - contains_all(query_tokens, tokenize(field)) - for field in fields + placeholders = ",".join( + "?" + for _ in chunk_ids ) + rows = conn.execute( + f""" + SELECT chunk_id, {column} + FROM {table} + WHERE chunk_id IN ({placeholders}) + ORDER BY chunk_id, {column} + """, + chunk_ids, + ).fetchall() -def score_item( + values: dict[str, list[str]] = defaultdict( + list + ) + + for chunk_id, value in rows: + values[chunk_id].append(value) + + return dict(values) + + +def run_fts_query( + conn: sqlite3.Connection, + match_query: str, + candidate_limit: int, + published_only: bool, +) -> list[dict[str, Any]]: + rows = conn.execute( + f""" + SELECT + chunks.chunk_id, + chunks.document_path, + chunks.title, + chunks.author, + chunks.published, + chunks.chunk_index, + chunks.heading_paths_json, + chunks.text, + chunks.text_length, + chunks.token_count, + chunks.content_hash, + chunks_fts.rank AS bm25_score, + snippet( + chunks_fts, + 6, + '', + '', + ' … ', + 36 + ) AS snippet + FROM chunks_fts + JOIN chunks + ON chunks.id = chunks_fts.rowid + WHERE chunks_fts MATCH ? + AND chunks_fts.rank MATCH + 'bm25({BM25_SQL})' + AND ( + ? = 0 + OR chunks.published = 1 + ) + ORDER BY + chunks_fts.rank ASC, + chunks.id ASC + LIMIT ? + """, + ( + match_query, + 1 if published_only else 0, + candidate_limit, + ), + ).fetchall() + + return [ + dict(row) + for row in rows + ] + + +def exact_match_bonus( query: str, - query_tokens: list[str], item: dict[str, Any], - mode: str, -) -> int: - title_tokens = tokenize(item.get("title") or "") - path_tokens = tokenize(item.get("document_path") or "") - author_tokens = tokenize(item.get("author") or "") - text_tokens = tokenize(item.get("text") or "") - tag_tokens = tokenize(" ".join(item.get("tags") or [])) - category_tokens = tokenize(" ".join(item.get("categories") or [])) + tags: list[str], + categories: list[str], +) -> float: + normalized_query = normalize_for_compare( + query + ) - if mode == "person": - score = 0 - score += score_tokens(query_tokens, title_tokens, 30) - score += score_tokens(query_tokens, path_tokens, 30) - score += score_tokens(query_tokens, author_tokens, 15) - score += score_tokens(query_tokens, text_tokens, 2) + if not normalized_query: + return 0.0 - if contains_all(query_tokens, title_tokens): - score += 100 + title = normalize_for_compare( + item.get("title") or "" + ) - if contains_all(query_tokens, path_tokens): - score += 100 + author = normalize_for_compare( + item.get("author") or "" + ) - if contains_all(query_tokens, author_tokens): - score += 60 + path = normalize_for_compare( + item.get("document_path") or "" + ) - return score + text = normalize_for_compare( + item.get("text") or "" + ) - score = 0 - score += score_tokens(query_tokens, title_tokens, 12) - score += score_tokens(query_tokens, path_tokens, 12) - score += score_tokens(query_tokens, tag_tokens, 10) - score += score_tokens(query_tokens, category_tokens, 6) - score += score_tokens(query_tokens, author_tokens, 3) - score += score_tokens(query_tokens, text_tokens, 2) + normalized_tags = [ + normalize_for_compare(value) + for value in tags + ] - normalized_query = normalize_text(query) - normalized_title = normalize_text(item.get("title") or "") - normalized_path = normalize_text(item.get("document_path") or "") + normalized_categories = [ + normalize_for_compare(value) + for value in categories + ] - if normalized_query and normalized_query in normalized_title: - score += 30 + bonus = 0.0 - if normalized_query and normalized_query in normalized_path: - score += 30 + if title == normalized_query: + bonus += 6.0 - if query_tokens and contains_all(query_tokens, title_tokens): - score += 25 + elif normalized_query in title: + bonus += 3.0 - if query_tokens and contains_all(query_tokens, path_tokens): - score += 25 + if author == normalized_query: + bonus += 5.0 - return score + elif normalized_query in author: + bonus += 2.0 + + if normalized_query in path: + bonus += 2.0 + + if normalized_query in normalized_tags: + bonus += 4.0 + + if normalized_query in normalized_categories: + bonus += 3.0 + + if normalized_query in text: + bonus += 1.5 + + return bonus + + +def database_bool( + value: Any, +) -> bool | None: + """Prevedie SQLite 0/1 na API boolean.""" + if value is None: + return None + + return bool(value) + + +def add_labels_and_scores( + conn: sqlite3.Connection, + query: str, + candidates: list[dict[str, Any]], +) -> list[dict[str, Any]]: + chunk_ids = [ + item["chunk_id"] + for item in candidates + ] + + tags_by_chunk = load_labels( + conn, + "chunk_tags", + "tag", + chunk_ids, + ) + + categories_by_chunk = load_labels( + conn, + "chunk_categories", + "category", + chunk_ids, + ) + + results: list[dict[str, Any]] = [] + + for item in candidates: + chunk_id = item["chunk_id"] + + tags = tags_by_chunk.get( + chunk_id, + [], + ) + + categories = categories_by_chunk.get( + chunk_id, + [], + ) + + bm25_score = float( + item.pop("bm25_score") + ) + + strategy = item.pop("strategy") + + base_score = max( + 0.0, + -bm25_score, + ) + + score = ( + base_score + + exact_match_bonus( + query, + item, + tags, + categories, + ) + ) + + try: + heading_paths = json.loads( + item.pop( + "heading_paths_json" + ) + or "[]" + ) + + except json.JSONDecodeError: + heading_paths = [] + + item["published"] = database_bool( + item.get("published") + ) + + item["_strategy_priority"] = ( + STRATEGY_PRIORITY[strategy] + ) + + item.update( + { + "heading_paths": heading_paths, + "tags": tags, + "categories": categories, + "score": round( + score, + 6, + ), + "bm25_score": round( + bm25_score, + 6, + ), + "match_strategy": strategy, + "source_url": make_source_url( + item["document_path"] + ), + } + ) + + results.append(item) + + results.sort( + key=lambda item: ( + -item["_strategy_priority"], + -item["score"], + item["bm25_score"], + item["document_path"], + item["chunk_index"], + ) + ) + + for item in results: + item.pop( + "_strategy_priority", + None, + ) + + return results + + +def diversify_results( + results: list[dict[str, Any]], + limit: int, + max_per_document: int, +) -> list[dict[str, Any]]: + if max_per_document <= 0: + return results[:limit] + + selected: list[dict[str, Any]] = [] + + document_counts: dict[str, int] = ( + defaultdict(int) + ) + + for item in results: + document_path = item[ + "document_path" + ] + + if ( + document_counts[document_path] + >= max_per_document + ): + continue + + selected.append(item) + + document_counts[ + document_path + ] += 1 + + if len(selected) >= limit: + break + + return selected def search_database( db_file: Path, query: str, limit: int = 10, -) -> tuple[str, list[dict[str, Any]]]: + published_only: bool = False, + max_per_document: int = 3, +) -> dict[str, Any]: if not db_file.exists(): - raise FileNotFoundError(f"Databáza neexistuje: {db_file}") + raise FileNotFoundError( + f"Databáza neexistuje: {db_file}" + ) - query_tokens = tokenize(query) - mode = detect_search_mode(query_tokens) + clean_query = query.strip() - with sqlite3.connect(db_file) as conn: + if not clean_query: + return { + "engine": "sqlite_fts5", + "strategies": [], + "results": [], + } + + match_queries = build_match_queries( + clean_query + ) + + if not match_queries: + return { + "engine": "sqlite_fts5", + "strategies": [], + "results": [], + } + + candidate_limit = max( + MIN_CANDIDATES, + limit * DEFAULT_CANDIDATE_MULTIPLIER, + ) + + with sqlite3.connect( + db_file, + timeout=5.0, + ) as conn: conn.row_factory = sqlite3.Row + conn.execute( + "PRAGMA query_only = ON" + ) - tags_by_chunk = load_labels(conn, "chunk_tags", "tag") - categories_by_chunk = load_labels(conn, "chunk_categories", "category") + verify_search_schema(conn) - rows = conn.execute( - """ - SELECT - chunk_id, - document_path, - title, - author, - chunk_index, - text, - text_length - FROM chunks - """ - ).fetchall() + candidates: list[ + dict[str, Any] + ] = [] - results = [] + used_strategies: list[str] = [] - for row in rows: - item = dict(row) - chunk_id = item["chunk_id"] + # Použije sa iba prvá stratégia, + # ktorá nájde aspoň jeden výsledok: + # + # all_terms -> prefix_terms -> any_term + # + # any_term teda nedopĺňa presné + # výsledky nerelevantným obsahom. + for strategy, match_query in match_queries: + rows = run_fts_query( + conn, + match_query, + candidate_limit, + published_only, + ) - item["tags"] = tags_by_chunk.get(chunk_id, []) - item["categories"] = categories_by_chunk.get(chunk_id, []) + if not rows: + continue - if mode == "person" and not person_matches(query_tokens, item): - continue + for row in rows: + row["strategy"] = strategy - score = score_item(query, query_tokens, item, mode) + candidates = rows + used_strategies = [ + strategy + ] - if score <= 0: - continue + break - item["score"] = score - item["source_url"] = make_source_url(item["document_path"]) + results = add_labels_and_scores( + conn, + clean_query, + candidates, + ) - results.append(item) - - results.sort(key=lambda item: item["score"], reverse=True) - - return mode, results[:limit] + return { + "engine": "sqlite_fts5", + "strategies": used_strategies, + "results": diversify_results( + results, + limit, + max_per_document, + ), + } diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..564d5a7 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +@pytest.fixture(autouse=True) +def security_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Každý test dostane platnú bezpečnostnú konfiguráciu.""" + monkeypatch.setenv("WEBHOOK_SECRET", "w" * 64) + monkeypatch.setenv("SYNC_API_KEY", "s" * 64) + monkeypatch.setenv( + "EXPECTED_GITEA_REPOSITORY", + "KEMT/zpwiki", + ) + monkeypatch.setenv("WEBHOOK_PULL_GIT", "false") diff --git a/test/conftest.py:Zone.Identifier b/test/conftest.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/conftest.py:Zone.Identifier differ diff --git a/test/test_api.py b/test/test_api.py new file mode 100644 index 0000000..bad7c22 --- /dev/null +++ b/test/test_api.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import hashlib +import hmac +import json + +import pytest +from fastapi.testclient import TestClient + +import app.main as main +from scripts.rebuild_index import ReindexInProgressError + + +WEBHOOK_SECRET = "w" * 64 +SYNC_API_KEY = "s" * 64 + + +def fake_rebuild_result() -> dict: + return { + "duration_seconds": 0.01, + "counts": { + "documents": 1, + "chunks": 1, + "fts_chunks": 1, + "tags": 0, + "categories": 0, + }, + } + + +def sign(body: bytes) -> str: + return hmac.new( + WEBHOOK_SECRET.encode("utf-8"), + body, + hashlib.sha256, + ).hexdigest() + + +@pytest.fixture +def client() -> TestClient: + with TestClient(main.app) as test_client: + yield test_client + + +def test_startup_rejects_missing_secret( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("WEBHOOK_SECRET") + + with pytest.raises(RuntimeError, match="WEBHOOK_SECRET"): + with TestClient(main.app): + pass + + +def test_startup_rejects_short_secret( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SYNC_API_KEY", "short") + + with pytest.raises(RuntimeError, match="aspoň 32"): + with TestClient(main.app): + pass + + +def test_health_endpoint(client: TestClient) -> None: + response = client.get("/health") + + assert response.status_code == 200 + payload = response.json() + assert payload["status"] == "ok" + assert payload["search_engine"] == "sqlite_fts5" + assert payload["security_configured"] is True + + +def test_search_endpoint_uses_shared_search_logic( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + main, + "search_database", + lambda *args, **kwargs: { + "engine": "sqlite_fts5", + "strategies": ["all_terms"], + "results": [{"chunk_id": "test::0", "published": True}], + }, + ) + + response = client.post( + "/search", + json={"query": "jan ptak", "limit": 5}, + ) + + assert response.status_code == 200 + assert response.json()["count"] == 1 + assert response.json()["results"][0]["chunk_id"] == "test::0" + + +def test_search_rejects_empty_query(client: TestClient) -> None: + response = client.post("/search", json={"query": ""}) + assert response.status_code == 422 + + +def test_sync_rejects_missing_api_key(client: TestClient) -> None: + response = client.post("/sync", json={"pull_git": False}) + assert response.status_code == 401 + + +def test_sync_rejects_wrong_api_key(client: TestClient) -> None: + response = client.post( + "/sync", + headers={"X-API-Key": "x" * 64}, + json={"pull_git": False}, + ) + assert response.status_code == 401 + + +def test_sync_accepts_valid_api_key( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + main, + "rebuild_index", + lambda pull_git=False: fake_rebuild_result(), + ) + + response = client.post( + "/sync", + headers={"X-API-Key": SYNC_API_KEY}, + json={"pull_git": False}, + ) + + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + +def test_sync_returns_conflict_when_reindex_is_running( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def busy(*args, **kwargs): + raise ReindexInProgressError("Reindexovanie už prebieha") + + monkeypatch.setattr(main, "rebuild_index", busy) + + response = client.post( + "/sync", + headers={"X-API-Key": SYNC_API_KEY}, + json={"pull_git": False}, + ) + + assert response.status_code == 409 + + +def test_webhook_rejects_invalid_signature(client: TestClient) -> None: + body = json.dumps( + {"repository": {"full_name": "KEMT/zpwiki"}} + ).encode("utf-8") + + response = client.post( + "/webhook/gitea", + content=body, + headers={ + "Content-Type": "application/json", + "X-Gitea-Event": "push", + "X-Gitea-Signature": "0" * 64, + }, + ) + + assert response.status_code == 401 + + +def test_webhook_rejects_invalid_json_with_valid_signature( + client: TestClient, +) -> None: + body = b"not-json" + + response = client.post( + "/webhook/gitea", + content=body, + headers={ + "Content-Type": "application/json", + "X-Gitea-Event": "push", + "X-Gitea-Signature": sign(body), + }, + ) + + assert response.status_code == 400 + + +def test_webhook_requires_event_header(client: TestClient) -> None: + body = json.dumps( + {"repository": {"full_name": "KEMT/zpwiki"}} + ).encode("utf-8") + + response = client.post( + "/webhook/gitea", + content=body, + headers={ + "Content-Type": "application/json", + "X-Gitea-Signature": sign(body), + }, + ) + + assert response.status_code == 400 + + +def test_webhook_ignores_non_push_event( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = 0 + + def fake_rebuild(*args, **kwargs): + nonlocal calls + calls += 1 + return fake_rebuild_result() + + monkeypatch.setattr(main, "rebuild_index", fake_rebuild) + body = json.dumps( + {"repository": {"full_name": "KEMT/zpwiki"}} + ).encode("utf-8") + + response = client.post( + "/webhook/gitea", + content=body, + headers={ + "Content-Type": "application/json", + "X-Gitea-Event": "issues", + "X-Gitea-Signature": sign(body), + }, + ) + + assert response.status_code == 202 + assert response.json()["status"] == "ignored" + assert calls == 0 + + +def test_webhook_rejects_unexpected_repository(client: TestClient) -> None: + body = json.dumps( + {"repository": {"full_name": "OTHER/repository"}} + ).encode("utf-8") + + response = client.post( + "/webhook/gitea", + content=body, + headers={ + "Content-Type": "application/json", + "X-Gitea-Event": "push", + "X-Gitea-Signature": sign(body), + }, + ) + + assert response.status_code == 403 + + +def test_webhook_accepts_signed_push( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + main, + "rebuild_index", + lambda pull_git=False: fake_rebuild_result(), + ) + body = json.dumps( + {"repository": {"full_name": "KEMT/zpwiki"}} + ).encode("utf-8") + + response = client.post( + "/webhook/gitea", + content=body, + headers={ + "Content-Type": "application/json", + "X-Gitea-Event": "push", + "X-Gitea-Signature": sign(body), + }, + ) + + assert response.status_code == 200 + assert response.json()["verified_by"] == "hmac_sha256" + assert response.json()["repository"] == "KEMT/zpwiki" + + +def test_webhook_returns_conflict_when_reindex_is_running( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def busy(*args, **kwargs): + raise ReindexInProgressError("Reindexovanie už prebieha") + + monkeypatch.setattr(main, "rebuild_index", busy) + body = json.dumps( + {"repository": {"full_name": "KEMT/zpwiki"}} + ).encode("utf-8") + + response = client.post( + "/webhook/gitea", + content=body, + headers={ + "Content-Type": "application/json", + "X-Gitea-Event": "push", + "X-Gitea-Signature": sign(body), + }, + ) + + assert response.status_code == 409 diff --git a/test/test_api.py:Zone.Identifier b/test/test_api.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_api.py:Zone.Identifier differ diff --git a/test/test_chunking.py b/test/test_chunking.py new file mode 100644 index 0000000..ff32810 --- /dev/null +++ b/test/test_chunking.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import pytest + +import scripts.build_chunks as chunking + + +def configure_small_chunks( + monkeypatch: pytest.MonkeyPatch, + *, + maximum: int = 90, + overlap: int = 18, + minimum: int = 25, +) -> None: + monkeypatch.setattr(chunking, "MAX_TOKENS", maximum) + monkeypatch.setattr(chunking, "OVERLAP_TOKENS", overlap) + monkeypatch.setattr(chunking, "MIN_CHUNK_TOKENS", minimum) + + +def body_without_context(text: str) -> str: + parts = text.split("\n\n", maxsplit=1) + return parts[1] if len(parts) == 2 else "" + + +def test_chunks_respect_token_limit_and_keep_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + configure_small_chunks(monkeypatch) + + sentences = [ + f"Veta {index} obsahuje dostatočne dlhý skúšobný text." + for index in range(1, 30) + ] + markdown = "## Obsah\n\n" + " ".join(sentences) + + drafts = chunking.chunk_markdown( + markdown, + document_title="Testovací dokument", + ) + + assert len(drafts) > 1 + + for draft in drafts: + assert draft.text.startswith("Dokument: Testovací dokument") + assert "Sekcia: Obsah" in draft.text + assert chunking.TOKEN_COUNTER.count(draft.text) <= 90 + assert body_without_context(draft.text).startswith("Veta ") + + +def test_overlap_never_starts_in_middle_of_word_or_sentence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + configure_small_chunks(monkeypatch, maximum=75, overlap=20, minimum=20) + + markdown = "## Sekcia\n\n" + " ".join( + f"Presná veta číslo {index} sa končí bodkou." + for index in range(1, 24) + ) + + drafts = chunking.chunk_markdown(markdown, document_title="Dokument") + + assert len(drafts) > 1 + + for draft in drafts: + body = body_without_context(draft.text) + assert body.startswith("Presná veta číslo ") + assert not body.startswith(("t is done", "he databases", "te down")) + + +def test_short_sections_are_merged_or_balanced( + monkeypatch: pytest.MonkeyPatch, +) -> None: + configure_small_chunks(monkeypatch, maximum=170, overlap=20, minimum=50) + + markdown = """ +## Úlohy + +Krátke zadanie. + +## Výsledky + +Táto sekcia obsahuje dlhší text s výsledkami a ďalším vysvetlením. Druhá veta dopĺňa kontext a umožní bezpečné spojenie krátkej časti. +""" + + drafts = chunking.chunk_markdown(markdown, document_title="Test") + + assert drafts + assert any("Krátke zadanie." in draft.text for draft in drafts) + assert all(draft.units for draft in drafts) + assert all( + draft.text.strip() not in { + "Dokument: Test\nSekcia: Úlohy", + "Dokument: Test\n\nSekcia: Úlohy", + } + for draft in drafts + ) + + +def test_empty_markdown_produces_no_chunks() -> None: + assert chunking.chunk_markdown("", document_title="Prázdny") == [] + assert chunking.chunk_markdown("\n\n", document_title="Prázdny") == [] + + +def test_heading_inside_code_fence_is_not_section( + monkeypatch: pytest.MonkeyPatch, +) -> None: + configure_small_chunks(monkeypatch, maximum=220, overlap=20, minimum=20) + + markdown = """ +# Skutočná sekcia + +```markdown +## Toto nie je nadpis +print("ahoj") +``` + +Text po bloku kódu. +""" + + drafts = chunking.chunk_markdown(markdown, document_title="Kód") + + assert drafts + assert any("## Toto nie je nadpis" in draft.text for draft in drafts) + assert all( + "Toto nie je nadpis" not in path + for draft in drafts + for path in draft.heading_paths + ) + + +def test_no_chunk_exceeds_configured_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + configure_small_chunks(monkeypatch, maximum=60, overlap=10, minimum=10) + + markdown = "# Dlhý text\n\n" + " ".join( + f"slovo{index}" for index in range(1, 250) + ) + + drafts = chunking.chunk_markdown(markdown, document_title="Limit") + + assert drafts + assert max(chunking.TOKEN_COUNTER.count(draft.text) for draft in drafts) <= 60 diff --git a/test/test_chunking.py:Zone.Identifier b/test/test_chunking.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_chunking.py:Zone.Identifier differ diff --git a/test/test_common.py b/test/test_common.py new file mode 100644 index 0000000..dea04d9 --- /dev/null +++ b/test/test_common.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +import scripts.common as common + + +def write_page(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def patch_root( + monkeypatch: pytest.MonkeyPatch, + root: Path, +) -> None: + monkeypatch.setattr(common, "ZPWIKI_ROOT", root) + monkeypatch.setattr(common, "PAGES_ROOT", root / "pages") + + +def test_yaml_metadata_and_title_have_priority( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "zpwiki" + page = root / "pages" / "student" / "README.md" + + write_page( + page, + """--- +title: YAML názov +published: true +taxonomy: + category: [dp2027] + tag: [rag, nlp] + author: Daniel Hladek +--- +# Markdown názov + +Obsah. +""", + ) + + patch_root(monkeypatch, root) + document = common.load_zpwiki_page(page) + + assert document["path"] == "pages/student/README.md" + assert document["title"] == "YAML názov" + assert document["published"] is True + assert document["categories"] == ["dp2027"] + assert document["tags"] == ["rag", "nlp"] + assert document["author"] == "Daniel Hladek" + assert document["content"] == "# Markdown názov\n\nObsah." + + +def test_first_markdown_heading_is_title_fallback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "zpwiki" + page = root / "pages" / "translation" / "README.md" + + write_page( + page, + """```markdown +# Toto nie je názov dokumentu +``` + +# **Strojový** [preklad](https://example.com) + +Obsah. +""", + ) + + patch_root(monkeypatch, root) + document = common.load_zpwiki_page(page) + + assert document["title"] == "Strojový preklad" + + +def test_parent_directory_is_title_fallback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "zpwiki" + page = root / "pages" / "patrik_pavlisin" / "README.md" + + write_page(page, "Text bez YAML title a bez nadpisu.") + patch_root(monkeypatch, root) + + document = common.load_zpwiki_page(page) + + assert document["title"] == "Patrik Pavlisin" + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (True, True), + (False, False), + (1, True), + (0, False), + ("true", True), + ("ÁNO", True), + ("false", False), + ("nie", False), + (None, None), + ("neznáma hodnota", None), + ], +) +def test_optional_bool_normalization(value, expected) -> None: + assert common.normalize_optional_bool(value) is expected + + +def test_normalize_list_removes_empty_values_and_duplicates() -> None: + assert common.normalize_list("rag, nlp, rag, ") == ["rag", "nlp"] + assert common.normalize_list(["rag", "", "rag", "nlp"]) == [ + "rag", + "nlp", + ] + assert common.normalize_list(None) == [] + + +def test_json_roundtrip(tmp_path: Path) -> None: + path = tmp_path / "nested" / "data.json" + value = {"text": "Ján Pták", "published": True} + + common.write_json(path, value) + + assert common.read_json(path) == value diff --git a/test/test_common.py:Zone.Identifier b/test/test_common.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_common.py:Zone.Identifier differ diff --git a/test/test_configuration.py b/test/test_configuration.py new file mode 100644 index 0000000..1fd79fb --- /dev/null +++ b/test/test_configuration.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import re +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def read_project_file(name: str) -> str: + return (PROJECT_ROOT / name).read_text(encoding="utf-8") + + +def requirement_names(text: str) -> set[str]: + names: set[str] = set() + + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line or line.startswith(("#", "-r")): + continue + + match = re.match(r"([A-Za-z0-9_.-]+)", line) + if match: + names.add(match.group(1).casefold().replace("_", "-")) + + return names + + +def test_no_development_secret_is_committed_in_runtime_config() -> None: + files = [ + read_project_file("docker-compose.yml"), + read_project_file("app/main.py"), + ] + + assert all("dev-secret" not in text for text in files) + + +def test_compose_uses_env_file_and_keeps_chunk_configuration() -> None: + compose = read_project_file("docker-compose.yml") + + assert "env_file:" in compose + assert ".env" in compose + assert "CHUNK_MAX_TOKENS" in compose + assert "CHUNK_OVERLAP_TOKENS" in compose + assert "CHUNK_MIN_TOKENS" in compose + assert "CHUNK_TOKEN_ENCODING" in compose + assert "./data:/app/data" in compose + assert "../zpwiki:/zpwiki" in compose + + +def test_secrets_are_ignored_by_git_and_docker() -> None: + gitignore = read_project_file(".gitignore") + dockerignore = read_project_file(".dockerignore") + + assert re.search(r"(?m)^\.env$", gitignore) + assert re.search(r"(?m)^\.env$", dockerignore) + + +def test_runtime_requirements_contain_only_direct_dependencies() -> None: + names = requirement_names(read_project_file("requirements.txt")) + + required = { + "fastapi", + "pydantic", + "python-frontmatter", + "rich", + "tiktoken", + "uvicorn", + } + forbidden = { + "gitpython", + "gitdb", + "smmap", + "starlette", + "pydantic-core", + "annotated-types", + } + + assert required <= names + assert names.isdisjoint(forbidden) + + +def test_all_application_python_files_compile() -> None: + targets = [PROJECT_ROOT / "app", PROJECT_ROOT / "scripts"] + + for directory in targets: + for path in directory.rglob("*.py"): + source = path.read_text(encoding="utf-8") + compile(source, str(path), "exec") diff --git a/test/test_configuration.py:Zone.Identifier b/test/test_configuration.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_configuration.py:Zone.Identifier differ diff --git a/test/test_database.py b/test/test_database.py new file mode 100644 index 0000000..63e16a2 --- /dev/null +++ b/test/test_database.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest + +import scripts.build_sqlite_index as indexer + + +def write_json(path: Path, value) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, ensure_ascii=False), + encoding="utf-8", + ) + + +def sample_documents() -> list[dict]: + return [ + { + "path": "pages/test/README.md", + "title": "Strojový preklad", + "author": "Autor", + "published": True, + "content_length": 100, + "metadata": {"published": True}, + } + ] + + +def sample_chunks() -> list[dict]: + return [ + { + "chunk_id": "pages/test/README.md::chunk-0", + "document_path": "pages/test/README.md", + "title": "Strojový preklad", + "author": "Autor", + "published": True, + "chunk_index": 0, + "heading_paths": [["Úvod"]], + "text": "Dokument: Strojový preklad. Neurónový preklad textu.", + "text_length": 58, + "token_count": 16, + "content_hash": "abc", + "tags": ["translation", "nlp"], + "categories": ["project"], + } + ] + + +def configure_indexer( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> tuple[Path, Path, Path]: + documents_file = tmp_path / "documents.json" + chunks_file = tmp_path / "chunks.json" + db_file = tmp_path / "zp_index.sqlite" + + write_json(documents_file, sample_documents()) + write_json(chunks_file, sample_chunks()) + + monkeypatch.setattr(indexer, "DOCUMENTS_FILE", documents_file) + monkeypatch.setattr(indexer, "CHUNKS_FILE", chunks_file) + monkeypatch.setattr(indexer, "DB_FILE", db_file) + + return documents_file, chunks_file, db_file + + +def test_database_contains_documents_chunks_metadata_and_fts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, _, db_file = configure_indexer(monkeypatch, tmp_path) + + counts = indexer.build_database() + + assert counts == { + "documents": 1, + "chunks": 1, + "fts_chunks": 1, + "tags": 2, + "categories": 1, + } + + with sqlite3.connect(db_file) as conn: + assert conn.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] + assert conn.execute("SELECT published FROM chunks").fetchone()[0] == 1 + assert conn.execute("SELECT COUNT(*) FROM chunk_tags").fetchone()[0] == 2 + assert conn.execute("SELECT COUNT(*) FROM chunk_categories").fetchone()[0] == 1 + assert conn.execute( + "SELECT COUNT(*) FROM chunks_fts WHERE chunks_fts MATCH 'strojovy'" + ).fetchone()[0] == 1 + + +def test_database_rebuild_replaces_old_database_only_after_success( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, _, db_file = configure_indexer(monkeypatch, tmp_path) + + with sqlite3.connect(db_file) as conn: + conn.execute("CREATE TABLE marker(value TEXT)") + conn.execute("INSERT INTO marker VALUES ('old database')") + conn.commit() + + def fail_validation(conn: sqlite3.Connection) -> None: + raise RuntimeError("úmyselná chyba validácie") + + monkeypatch.setattr(indexer, "validate_database", fail_validation) + + with pytest.raises(RuntimeError, match="úmyselná chyba"): + indexer.build_database() + + with sqlite3.connect(db_file) as conn: + assert conn.execute("SELECT value FROM marker").fetchone()[0] == "old database" + + assert not indexer.temporary_database_path(db_file).exists() + + +def test_database_rejects_chunk_without_chunk_id( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + documents_file, chunks_file, _ = configure_indexer(monkeypatch, tmp_path) + broken = sample_chunks() + broken[0]["chunk_id"] = "" + write_json(documents_file, sample_documents()) + write_json(chunks_file, broken) + + with pytest.raises(ValueError, match="chunk_id"): + indexer.build_database() diff --git a/test/test_database.py:Zone.Identifier b/test/test_database.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_database.py:Zone.Identifier differ diff --git a/test/test_live_data.py b/test/test_live_data.py new file mode 100644 index 0000000..bfdd917 --- /dev/null +++ b/test/test_live_data.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json +import os +import sqlite3 + +import pytest + +from scripts.common import CHUNKS_FILE, DB_FILE, DOCUMENTS_FILE +from scripts.search_utils import search_database + + +pytestmark = pytest.mark.skipif( + os.getenv("RUN_LIVE_TESTS") != "1", + reason="Spusti s RUN_LIVE_TESTS=1 po vytvorení reálneho indexu.", +) + + +def test_real_generated_files_are_consistent() -> None: + assert DOCUMENTS_FILE.exists() + assert CHUNKS_FILE.exists() + assert DB_FILE.exists() + + documents = json.loads(DOCUMENTS_FILE.read_text(encoding="utf-8")) + chunks = json.loads(CHUNKS_FILE.read_text(encoding="utf-8")) + + assert documents + assert chunks + assert all(document.get("title") for document in documents) + assert all(chunk.get("chunk_id") for chunk in chunks) + assert all(chunk.get("title") for chunk in chunks) + assert all(chunk.get("text") for chunk in chunks) + assert all(chunk.get("token_count", 0) <= 450 for chunk in chunks) + + with sqlite3.connect(DB_FILE) as conn: + db_documents = conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] + db_chunks = conn.execute("SELECT COUNT(*) FROM chunks").fetchone()[0] + fts_chunks = conn.execute("SELECT COUNT(*) FROM chunks_fts").fetchone()[0] + integrity = conn.execute("PRAGMA integrity_check").fetchone()[0] + foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() + + assert db_documents == len(documents) + assert db_chunks == len(chunks) + assert fts_chunks == len(chunks) + assert integrity == "ok" + assert foreign_keys == [] + + +def test_real_search_finds_expected_baseline_results() -> None: + person = search_database(DB_FILE, "jan ptak", limit=5) + topic = search_database(DB_FILE, "strojovy preklad", limit=5) + inflection = search_database(DB_FILE, "hlboke ucenie", limit=5) + + assert person["strategies"] == ["all_terms"] + assert person["results"] + assert person["results"][0]["title"] == "Ján Pták" + + assert topic["results"] + assert topic["results"][0]["title"] == "Strojový preklad" + + assert inflection["results"] + assert inflection["strategies"] in (["all_terms"], ["prefix_terms"]) diff --git a/test/test_live_data.py:Zone.Identifier b/test/test_live_data.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_live_data.py:Zone.Identifier differ diff --git a/test/test_rebuild_index.py b/test/test_rebuild_index.py new file mode 100644 index 0000000..2911cd1 --- /dev/null +++ b/test/test_rebuild_index.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import multiprocessing +import subprocess +from pathlib import Path + +import pytest + +import scripts.rebuild_index as rebuild + + +def hold_lock(lock_path: str, ready, release) -> None: + with rebuild.acquire_reindex_lock(Path(lock_path)): + ready.set() + release.wait(timeout=10) + + +def test_only_one_process_can_reindex(tmp_path: Path) -> None: + lock_file = tmp_path / "reindex.lock" + context = multiprocessing.get_context("fork") + ready = context.Event() + release = context.Event() + process = context.Process( + target=hold_lock, + args=(str(lock_file), ready, release), + ) + process.start() + + try: + assert ready.wait(timeout=5) + + with pytest.raises( + rebuild.ReindexInProgressError, + match="už prebieha", + ): + with rebuild.acquire_reindex_lock(lock_file): + pass + finally: + release.set() + process.join(timeout=5) + + assert process.exitcode == 0 + + with rebuild.acquire_reindex_lock(lock_file): + pass + + +def test_rebuild_runs_pipeline_in_correct_order( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + + monkeypatch.setattr( + rebuild, + "git_pull", + lambda: calls.append("git_pull"), + ) + monkeypatch.setattr( + rebuild, + "scan_pages", + lambda: calls.append("scan") or [{"path": "a"}], + ) + monkeypatch.setattr( + rebuild, + "build_chunks", + lambda: calls.append("chunks") or [{"chunk_id": "a::0"}], + ) + monkeypatch.setattr( + rebuild, + "build_database", + lambda: calls.append("database") + or { + "documents": 1, + "chunks": 1, + "fts_chunks": 1, + "tags": 0, + "categories": 0, + }, + ) + + result = rebuild.rebuild_index( + pull_git=True, + lock_file=tmp_path / "lock", + ) + + assert calls == ["git_pull", "scan", "chunks", "database"] + assert result["documents_scanned"] == 1 + assert result["chunks_created"] == 1 + assert result["counts"]["fts_chunks"] == 1 + + +def test_git_pull_uses_fast_forward_only( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + (repo / ".git").mkdir(parents=True) + captured = {} + + def fake_run(args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + return subprocess.CompletedProcess(args, 0, stdout="OK", stderr="") + + monkeypatch.setattr(rebuild.subprocess, "run", fake_run) + + rebuild.git_pull(repo) + + assert captured["args"] == ["git", "pull", "--ff-only"] + assert captured["kwargs"]["cwd"] == repo + assert captured["kwargs"]["timeout"] == rebuild.GIT_PULL_TIMEOUT_SECONDS + + +def test_git_pull_rejects_non_repository(tmp_path: Path) -> None: + directory = tmp_path / "not-repo" + directory.mkdir() + + with pytest.raises(RuntimeError, match="Nie je to git repozitár"): + rebuild.git_pull(directory) + + +def test_git_pull_reports_timeout( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + (repo / ".git").mkdir(parents=True) + + def timeout(*args, **kwargs): + raise subprocess.TimeoutExpired(cmd="git pull", timeout=120) + + monkeypatch.setattr(rebuild.subprocess, "run", timeout) + + with pytest.raises(RuntimeError, match="prekročil limit"): + rebuild.git_pull(repo) + + +def test_git_pull_reports_nonzero_exit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + (repo / ".git").mkdir(parents=True) + + monkeypatch.setattr( + rebuild.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], + 1, + stdout="", + stderr="chyba", + ), + ) + + with pytest.raises(RuntimeError, match="návratovým kódom 1"): + rebuild.git_pull(repo) diff --git a/test/test_rebuild_index.py:Zone.Identifier b/test/test_rebuild_index.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_rebuild_index.py:Zone.Identifier differ diff --git a/test/test_scan_zpwiki.py b/test/test_scan_zpwiki.py new file mode 100644 index 0000000..ca3933f --- /dev/null +++ b/test/test_scan_zpwiki.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +import scripts.common as common +import scripts.scan_zpwiki as scanner + + +def write_page(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def test_scan_pages_creates_document_manifest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "zpwiki" + pages = root / "pages" + output = tmp_path / "data" / "documents.json" + + write_page( + pages / "a" / "README.md", + """--- +title: Prvý dokument +published: true +taxonomy: + category: [project] + tag: [rag] +--- +# Úvod + +Prvý obsah. +""", + ) + write_page( + pages / "b" / "README.md", + "# Druhý dokument\n\nDruhý obsah.", + ) + + monkeypatch.setattr(common, "ZPWIKI_ROOT", root) + monkeypatch.setattr(common, "PAGES_ROOT", pages) + monkeypatch.setattr(scanner, "ZPWIKI_ROOT", root) + monkeypatch.setattr(scanner, "PAGES_ROOT", pages) + monkeypatch.setattr(scanner, "DOCUMENTS_FILE", output) + + documents = scanner.scan_pages() + + assert len(documents) == 2 + assert output.exists() + + stored = json.loads(output.read_text(encoding="utf-8")) + assert stored == documents + assert all("content" not in document for document in documents) + assert all("content_preview" in document for document in documents) + assert all(document["content_length"] > 0 for document in documents) + assert documents[0]["title"] == "Prvý dokument" + assert documents[1]["title"] == "Druhý dokument" + + +def test_scan_pages_rejects_missing_pages_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + missing = tmp_path / "missing-pages" + monkeypatch.setattr(scanner, "PAGES_ROOT", missing) + + with pytest.raises(SystemExit, match="Neexistuje priečinok"): + scanner.scan_pages() diff --git a/test/test_scan_zpwiki.py:Zone.Identifier b/test/test_scan_zpwiki.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_scan_zpwiki.py:Zone.Identifier differ diff --git a/test/test_search.py b/test/test_search.py new file mode 100644 index 0000000..11cee65 --- /dev/null +++ b/test/test_search.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest + +import scripts.build_sqlite_index as indexer +from scripts.search_utils import search_database + + +def write_json(path: Path, value) -> None: + path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8") + + +def build_search_database( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Path: + documents_file = tmp_path / "documents.json" + chunks_file = tmp_path / "chunks.json" + db_file = tmp_path / "search.sqlite" + + documents = [ + { + "path": "pages/students/jan_ptak/README.md", + "title": "Ján Pták", + "author": "Daniel Hladek", + "published": True, + "content_length": 100, + "metadata": {}, + }, + { + "path": "pages/students/jan_holp/README.md", + "title": "Ján Holp", + "author": "Daniel Hladek", + "published": True, + "content_length": 100, + "metadata": {}, + }, + { + "path": "pages/topics/translation/README.md", + "title": "Strojový preklad", + "author": "Daniel Hladek", + "published": True, + "content_length": 200, + "metadata": {}, + }, + { + "path": "pages/topics/open/README.md", + "title": "Otvorené projekty", + "author": "Daniel Hladek", + "published": True, + "content_length": 100, + "metadata": {}, + }, + { + "path": "pages/topics/deep/README.md", + "title": "Neurónové siete", + "author": "Daniel Hladek", + "published": False, + "content_length": 100, + "metadata": {}, + }, + ] + + def chunk( + chunk_id: str, + path: str, + title: str, + text: str, + published: bool, + index: int = 0, + tags: list[str] | None = None, + categories: list[str] | None = None, + ) -> dict: + return { + "chunk_id": chunk_id, + "document_path": path, + "title": title, + "author": "Daniel Hladek", + "published": published, + "chunk_index": index, + "heading_paths": [], + "text": text, + "text_length": len(text), + "token_count": 30, + "content_hash": chunk_id, + "tags": tags or [], + "categories": categories or [], + } + + chunks = [ + chunk( + "jan-ptak::0", + "pages/students/jan_ptak/README.md", + "Ján Pták", + "Dokument: Ján Pták. Agent pre manažment záverečných prác.", + True, + tags=["rag", "nlp"], + categories=["dp2027"], + ), + chunk( + "jan-holp::0", + "pages/students/jan_holp/README.md", + "Ján Holp", + "Dokument: Ján Holp. Získavanie informácií a PageRank.", + True, + tags=["ir"], + ), + chunk( + "translation::0", + "pages/topics/translation/README.md", + "Strojový preklad", + "Dokument: Strojový preklad. Štatistický strojový preklad.", + True, + index=0, + tags=["translation"], + categories=["project"], + ), + chunk( + "translation::1", + "pages/topics/translation/README.md", + "Strojový preklad", + "Dokument: Strojový preklad. Neurónový preklad viet.", + True, + index=1, + tags=["translation"], + categories=["project"], + ), + chunk( + "open::0", + "pages/topics/open/README.md", + "Otvorené projekty", + "Téma pre strojový preklad slovenského jazyka.", + True, + categories=["info"], + ), + chunk( + "deep::0", + "pages/topics/deep/README.md", + "Neurónové siete", + "Modely hlbokého učenia a trénovanie hlbokých neurónových sietí.", + False, + tags=["nn"], + ), + ] + + write_json(documents_file, documents) + write_json(chunks_file, chunks) + + monkeypatch.setattr(indexer, "DOCUMENTS_FILE", documents_file) + monkeypatch.setattr(indexer, "CHUNKS_FILE", chunks_file) + monkeypatch.setattr(indexer, "DB_FILE", db_file) + indexer.build_database() + + return db_file + + +def test_person_query_does_not_add_any_term_noise( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_file = build_search_database(tmp_path, monkeypatch) + + response = search_database(db_file, "jan ptak", limit=5) + + assert response["strategies"] == ["all_terms"] + assert [item["title"] for item in response["results"]] == ["Ján Pták"] + + +def test_topic_query_returns_best_topic_first( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_file = build_search_database(tmp_path, monkeypatch) + + response = search_database(db_file, "strojovy preklad", limit=5) + + assert response["strategies"] == ["all_terms"] + assert response["results"][0]["title"] == "Strojový preklad" + + +def test_prefix_fallback_handles_slovak_inflection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_file = build_search_database(tmp_path, monkeypatch) + + response = search_database(db_file, "hlboke ucenie", limit=5) + + assert response["strategies"] == ["prefix_terms"] + assert response["results"][0]["title"] == "Neurónové siete" + + +def test_any_term_is_used_only_when_stricter_queries_find_nothing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_file = build_search_database(tmp_path, monkeypatch) + + response = search_database(db_file, "nezmysel preklad", limit=5) + + assert response["strategies"] == ["any_term"] + assert response["results"] + assert all("preklad" in item["text"].casefold() for item in response["results"]) + + +def test_published_only_excludes_unpublished_chunks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_file = build_search_database(tmp_path, monkeypatch) + + all_results = search_database(db_file, "hlboke ucenie", limit=5) + public_results = search_database( + db_file, + "hlboke ucenie", + limit=5, + published_only=True, + ) + + assert all_results["results"] + assert public_results["results"] == [] + + +def test_results_are_diversified_by_document( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_file = build_search_database(tmp_path, monkeypatch) + + response = search_database( + db_file, + "preklad", + limit=5, + max_per_document=1, + ) + + paths = [item["document_path"] for item in response["results"]] + assert len(paths) == len(set(paths)) + + +def test_public_result_format_has_no_internal_id_and_uses_boolean( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_file = build_search_database(tmp_path, monkeypatch) + + result = search_database(db_file, "jan ptak", limit=1)["results"][0] + + assert "id" not in result + assert result["published"] is True + assert result["chunk_id"] == "jan-ptak::0" + assert result["source_url"].endswith("students/jan_ptak") + + +def test_empty_query_returns_empty_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_file = build_search_database(tmp_path, monkeypatch) + + response = search_database(db_file, " ", limit=5) + + assert response == { + "engine": "sqlite_fts5", + "strategies": [], + "results": [], + } + + +def test_missing_database_is_reported(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + search_database(tmp_path / "missing.sqlite", "rag") + + +def test_missing_fts_schema_is_reported(tmp_path: Path) -> None: + db_file = tmp_path / "broken.sqlite" + with sqlite3.connect(db_file) as conn: + conn.execute("CREATE TABLE chunks(id INTEGER PRIMARY KEY)") + + with pytest.raises(RuntimeError, match="FTS5 index"): + search_database(db_file, "rag") diff --git a/test/test_search.py:Zone.Identifier b/test/test_search.py:Zone.Identifier new file mode 100644 index 0000000..b5d9278 Binary files /dev/null and b/test/test_search.py:Zone.Identifier differ