diff --git a/.gitignore b/.gitignore index 2d63932..df2cb7e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ +.env +data/ .venv/ __pycache__/ *.py[cod] -.env *.log .pytest_cache/ .coverage htmlcov/ -data/ diff --git a/README.md b/README.md index 22828aa..bae2098 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Backend pre indexovanie a vyhľadávanie v repozitári záverečných prác `zpwiki`. -Projekt načítava Markdown dokumenty, spracuje YAML metadata, rozdelí obsah na tokenové chunky a vytvorí SQLite FTS5 index. Vyhľadávanie je dostupné cez FastAPI a systém podporuje manuálnu aj webhookovú synchronizáciu. +Projekt načítava Markdown dokumenty, spracuje YAML metadata, rozdelí obsah na tokenové chunky a vytvorí SQLite index kombinujúci FTS5 fulltextové vyhľadávanie a embeddingy. Vyhľadávanie je dostupné cez FastAPI a systém podporuje manuálnu aj webhookovú synchronizáciu. ## Implementované @@ -12,13 +12,19 @@ Projekt načítava Markdown dokumenty, spracuje YAML metadata, rozdelí obsah na - zachovanie názvu dokumentu a hierarchie nadpisov v chunku, - SQLite databáza a FTS5 fulltextový index, - BM25 vyhľadávanie s podporou diakritiky a prefixových výrazov, +- embeddingy pre každý chunk pomocou modelu `intfloat/multilingual-e5-small`, +- uloženie embeddingov priamo v SQLite, +- vektorové vyhľadávanie pomocou cosine similarity, +- hybridné vyhľadávanie FTS5 + embeddings pomocou RRF, +- nižšia váha pre slabú `any_term` FTS stratégiu, +- zachovanie presných `all_terms` a `prefix_terms` výsledkov bez vektorového šumu, - filtrovanie publikovaných dokumentov, - FastAPI endpointy `/health`, `/search`, `/sync` a `/webhook/gitea`, -- autorizácia `/sync` pomocou API kľúča, +- autorizácia `/search` a `/sync` pomocou API kľúča, - Gitea webhook s HMAC-SHA256 podpisom a kontrolou udalosti a repozitára, - zámok proti súbežnému reindexovaniu, - atomická výmena databázy po úspešnom reindexovaní, -- automatizované a integračné testy nad reálnymi dátami. +- automatizované a integračné testy. ## Štruktúra @@ -31,6 +37,7 @@ zp-agent/ │ ├── scan_zpwiki.py │ ├── build_chunks.py │ ├── build_sqlite_index.py +│ ├── embedding_utils.py │ ├── rebuild_index.py │ ├── search_db.py │ └── search_utils.py @@ -58,8 +65,13 @@ V koreňovom priečinku vytvor `.env`: ```dotenv WEBHOOK_SECRET= SYNC_API_KEY= +SEARCH_API_KEY=<ďalšia náhodná hodnota s minimálne 32 znakmi> EXPECTED_GITEA_REPOSITORY=KEMT/zpwiki WEBHOOK_PULL_GIT=false + +# Voliteľné +EMBEDDING_MODEL=intfloat/multilingual-e5-small +EMBEDDING_BATCH_SIZE=32 ``` Tajomstvá je možné vygenerovať príkazom: @@ -73,7 +85,7 @@ Súbor `.env` sa nesmie commitovať. ## Spustenie cez Docker ```bash -docker compose build --no-cache +docker compose build docker compose up -d ``` @@ -97,7 +109,7 @@ docker compose down ## Reindexovanie -Celý proces načíta dokumenty, vytvorí chunky a obnoví SQLite FTS5 index: +Celý proces načíta dokumenty, vytvorí chunky, obnoví FTS5 index a vytvorí embedding pre každý chunk: ```bash docker compose run --rm zp-agent-api python scripts/rebuild_index.py @@ -111,18 +123,44 @@ data/chunks.json data/zp_index.sqlite ``` +Databáza obsahuje dokumenty, chunky, FTS5 index, metadata a embeddingy. + ## Vyhľadávanie +Vyhľadávanie kombinuje: + +```text +dotaz +├── FTS5 / BM25 +└── embeddingové vyhľadávanie + ↓ + RRF fusion + ↓ + výsledky +``` + Test z terminálu: ```bash -docker compose run --rm zp-agent-api python scripts/search_db.py "rag agent" --limit 5 +docker compose run --rm zp-agent-api \ + python scripts/search_db.py "rag agent" --limit 5 ``` -Vyhľadávanie cez API: +Pred volaním API načítaj premenné z `.env`: ```bash -curl -X POST http://127.0.0.1:8000/search -H "Content-Type: application/json" -d '{ +set -a +source .env +set +a +``` + +Vyhľadávanie cez zabezpečené API: + +```bash +curl -X POST http://127.0.0.1:8000/search \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $SEARCH_API_KEY" \ + -d '{ "query": "rag agent", "limit": 5, "published_only": false, @@ -130,10 +168,19 @@ curl -X POST http://127.0.0.1:8000/search -H "Content-Type: application/json" }' ``` +API vracia hybridný engine: + +```text +hybrid_fts5_embeddings +``` + Manuálne reindexovanie cez zabezpečený endpoint: ```bash -curl -X POST http://127.0.0.1:8000/sync -H "Content-Type: application/json" -H "X-API-Key: $SYNC_API_KEY" -d '{"pull_git": false}' +curl -X POST http://127.0.0.1:8000/sync \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $SYNC_API_KEY" \ + -d '{"pull_git": false}' ``` ## Testy @@ -150,14 +197,18 @@ Bežné automatizované testy: pytest -q test ``` +Aktuálna testovacia sada: + +```text +65 passed, 2 skipped +``` + Testy vrátane kontroly reálne vygenerovaných dát a databázy: ```bash RUN_LIVE_TESTS=1 pytest -q test ``` -Aktuálna implementácia prešla všetkými 65 testami vrátane live testov. - ## Ďalší krok -Najbližšia etapa je integrácia s OpenWebUI a vytvorenie agentového rozhrania. Následne sa doplnia embeddingy, hybridné vyhľadávanie a RAG odpovede s citáciami. +Najbližšia etapa je integrácia s OpenWebUI a vytvorenie agentového rozhrania. Následne sa doplnia RAG odpovede so zdrojmi a citáciami. diff --git "a/a nachadza v staged obsahu\"" "b/a nachadza v staged obsahu\"" new file mode 100644 index 0000000..333a0b5 --- /dev/null +++ "b/a nachadza v staged obsahu\"" @@ -0,0 +1,258 @@ + + SSUUMMMMAARRYY OOFF LLEESSSS CCOOMMMMAANNDDSS + + Commands marked with * may be preceded by a number, _N. + Notes in parentheses indicate the behavior if _N is given. + A key preceded by a caret indicates the Ctrl key; thus ^K is ctrl-K. + + h H Display this help. + q :q Q :Q ZZ Exit. + --------------------------------------------------------------------------- + + MMOOVVIINNGG + + e ^E j ^N CR * Forward one line (or _N lines). + y ^Y k ^K ^P * Backward one line (or _N lines). + f ^F ^V SPACE * Forward one window (or _N lines). + b ^B ESC-v * Backward one window (or _N lines). + z * Forward one window (and set window to _N). + w * Backward one window (and set window to _N). + ESC-SPACE * Forward one window, but don't stop at end-of-file. + d ^D * Forward one half-window (and set half-window to _N). + u ^U * Backward one half-window (and set half-window to _N). + ESC-) RightArrow * Right one half screen width (or _N positions). + ESC-( LeftArrow * Left one half screen width (or _N positions). + ESC-} ^RightArrow Right to last column displayed. + ESC-{ ^LeftArrow Left to first column. + F Forward forever; like "tail -f". + ESC-F Like F but stop when search pattern is found. + r ^R ^L Repaint screen. + R Repaint screen, discarding buffered input. + --------------------------------------------------- + Default "window" is the screen height. + Default "half-window" is half of the screen height. + --------------------------------------------------------------------------- + + SSEEAARRCCHHIINNGG + + /_p_a_t_t_e_r_n * Search forward for (_N-th) matching line. + ?_p_a_t_t_e_r_n * Search backward for (_N-th) matching line. + n * Repeat previous search (for _N-th occurrence). + N * Repeat previous search in reverse direction. + ESC-n * Repeat previous search, spanning files. + ESC-N * Repeat previous search, reverse dir. & spanning files. + ESC-u Undo (toggle) search highlighting. + ESC-U Clear search highlighting. + &_p_a_t_t_e_r_n * Display only matching lines. + --------------------------------------------------- + A search pattern may begin with one or more of: + ^N or ! Search for NON-matching lines. + ^E or * Search multiple files (pass thru END OF FILE). + ^F or @ Start search at FIRST file (for /) or last file (for ?). + ^K Highlight matches, but don't move (KEEP position). + ^R Don't use REGULAR EXPRESSIONS. + ^W WRAP search if no match found. + --------------------------------------------------------------------------- + + JJUUMMPPIINNGG + + g < ESC-< * Go to first line in file (or line _N). + G > ESC-> * Go to last line in file (or line _N). + p % * Go to beginning of file (or _N percent into file). + t * Go to the (_N-th) next tag. + T * Go to the (_N-th) previous tag. + { ( [ * Find close bracket } ) ]. + } ) ] * Find open bracket { ( [. + ESC-^F _<_c_1_> _<_c_2_> * Find close bracket _<_c_2_>. + ESC-^B _<_c_1_> _<_c_2_> * Find open bracket _<_c_1_>. + --------------------------------------------------- + Each "find close bracket" command goes forward to the close bracket + matching the (_N-th) open bracket in the top line. + Each "find open bracket" command goes backward to the open bracket + matching the (_N-th) close bracket in the bottom line. + + m_<_l_e_t_t_e_r_> Mark the current top line with . + M_<_l_e_t_t_e_r_> Mark the current bottom line with . + '_<_l_e_t_t_e_r_> Go to a previously marked position. + '' Go to the previous position. + ^X^X Same as '. + ESC-M_<_l_e_t_t_e_r_> Clear a mark. + --------------------------------------------------- + A mark is any upper-case or lower-case letter. + Certain marks are predefined: + ^ means beginning of the file + $ means end of the file + --------------------------------------------------------------------------- + + CCHHAANNGGIINNGG FFIILLEESS + + :e [_f_i_l_e] Examine a new file. + ^X^V Same as :e. + :n * Examine the (_N-th) next file from the command line. + :p * Examine the (_N-th) previous file from the command line. + :x * Examine the first (or _N-th) file from the command line. + :d Delete the current file from the command line list. + = ^G :f Print current file name. + --------------------------------------------------------------------------- + + MMIISSCCEELLLLAANNEEOOUUSS CCOOMMMMAANNDDSS + + -_<_f_l_a_g_> Toggle a command line option [see OPTIONS below]. + --_<_n_a_m_e_> Toggle a command line option, by name. + __<_f_l_a_g_> Display the setting of a command line option. + ___<_n_a_m_e_> Display the setting of an option, by name. + +_c_m_d Execute the less cmd each time a new file is examined. + + !_c_o_m_m_a_n_d Execute the shell command with $SHELL. + |XX_c_o_m_m_a_n_d Pipe file between current pos & mark XX to shell command. + s _f_i_l_e Save input to a file. + v Edit the current file with $VISUAL or $EDITOR. + V Print version number of "less". + --------------------------------------------------------------------------- + + OOPPTTIIOONNSS + + Most options may be changed either on the command line, + or from within less by using the - or -- command. + Options may be given in one of two forms: either a single + character preceded by a -, or a name preceded by --. + + -? ........ --help + Display help (from command line). + -a ........ --search-skip-screen + Search skips current screen. + -A ........ --SEARCH-SKIP-SCREEN + Search starts just after target line. + -b [_N] .... --buffers=[_N] + Number of buffers. + -B ........ --auto-buffers + Don't automatically allocate buffers for pipes. + -c ........ --clear-screen + Repaint by clearing rather than scrolling. + -d ........ --dumb + Dumb terminal. + -D xx_c_o_l_o_r . --color=xx_c_o_l_o_r + Set screen colors. + -e -E .... --quit-at-eof --QUIT-AT-EOF + Quit at end of file. + -f ........ --force + Force open non-regular files. + -F ........ --quit-if-one-screen + Quit if entire file fits on first screen. + -g ........ --hilite-search + Highlight only last match for searches. + -G ........ --HILITE-SEARCH + Don't highlight any matches for searches. + -h [_N] .... --max-back-scroll=[_N] + Backward scroll limit. + -i ........ --ignore-case + Ignore case in searches that do not contain uppercase. + -I ........ --IGNORE-CASE + Ignore case in all searches. + -j [_N] .... --jump-target=[_N] + Screen position of target lines. + -J ........ --status-column + Display a status column at left edge of screen. + -k [_f_i_l_e] . --lesskey-file=[_f_i_l_e] + Use a lesskey file. + -K ........ --quit-on-intr + Exit less in response to ctrl-C. + -L ........ --no-lessopen + Ignore the LESSOPEN environment variable. + -m -M .... --long-prompt --LONG-PROMPT + Set prompt style. + -n -N .... --line-numbers --LINE-NUMBERS + Don't use line numbers. + -o [_f_i_l_e] . --log-file=[_f_i_l_e] + Copy to log file (standard input only). + -O [_f_i_l_e] . --LOG-FILE=[_f_i_l_e] + Copy to log file (unconditionally overwrite). + -p [_p_a_t_t_e_r_n] --pattern=[_p_a_t_t_e_r_n] + Start at pattern (from command line). + -P [_p_r_o_m_p_t] --prompt=[_p_r_o_m_p_t] + Define new prompt. + -q -Q .... --quiet --QUIET --silent --SILENT + Quiet the terminal bell. + -r -R .... --raw-control-chars --RAW-CONTROL-CHARS + Output "raw" control characters. + -s ........ --squeeze-blank-lines + Squeeze multiple blank lines. + -S ........ --chop-long-lines + Chop (truncate) long lines rather than wrapping. + -t [_t_a_g] .. --tag=[_t_a_g] + Find a tag. + -T [_t_a_g_s_f_i_l_e] --tag-file=[_t_a_g_s_f_i_l_e] + Use an alternate tags file. + -u -U .... --underline-special --UNDERLINE-SPECIAL + Change handling of backspaces. + -V ........ --version + Display the version number of "less". + -w ........ --hilite-unread + Highlight first new line after forward-screen. + -W ........ --HILITE-UNREAD + Highlight first new line after any forward movement. + -x [_N[,...]] --tabs=[_N[,...]] + Set tab stops. + -X ........ --no-init + Don't use termcap init/deinit strings. + -y [_N] .... --max-forw-scroll=[_N] + Forward scroll limit. + -z [_N] .... --window=[_N] + Set size of window. + -" [_c[_c]] . --quotes=[_c[_c]] + Set shell quote characters. + -~ ........ --tilde + Don't display tildes after end of file. + -# [_N] .... --shift=[_N] + Set horizontal scroll amount (0 = one half screen width). + --file-size + Automatically determine the size of the input file. + --follow-name + The F command changes files if the input file is renamed. + --incsearch + Search file as each pattern character is typed in. + --line-num-width=N + Set the width of the -N line number field to N characters. + --mouse + Enable mouse input. + --no-keypad + Don't send termcap keypad init/deinit strings. + --no-histdups + Remove duplicates from command history. + --rscroll=C + Set the character used to mark truncated lines. + --save-marks + Retain marks across invocations of less. + --status-col-width=N + Set the width of the -J status column to N characters. + --use-backslash + Subsequent options use backslash as escape char. + --use-color + Enables colored text. + --wheel-lines=N + Each click of the mouse wheel moves N lines. + + + --------------------------------------------------------------------------- + + LLIINNEE EEDDIITTIINNGG + + These keys can be used to edit text being entered + on the "command line" at the bottom of the screen. + + RightArrow ..................... ESC-l ... Move cursor right one character. + LeftArrow ...................... ESC-h ... Move cursor left one character. + ctrl-RightArrow ESC-RightArrow ESC-w ... Move cursor right one word. + ctrl-LeftArrow ESC-LeftArrow ESC-b ... Move cursor left one word. + HOME ........................... ESC-0 ... Move cursor to start of line. + END ............................ ESC-$ ... Move cursor to end of line. + BACKSPACE ................................ Delete char to left of cursor. + DELETE ......................... ESC-x ... Delete char under cursor. + ctrl-BACKSPACE ESC-BACKSPACE ........... Delete word to left of cursor. + ctrl-DELETE .... ESC-DELETE .... ESC-X ... Delete word under cursor. + ctrl-U ......... ESC (MS-DOS only) ....... Delete entire line. + UpArrow ........................ ESC-k ... Retrieve previous command line. + DownArrow ...................... ESC-j ... Retrieve next command line. + TAB ...................................... Complete filename & cycle. + SHIFT-TAB ...................... ESC-TAB Complete filename & reverse cycle. + ctrl-L ................................... Complete filename, list all. diff --git a/app/main.py b/app/main.py index 21e1f01..e31f1bb 100644 --- a/app/main.py +++ b/app/main.py @@ -10,26 +10,44 @@ from contextlib import asynccontextmanager from pathlib import Path from typing import Any -from fastapi import Depends, FastAPI, Header, HTTPException, Request, Security, status +from fastapi import ( + Depends, + FastAPI, + Header, + HTTPException, + Request, + Security, + status, +) from fastapi.responses import JSONResponse from fastapi.security import APIKeyHeader from pydantic import BaseModel, Field - PROJECT_ROOT = Path(__file__).resolve().parents[1] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) - from scripts.common import DB_FILE, ZPWIKI_ROOT -from scripts.rebuild_index import ReindexInProgressError, rebuild_index +from scripts.rebuild_index import ( + ReindexInProgressError, + rebuild_index, +) from scripts.search_utils import search_database MIN_SECRET_LENGTH = 32 + +SEARCH_API_KEY_HEADER = "X-API-Key" SYNC_API_KEY_HEADER = "X-API-Key" + +search_api_key_scheme = APIKeyHeader( + name=SEARCH_API_KEY_HEADER, + auto_error=False, + description="API kľúč pre vyhľadávanie v zpwiki.", +) + sync_api_key_scheme = APIKeyHeader( name=SYNC_API_KEY_HEADER, auto_error=False, @@ -38,10 +56,22 @@ sync_api_key_scheme = APIKeyHeader( class SearchRequest(BaseModel): - query: str = Field(..., min_length=1, max_length=500) - limit: int = Field(default=10, ge=1, le=50) + query: str = Field( + ..., + min_length=1, + max_length=500, + ) + limit: int = Field( + default=10, + ge=1, + le=50, + ) published_only: bool = False - max_per_document: int = Field(default=3, ge=0, le=10) + max_per_document: int = Field( + default=3, + ge=0, + le=10, + ) class SyncRequest(BaseModel): @@ -55,7 +85,9 @@ def required_environment_value(name: str) -> str: value = os.getenv(name, "").strip() if not value: - raise RuntimeError(f"Chýba povinná environment premenná {name}") + raise RuntimeError( + f"Chýba povinná environment premenná {name}" + ) return value @@ -72,53 +104,104 @@ def validate_secret(name: str) -> str: def expected_gitea_repository() -> str: - value = required_environment_value("EXPECTED_GITEA_REPOSITORY") + value = required_environment_value( + "EXPECTED_GITEA_REPOSITORY" + ) if "/" not in value: raise RuntimeError( - "EXPECTED_GITEA_REPOSITORY musí mať tvar vlastník/repozitár" + "EXPECTED_GITEA_REPOSITORY musí mať tvar " + "vlastník/repozitár" ) return value def webhook_should_pull_git() -> bool: - value = os.getenv("WEBHOOK_PULL_GIT", "false").strip().casefold() + value = os.getenv( + "WEBHOOK_PULL_GIT", + "false", + ).strip().casefold() - return value in {"1", "true", "yes", "on"} + return value in { + "1", + "true", + "yes", + "on", + } def validate_security_configuration() -> None: validate_secret("WEBHOOK_SECRET") validate_secret("SYNC_API_KEY") + validate_secret("SEARCH_API_KEY") expected_gitea_repository() @asynccontextmanager async def lifespan(_: FastAPI): - # Aplikácia sa nespustí s chýbajúcim alebo slabým tajomstvom. validate_security_configuration() yield app = FastAPI( title="ZP Agent API", - description="API pre vyhľadávanie v repozitári záverečných prác zpwiki.", + description=( + "API pre vyhľadávanie v repozitári " + "záverečných prác zpwiki." + ), version="0.6.0", lifespan=lifespan, ) -def require_sync_api_key( - api_key: str | None = Security(sync_api_key_scheme), +def require_search_api_key( + api_key: str | None = Security( + search_api_key_scheme + ), ) -> None: - expected = validate_secret("SYNC_API_KEY") + expected = validate_secret( + "SEARCH_API_KEY" + ) - if not api_key or not hmac.compare_digest(api_key, expected): + if ( + not api_key + or not hmac.compare_digest( + api_key, + expected, + ) + ): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Neplatný alebo chýbajúci API kľúč", - headers={"WWW-Authenticate": "ApiKey"}, + headers={ + "WWW-Authenticate": "ApiKey", + }, + ) + + +def require_sync_api_key( + api_key: str | None = Security( + sync_api_key_scheme + ), +) -> None: + expected = validate_secret( + "SYNC_API_KEY" + ) + + if ( + not api_key + or not hmac.compare_digest( + api_key, + expected, + ) + ): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Neplatný alebo chýbajúci API kľúč", + headers={ + "WWW-Authenticate": "ApiKey", + }, ) @@ -132,16 +215,18 @@ def verify_gitea_signature( supplied = signature.strip().casefold() - # X-Gitea-Signature je čistý hex digest. Prefix prijímame iba - # kvôli kompatibilite s X-Hub-Signature-256. + # Kompatibilita podpisu. if supplied.startswith("sha256="): - supplied = supplied.removeprefix("sha256=") + supplied = supplied.removeprefix( + "sha256=" + ) if len(supplied) != 64: return False try: int(supplied, 16) + except ValueError: return False @@ -151,7 +236,10 @@ def verify_gitea_signature( hashlib.sha256, ).hexdigest() - return hmac.compare_digest(expected, supplied) + return hmac.compare_digest( + expected, + supplied, + ) def repository_name_from_payload( @@ -195,17 +283,28 @@ def health() -> dict[str, Any]: "zpwiki_root": str(ZPWIKI_ROOT), "zpwiki_exists": ZPWIKI_ROOT.exists(), "security_configured": all( - bool(os.getenv(name, "").strip()) + bool( + os.getenv( + name, + "", + ).strip() + ) for name in ( "WEBHOOK_SECRET", "SYNC_API_KEY", + "SEARCH_API_KEY", "EXPECTED_GITEA_REPOSITORY", ) ), } -@app.post("/search") +@app.post( + "/search", + dependencies=[ + Depends(require_search_api_key) + ], +) def search( request: SearchRequest, ) -> dict[str, Any]: @@ -214,8 +313,12 @@ def search( DB_FILE, request.query, request.limit, - published_only=request.published_only, - max_per_document=request.max_per_document, + published_only=( + request.published_only + ), + max_per_document=( + request.max_per_document + ), ) except FileNotFoundError as error: @@ -249,7 +352,9 @@ def search( @app.post( "/sync", - dependencies=[Depends(require_sync_api_key)], + dependencies=[ + Depends(require_sync_api_key) + ], ) def sync( request: SyncRequest, @@ -274,14 +379,16 @@ def sync( return { "status": "ok", "pull_git": request.pull_git, - "duration_seconds": result["duration_seconds"], + "duration_seconds": ( + result["duration_seconds"] + ), "counts": result["counts"], } @app.post( - "/webhook/gitea", - response_model = None, + "/webhook/gitea", + response_model=None, ) async def gitea_webhook( request: Request, @@ -295,7 +402,10 @@ async def gitea_webhook( ), ) -> dict[str, Any] | JSONResponse: raw_body = await request.body() - secret = validate_secret("WEBHOOK_SECRET") + + secret = validate_secret( + "WEBHOOK_SECRET" + ) if not verify_gitea_signature( raw_body, @@ -303,7 +413,9 @@ async def gitea_webhook( secret, ): raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, + status_code=( + status.HTTP_401_UNAUTHORIZED + ), detail="Neplatný webhook podpis", ) @@ -318,24 +430,35 @@ async def gitea_webhook( ) as error: raise HTTPException( status_code=400, - detail="Webhook payload nie je platný JSON", + detail=( + "Webhook payload nie je " + "platný JSON" + ), ) from error if not isinstance(payload, dict): raise HTTPException( status_code=400, - detail="Webhook payload musí byť JSON objekt", + detail=( + "Webhook payload musí byť " + "JSON objekt" + ), ) if not x_gitea_event: raise HTTPException( status_code=400, - detail="Chýba hlavička X-Gitea-Event", + detail=( + "Chýba hlavička " + "X-Gitea-Event" + ), ) if x_gitea_event.casefold() != "push": return JSONResponse( - status_code=status.HTTP_202_ACCEPTED, + status_code=( + status.HTTP_202_ACCEPTED + ), content={ "status": "ignored", "reason": "unsupported_event", @@ -343,8 +466,10 @@ async def gitea_webhook( }, ) - repository_name = repository_name_from_payload( - payload + repository_name = ( + repository_name_from_payload( + payload + ) ) if repository_name is None: @@ -356,7 +481,9 @@ async def gitea_webhook( ), ) - expected_repository = expected_gitea_repository() + expected_repository = ( + expected_gitea_repository() + ) if not same_repository( repository_name, @@ -373,7 +500,9 @@ async def gitea_webhook( try: result = await asyncio.to_thread( rebuild_index, - pull_git=webhook_should_pull_git(), + pull_git=( + webhook_should_pull_git() + ), ) except ReindexInProgressError as error: @@ -393,6 +522,8 @@ async def gitea_webhook( "event": x_gitea_event, "repository": repository_name, "verified_by": "hmac_sha256", - "duration_seconds": result["duration_seconds"], + "duration_seconds": ( + result["duration_seconds"] + ), "counts": result["counts"], } diff --git a/et -a b/et -a new file mode 100644 index 0000000..67f74af --- /dev/null +++ b/et -a @@ -0,0 +1,40 @@ +README.md:66:WEBHOOK_SECRET= +README.md:67:SYNC_API_KEY= +README.md:68:SEARCH_API_KEY=<ďalšia náhodná hodnota s minimálne 32 znakmi> +README.md:162: -H "X-API-Key: $SEARCH_API_KEY" \ +README.md:182: -H "X-API-Key: $SYNC_API_KEY" \ +app/main.py:41:SEARCH_API_KEY_HEADER = "X-API-Key" +app/main.py:42:SYNC_API_KEY_HEADER = "X-API-Key" +app/main.py:45:search_api_key_scheme = APIKeyHeader( +app/main.py:46: name=SEARCH_API_KEY_HEADER, +app/main.py:51:sync_api_key_scheme = APIKeyHeader( +app/main.py:52: name=SYNC_API_KEY_HEADER, +app/main.py:135: validate_secret("WEBHOOK_SECRET") +app/main.py:136: validate_secret("SYNC_API_KEY") +app/main.py:137: validate_secret("SEARCH_API_KEY") +app/main.py:158:def require_search_api_key( +app/main.py:160: search_api_key_scheme +app/main.py:164: "SEARCH_API_KEY" +app/main.py:183:def require_sync_api_key( +app/main.py:185: sync_api_key_scheme +app/main.py:189: "SYNC_API_KEY" +app/main.py:293: "WEBHOOK_SECRET", +app/main.py:294: "SYNC_API_KEY", +app/main.py:295: "SEARCH_API_KEY", +app/main.py:305: Depends(require_search_api_key) +app/main.py:356: Depends(require_sync_api_key) +app/main.py:407: "WEBHOOK_SECRET" +test/conftest.py:21: "WEBHOOK_SECRET", +test/conftest.py:26: "SYNC_API_KEY", +test/conftest.py:31: "SEARCH_API_KEY", +test/test_api.py:14:WEBHOOK_SECRET = "w" * 64 +test/test_api.py:15:SYNC_API_KEY = "s" * 64 +test/test_api.py:16:SEARCH_API_KEY = "a" * 64 +test/test_api.py:34: WEBHOOK_SECRET.encode("utf-8"), +test/test_api.py:49: monkeypatch.delenv("WEBHOOK_SECRET") +test/test_api.py:53: match="WEBHOOK_SECRET", +test/test_api.py:63: "SYNC_API_KEY", +test/test_api.py:111: "X-API-Key": SEARCH_API_KEY, +test/test_api.py:133: "X-API-Key": SEARCH_API_KEY, +test/test_api.py:216: "X-API-Key": SYNC_API_KEY, +test/test_api.py:245: "X-API-Key": SYNC_API_KEY, diff --git a/requirements.txt b/requirements.txt index 5d4ad16..83462fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ python-frontmatter==1.3.0 rich==15.0.0 tiktoken>=0.8,<1 uvicorn[standard]==0.48.0 +numpy==2.2.6 +sentence-transformers==5.7.0 diff --git a/scripts/build_sqlite_index.py b/scripts/build_sqlite_index.py index 0ac0bb0..75fdfcd 100644 --- a/scripts/build_sqlite_index.py +++ b/scripts/build_sqlite_index.py @@ -16,7 +16,18 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from scripts.common import CHUNKS_FILE, DB_FILE, DOCUMENTS_FILE, read_json +from scripts.common import ( + CHUNKS_FILE, + DB_FILE, + DOCUMENTS_FILE, + read_json, +) +from scripts.embedding_utils import ( + build_embedding_text, + embed_passages, + embedding_model_name, + vector_to_blob, +) FTS_TOKENIZER = "unicode61 remove_diacritics 2" @@ -33,21 +44,29 @@ def published_to_db(value: Any) -> int | None: return None -def verify_fts5(conn: sqlite3.Connection) -> None: - """Overí, či aktuálna SQLite knižnica podporuje FTS5.""" +def verify_fts5( + conn: sqlite3.Connection, +) -> None: try: conn.execute( - "CREATE VIRTUAL TABLE temp.fts5_check USING fts5(value)" + "CREATE VIRTUAL TABLE temp.fts5_check " + "USING fts5(value)" ) - conn.execute("DROP TABLE temp.fts5_check") + conn.execute( + "DROP TABLE temp.fts5_check" + ) + except sqlite3.OperationalError as error: raise RuntimeError( "Táto inštalácia SQLite nemá dostupné FTS5. " - "Použi Python/SQLite zostavenie s podporou SQLITE_ENABLE_FTS5." + "Použi Python/SQLite zostavenie s podporou " + "SQLITE_ENABLE_FTS5." ) from error -def create_tables(conn: sqlite3.Connection) -> None: +def create_tables( + conn: sqlite3.Connection, +) -> None: conn.executescript( f""" PRAGMA foreign_keys = ON; @@ -58,9 +77,14 @@ def create_tables(conn: sqlite3.Connection) -> None: title TEXT, author TEXT, published INTEGER - CHECK (published IN (0, 1) OR published IS NULL), - content_length INTEGER NOT NULL DEFAULT 0, - metadata_json TEXT NOT NULL DEFAULT '{{}}' + CHECK ( + published IN (0, 1) + OR published IS NULL + ), + content_length INTEGER + NOT NULL DEFAULT 0, + metadata_json TEXT + NOT NULL DEFAULT '{{}}' ); CREATE TABLE chunks ( @@ -70,11 +94,16 @@ def create_tables(conn: sqlite3.Connection) -> None: title TEXT, author TEXT, published INTEGER - CHECK (published IN (0, 1) OR published IS NULL), + CHECK ( + published IN (0, 1) + OR published IS NULL + ), chunk_index INTEGER NOT NULL, - heading_paths_json TEXT NOT NULL DEFAULT '[]', + heading_paths_json TEXT + NOT NULL DEFAULT '[]', text TEXT NOT NULL, - text_length INTEGER NOT NULL DEFAULT 0, + text_length INTEGER + NOT NULL DEFAULT 0, token_count INTEGER, content_hash TEXT, FOREIGN KEY(document_path) @@ -103,6 +132,17 @@ def create_tables(conn: sqlite3.Connection) -> None: ON DELETE CASCADE ); + CREATE TABLE chunk_embeddings ( + chunk_id TEXT PRIMARY KEY, + model TEXT NOT NULL, + dimensions INTEGER NOT NULL, + embedding BLOB NOT NULL, + FOREIGN KEY(chunk_id) + REFERENCES chunks(chunk_id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + CREATE INDEX idx_documents_path ON documents(path); @@ -127,6 +167,9 @@ def create_tables(conn: sqlite3.Connection) -> None: CREATE INDEX idx_chunk_categories_category ON chunk_categories(category); + CREATE INDEX idx_chunk_embeddings_model + ON chunk_embeddings(model); + CREATE VIRTUAL TABLE chunks_fts USING fts5( chunk_id UNINDEXED, title, @@ -151,8 +194,13 @@ def insert_documents( document.get("path"), document.get("title"), document.get("author"), - published_to_db(document.get("published")), - int(document.get("content_length") or 0), + published_to_db( + document.get("published") + ), + int( + document.get("content_length") + or 0 + ), json.dumps( document.get("metadata") or {}, ensure_ascii=False, @@ -184,14 +232,19 @@ def insert_chunks( ) -> None: chunk_rows: list[tuple] = [] tag_rows: list[tuple[str, str]] = [] - category_rows: list[tuple[str, str]] = [] + category_rows: list[ + tuple[str, str] + ] = [] for chunk in chunks: - chunk_id = str(chunk.get("chunk_id") or "").strip() + chunk_id = str( + chunk.get("chunk_id") or "" + ).strip() if not chunk_id: raise ValueError( - "Chunk bez chunk_id nie je možné indexovať" + "Chunk bez chunk_id nie je " + "možné indexovať" ) text = chunk.get("text") or "" @@ -202,14 +255,25 @@ def insert_chunks( chunk.get("document_path"), chunk.get("title"), chunk.get("author"), - published_to_db(chunk.get("published")), - int(chunk.get("chunk_index") or 0), + published_to_db( + chunk.get("published") + ), + int( + chunk.get("chunk_index") + or 0 + ), json.dumps( - chunk.get("heading_paths") or [], + chunk.get( + "heading_paths" + ) + or [], ensure_ascii=False, ), text, - int(chunk.get("text_length") or len(text)), + int( + chunk.get("text_length") + or len(text) + ), chunk.get("token_count"), chunk.get("content_hash"), ) @@ -219,13 +283,27 @@ def insert_chunks( value = str(tag).strip() if value: - tag_rows.append((chunk_id, value)) + tag_rows.append( + ( + chunk_id, + value, + ) + ) - for category in chunk.get("categories") or []: - value = str(category).strip() + for category in ( + chunk.get("categories") or [] + ): + value = str( + category + ).strip() if value: - category_rows.append((chunk_id, value)) + category_rows.append( + ( + chunk_id, + value, + ) + ) conn.executemany( """ @@ -270,8 +348,83 @@ def insert_chunks( ) -def build_fts_index(conn: sqlite3.Connection) -> None: - """Vytvorí FTS5 index nad chunkmi a ich metadátami.""" +def insert_embeddings( + conn: sqlite3.Connection, + chunks: list[dict], +) -> None: + if not chunks: + return + + chunk_ids: list[str] = [] + texts: list[str] = [] + + for chunk in chunks: + chunk_id = str( + chunk.get("chunk_id") or "" + ).strip() + + if not chunk_id: + raise ValueError( + "Chunk bez chunk_id nemôže " + "mať embedding" + ) + + chunk_ids.append( + chunk_id + ) + + texts.append( + build_embedding_text( + chunk + ) + ) + + vectors = embed_passages( + texts + ) + + if len(vectors) != len(chunk_ids): + raise RuntimeError( + "Počet embeddingov sa " + "nezhoduje s počtom chunkov" + ) + + model_name = ( + embedding_model_name() + ) + + rows: list[tuple] = [] + + for chunk_id, vector in zip( + chunk_ids, + vectors, + ): + rows.append( + ( + chunk_id, + model_name, + int(vector.shape[0]), + vector_to_blob(vector), + ) + ) + + conn.executemany( + """ + INSERT INTO chunk_embeddings ( + chunk_id, + model, + dimensions, + embedding + ) + VALUES (?, ?, ?, ?) + """, + rows, + ) + + +def build_fts_index( + conn: sqlite3.Connection, +) -> None: conn.execute( """ INSERT INTO chunks_fts ( @@ -287,60 +440,89 @@ def build_fts_index(conn: sqlite3.Connection) -> None: SELECT chunks.id, chunks.chunk_id, - COALESCE(chunks.title, ''), - COALESCE(chunks.author, ''), + COALESCE( + chunks.title, + '' + ), + COALESCE( + chunks.author, + '' + ), chunks.document_path, - COALESCE(tags.values_text, ''), - COALESCE(categories.values_text, ''), + COALESCE( + tags.values_text, + '' + ), + COALESCE( + categories.values_text, + '' + ), chunks.text FROM chunks LEFT JOIN ( SELECT chunk_id, - GROUP_CONCAT(tag, ' ') AS values_text + GROUP_CONCAT( + tag, + ' ' + ) AS values_text FROM chunk_tags GROUP BY chunk_id ) AS tags - ON tags.chunk_id = chunks.chunk_id + ON tags.chunk_id + = chunks.chunk_id LEFT JOIN ( SELECT chunk_id, - GROUP_CONCAT(category, ' ') AS values_text + GROUP_CONCAT( + category, + ' ' + ) AS values_text FROM chunk_categories GROUP BY chunk_id ) AS categories - ON categories.chunk_id = chunks.chunk_id + ON categories.chunk_id + = chunks.chunk_id ORDER BY chunks.id """ ) conn.execute( - "INSERT INTO chunks_fts(chunks_fts) VALUES('optimize')" + "INSERT INTO chunks_fts" + "(chunks_fts) " + "VALUES('optimize')" ) -def validate_database(conn: sqlite3.Connection) -> None: +def validate_database( + conn: sqlite3.Connection, +) -> None: integrity = conn.execute( "PRAGMA integrity_check" ).fetchone()[0] if integrity != "ok": raise RuntimeError( - f"SQLite integrity check zlyhal: {integrity}" + "SQLite integrity check " + f"zlyhal: {integrity}" ) - foreign_key_errors = conn.execute( - "PRAGMA foreign_key_check" - ).fetchall() + foreign_key_errors = ( + conn.execute( + "PRAGMA foreign_key_check" + ).fetchall() + ) if foreign_key_errors: raise RuntimeError( - "Databáza obsahuje chyby cudzích kľúčov: " + "Databáza obsahuje chyby " + "cudzích kľúčov: " f"{foreign_key_errors[:5]}" ) conn.execute( - "INSERT INTO chunks_fts(chunks_fts) " + "INSERT INTO chunks_fts" + "(chunks_fts) " "VALUES('integrity-check')" ) @@ -352,72 +534,168 @@ def validate_database(conn: sqlite3.Connection) -> None: "SELECT COUNT(*) FROM chunks_fts" ).fetchone()[0] + embedding_count = conn.execute( + """ + SELECT COUNT(*) + FROM chunk_embeddings + """ + ).fetchone()[0] + if chunk_count != fts_count: raise RuntimeError( - "Počet záznamov v chunks a chunks_fts sa nezhoduje: " + "Počet záznamov v chunks " + "a chunks_fts sa nezhoduje: " f"{chunk_count} != {fts_count}" ) + if chunk_count != embedding_count: + raise RuntimeError( + "Počet chunkov a embeddingov " + "sa nezhoduje: " + f"{chunk_count} != " + f"{embedding_count}" + ) + + model_count = conn.execute( + """ + SELECT COUNT( + DISTINCT model + ) + FROM chunk_embeddings + """ + ).fetchone()[0] + + if ( + embedding_count > 0 + and model_count != 1 + ): + raise RuntimeError( + "Embedding index obsahuje " + "viac modelov" + ) + def get_counts( conn: sqlite3.Connection, ) -> dict[str, int]: return { "documents": conn.execute( - "SELECT COUNT(*) FROM documents" + """ + SELECT COUNT(*) + FROM documents + """ ).fetchone()[0], "chunks": conn.execute( - "SELECT COUNT(*) FROM chunks" + """ + SELECT COUNT(*) + FROM chunks + """ ).fetchone()[0], "fts_chunks": conn.execute( - "SELECT COUNT(*) FROM chunks_fts" + """ + SELECT COUNT(*) + FROM chunks_fts + """ + ).fetchone()[0], + "embedding_chunks": conn.execute( + """ + SELECT COUNT(*) + FROM chunk_embeddings + """ ).fetchone()[0], "tags": conn.execute( - "SELECT COUNT(*) FROM chunk_tags" + """ + SELECT COUNT(*) + FROM chunk_tags + """ ).fetchone()[0], "categories": conn.execute( - "SELECT COUNT(*) FROM chunk_categories" + """ + SELECT COUNT(*) + FROM chunk_categories + """ ).fetchone()[0], } -def temporary_database_path(db_file: Path) -> Path: +def temporary_database_path( + db_file: Path, +) -> Path: return db_file.with_name( f".{db_file.name}.tmp" ) def build_database() -> dict[str, int]: - documents = read_json(DOCUMENTS_FILE) - chunks = read_json(CHUNKS_FILE) + documents = read_json( + DOCUMENTS_FILE + ) + + chunks = read_json( + CHUNKS_FILE + ) DB_FILE.parent.mkdir( parents=True, exist_ok=True, ) - temporary_file = temporary_database_path(DB_FILE) + temporary_file = ( + temporary_database_path( + DB_FILE + ) + ) if temporary_file.exists(): temporary_file.unlink() try: - with sqlite3.connect(temporary_file) as conn: - conn.execute("PRAGMA foreign_keys = ON") - conn.execute("PRAGMA temp_store = MEMORY") + with sqlite3.connect( + temporary_file + ) as conn: + conn.execute( + "PRAGMA foreign_keys = ON" + ) + conn.execute( + "PRAGMA temp_store = MEMORY" + ) - verify_fts5(conn) + verify_fts5( + conn + ) with conn: - create_tables(conn) - insert_documents(conn, documents) - insert_chunks(conn, chunks) - build_fts_index(conn) + create_tables( + conn + ) - validate_database(conn) - counts = get_counts(conn) + insert_documents( + conn, + documents, + ) + + insert_chunks( + conn, + chunks, + ) + + build_fts_index( + conn + ) + + insert_embeddings( + conn, + chunks, + ) + + validate_database( + conn + ) + + counts = get_counts( + conn + ) - # Nová databáza nahradí starú až po úspešnom vytvorení. os.replace( temporary_file, DB_FILE, @@ -430,23 +708,38 @@ def build_database() -> dict[str, int]: raise print( - f"[green]SQLite index vytvorený:[/green] " + "[green]SQLite index " + "vytvorený:[/green] " f"{DB_FILE}" ) + print( - f"Dokumentov: {counts['documents']}" + f"Dokumentov: " + f"{counts['documents']}" ) + print( - f"Chunkov: {counts['chunks']}" + f"Chunkov: " + f"{counts['chunks']}" ) + print( - f"FTS5 chunkov: {counts['fts_chunks']}" + f"FTS5 chunkov: " + f"{counts['fts_chunks']}" ) + print( - f"Tag záznamov: {counts['tags']}" + f"Embedding chunkov: " + f"{counts['embedding_chunks']}" ) + print( - f"Kategória záznamov: " + f"Tag záznamov: " + f"{counts['tags']}" + ) + + print( + "Kategória záznamov: " f"{counts['categories']}" ) diff --git a/scripts/embedding_utils.py b/scripts/embedding_utils.py new file mode 100644 index 0000000..3a9a922 --- /dev/null +++ b/scripts/embedding_utils.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import os +from functools import lru_cache +from typing import Any + +import numpy as np + + +DEFAULT_EMBEDDING_MODEL = ( + "intfloat/multilingual-e5-small" +) + +DEFAULT_BATCH_SIZE = 32 + + +def embedding_model_name() -> str: + return ( + os.getenv( + "EMBEDDING_MODEL", + DEFAULT_EMBEDDING_MODEL, + ).strip() + or DEFAULT_EMBEDDING_MODEL + ) + + +def embedding_batch_size() -> int: + raw_value = os.getenv( + "EMBEDDING_BATCH_SIZE", + str(DEFAULT_BATCH_SIZE), + ).strip() + + try: + value = int( + raw_value + ) + + except ValueError: + return DEFAULT_BATCH_SIZE + + return max( + 1, + value, + ) + + +@lru_cache(maxsize=2) +def get_embedding_model( + model_name: str, +): + from sentence_transformers import ( + SentenceTransformer, + ) + + return SentenceTransformer( + model_name + ) + + +def build_embedding_text( + chunk: dict[str, Any], +) -> str: + parts: list[str] = [] + + title = str( + chunk.get("title") + or "" + ).strip() + + author = str( + chunk.get("author") + or "" + ).strip() + + tags = [ + str(value).strip() + for value in ( + chunk.get("tags") + or [] + ) + if str(value).strip() + ] + + categories = [ + str(value).strip() + for value in ( + chunk.get("categories") + or [] + ) + if str(value).strip() + ] + + text = str( + chunk.get("text") + or "" + ).strip() + + if title: + parts.append( + f"Názov: {title}" + ) + + if author: + parts.append( + f"Autor: {author}" + ) + + if tags: + parts.append( + "Tagy: " + + ", ".join( + tags + ) + ) + + if categories: + parts.append( + "Kategórie: " + + ", ".join( + categories + ) + ) + + if text: + parts.append( + text + ) + + return "\n".join( + parts + ) + + +def embed_passages( + texts: list[str], + *, + model_name: str | None = None, +) -> np.ndarray: + if not texts: + return np.empty( + ( + 0, + 0, + ), + dtype=np.float32, + ) + + selected_model = ( + model_name + or embedding_model_name() + ) + + model = get_embedding_model( + selected_model + ) + + prepared = [ + "passage: " + + text.strip() + for text in texts + ] + + vectors = model.encode( + prepared, + batch_size=( + embedding_batch_size() + ), + show_progress_bar=False, + convert_to_numpy=True, + normalize_embeddings=True, + ) + + return np.asarray( + vectors, + dtype=np.float32, + ) + + +def embed_query( + query: str, + *, + model_name: str | None = None, +) -> np.ndarray: + clean_query = query.strip() + + if not clean_query: + raise ValueError( + "Query nesmie byť prázdny" + ) + + selected_model = ( + model_name + or embedding_model_name() + ) + + model = get_embedding_model( + selected_model + ) + + vector = model.encode( + [ + "query: " + + clean_query + ], + batch_size=1, + show_progress_bar=False, + convert_to_numpy=True, + normalize_embeddings=True, + )[0] + + return np.asarray( + vector, + dtype=np.float32, + ) + + +def vector_to_blob( + vector: np.ndarray, +) -> bytes: + normalized = np.asarray( + vector, + dtype=np.float32, + ) + + return normalized.tobytes() + + +def blob_to_vector( + blob: bytes, + dimensions: int, +) -> np.ndarray: + vector = np.frombuffer( + blob, + dtype=np.float32, + ) + + if ( + vector.shape[0] + != dimensions + ): + raise RuntimeError( + "Neplatný rozmer " + "uloženého embeddingu" + ) + + return vector + + +def cosine_similarity( + first: np.ndarray, + second: np.ndarray, +) -> float: + if ( + first.shape + != second.shape + ): + raise ValueError( + "Embeddingy majú " + "rozdielny rozmer" + ) + + first_norm = float( + np.linalg.norm( + first + ) + ) + + second_norm = float( + np.linalg.norm( + second + ) + ) + + if ( + first_norm == 0.0 + or second_norm == 0.0 + ): + return 0.0 + + return float( + np.dot( + first, + second, + ) + / ( + first_norm + * second_norm + ) + ) diff --git a/scripts/search_utils.py b/scripts/search_utils.py index 426d272..fd7f045 100644 --- a/scripts/search_utils.py +++ b/scripts/search_utils.py @@ -8,6 +8,12 @@ from collections import defaultdict from pathlib import Path from typing import Any +from scripts.embedding_utils import ( + blob_to_vector, + cosine_similarity, + embed_query, +) + WORD_RE = re.compile( r"[^\W_]+", @@ -15,9 +21,6 @@ WORD_RE = re.compile( ) -# Poradie zodpovedá stĺpcom v chunks_fts: -# chunk_id, title, author, document_path, -# tags, categories, text BM25_WEIGHTS = ( 0.0, 10.0, @@ -38,6 +41,11 @@ DEFAULT_CANDIDATE_MULTIPLIER = 8 MIN_CANDIDATES = 50 MIN_STEM_PREFIX_LENGTH = 5 +RRF_K = 60 +FTS_RRF_WEIGHT = 1.0 +VECTOR_RRF_WEIGHT = 1.5 +ANY_TERM_RRF_WEIGHT = 0.25 + STRATEGY_PRIORITY = { "all_terms": 3, @@ -49,7 +57,6 @@ STRATEGY_PRIORITY = { def normalize_for_compare( text: str, ) -> str: - """Normalizácia pre pomocné bonusové skóre.""" text = unicodedata.normalize( "NFKD", text.casefold(), @@ -58,7 +65,9 @@ def normalize_for_compare( text = "".join( character for character in text - if not unicodedata.combining(character) + if not unicodedata.combining( + character + ) ) return " ".join( @@ -69,11 +78,12 @@ def normalize_for_compare( def query_tokens( query: str, ) -> list[str]: - """Vytvorí bezpečné tokeny pre FTS5.""" tokens: list[str] = [] seen: set[str] = set() - for token in WORD_RE.findall(query): + for token in WORD_RE.findall( + query + ): normalized = normalize_for_compare( token ) @@ -84,8 +94,13 @@ def query_tokens( if normalized in seen: continue - tokens.append(token) - seen.add(normalized) + tokens.append( + token + ) + + seen.add( + normalized + ) return tokens @@ -100,9 +115,12 @@ def quote_fts_token( if ( shorten - and len(value) > MIN_STEM_PREFIX_LENGTH + and len(value) + > MIN_STEM_PREFIX_LENGTH ): - value = value[:MIN_STEM_PREFIX_LENGTH] + value = value[ + :MIN_STEM_PREFIX_LENGTH + ] escaped = value.replace( '"', @@ -111,7 +129,10 @@ def quote_fts_token( suffix = ( "*" - if use_prefix and len(value) >= 4 + if ( + use_prefix + and len(value) >= 4 + ) else "" ) @@ -121,23 +142,24 @@ def quote_fts_token( def build_match_queries( query: str, ) -> list[tuple[str, str]]: - """ - Vráti stratégie od najpresnejšej: - - all_terms -> prefix_terms -> any_term - """ - tokens = query_tokens(query) + tokens = query_tokens( + query + ) if not tokens: return [] full_terms = [ - quote_fts_token(token) + quote_fts_token( + token + ) for token in tokens ] - all_terms_query = " AND ".join( - full_terms + all_terms_query = ( + " AND ".join( + full_terms + ) ) queries = [ @@ -155,11 +177,16 @@ def build_match_queries( for token in tokens ] - shortened_query = " AND ".join( - shortened_terms + shortened_query = ( + " AND ".join( + shortened_terms + ) ) - if shortened_query != all_terms_query: + if ( + shortened_query + != all_terms_query + ): queries.append( ( "prefix_terms", @@ -171,7 +198,9 @@ def build_match_queries( queries.append( ( "any_term", - " OR ".join(full_terms), + " OR ".join( + full_terms + ), ) ) @@ -181,7 +210,7 @@ def build_match_queries( def verify_search_schema( conn: sqlite3.Connection, ) -> None: - row = conn.execute( + fts_row = conn.execute( """ SELECT 1 FROM sqlite_master @@ -190,24 +219,67 @@ def verify_search_schema( """ ).fetchone() - if row is None: + if fts_row is None: raise RuntimeError( "FTS5 index v databáze chýba. " "Spusti python scripts/rebuild_index.py." ) + embedding_row = conn.execute( + """ + SELECT 1 + FROM sqlite_master + WHERE type = 'table' + AND name = 'chunk_embeddings' + """ + ).fetchone() + + if embedding_row is None: + raise RuntimeError( + "Embedding index v databáze chýba. " + "Spusti python scripts/rebuild_index.py." + ) + + +def embedding_index_info( + conn: sqlite3.Connection, +) -> tuple[str, int]: + row = conn.execute( + """ + SELECT + model, + dimensions + FROM chunk_embeddings + LIMIT 1 + """ + ).fetchone() + + if row is None: + raise RuntimeError( + "Embedding index je prázdny" + ) + + return ( + str(row["model"]), + int(row["dimensions"]), + ) + def make_source_url( document_path: str, ) -> str: clean_path = document_path - if clean_path.startswith("pages/"): + if clean_path.startswith( + "pages/" + ): clean_path = clean_path[ len("pages/"): ] - if clean_path.endswith("/README.md"): + if clean_path.endswith( + "/README.md" + ): clean_path = clean_path[ :-len("/README.md") ] @@ -234,22 +306,37 @@ def load_labels( rows = conn.execute( f""" - SELECT chunk_id, {column} + SELECT + chunk_id, + {column} FROM {table} - WHERE chunk_id IN ({placeholders}) - ORDER BY chunk_id, {column} + WHERE chunk_id IN ( + {placeholders} + ) + ORDER BY + chunk_id, + {column} """, chunk_ids, ).fetchall() - values: dict[str, list[str]] = defaultdict( + values: dict[ + str, + list[str], + ] = defaultdict( list ) for chunk_id, value in rows: - values[chunk_id].append(value) + values[ + chunk_id + ].append( + value + ) - return dict(values) + return dict( + values + ) def run_fts_query( @@ -309,6 +396,113 @@ def run_fts_query( ] +def run_vector_query( + conn: sqlite3.Connection, + query: str, + candidate_limit: int, + published_only: bool, +) -> list[dict[str, Any]]: + model_name, dimensions = ( + embedding_index_info( + conn + ) + ) + + query_vector = embed_query( + query, + model_name=model_name, + ) + + if ( + int(query_vector.shape[0]) + != dimensions + ): + raise RuntimeError( + "Rozmer query embeddingu " + "sa nezhoduje s indexom" + ) + + rows = conn.execute( + """ + SELECT + chunks.chunk_id, + chunks.document_path, + chunks.title, + chunks.author, + chunks.published, + chunks.chunk_index, + chunks.heading_paths_json, + chunks.text, + chunks.text_length, + chunks.token_count, + chunks.content_hash, + chunk_embeddings.embedding, + chunk_embeddings.dimensions + FROM chunk_embeddings + JOIN chunks + ON chunks.chunk_id + = chunk_embeddings.chunk_id + WHERE ( + ? = 0 + OR chunks.published = 1 + ) + """, + ( + 1 if published_only else 0, + ), + ).fetchall() + + candidates: list[ + dict[str, Any] + ] = [] + + for row in rows: + item = dict( + row + ) + + blob = item.pop( + "embedding" + ) + + stored_dimensions = int( + item.pop( + "dimensions" + ) + ) + + vector = blob_to_vector( + blob, + stored_dimensions, + ) + + similarity = cosine_similarity( + query_vector, + vector, + ) + + item["vector_score"] = round( + similarity, + 6, + ) + + candidates.append( + item + ) + + candidates.sort( + key=lambda item: ( + -item["vector_score"], + item["document_path"], + item["chunk_index"], + ) + ) + + return candidates[ + :candidate_limit + ] + + def exact_match_bonus( query: str, item: dict[str, Any], @@ -331,7 +525,8 @@ def exact_match_bonus( ) path = normalize_for_compare( - item.get("document_path") or "" + item.get("document_path") + or "" ) text = normalize_for_compare( @@ -339,12 +534,16 @@ def exact_match_bonus( ) normalized_tags = [ - normalize_for_compare(value) + normalize_for_compare( + value + ) for value in tags ] normalized_categories = [ - normalize_for_compare(value) + normalize_for_compare( + value + ) for value in categories ] @@ -365,10 +564,14 @@ def exact_match_bonus( if normalized_query in path: bonus += 2.0 - if normalized_query in normalized_tags: + if normalized_query in ( + normalized_tags + ): bonus += 4.0 - if normalized_query in normalized_categories: + if normalized_query in ( + normalized_categories + ): bonus += 3.0 if normalized_query in text: @@ -380,57 +583,108 @@ def exact_match_bonus( def database_bool( value: Any, ) -> bool | None: - """Prevedie SQLite 0/1 na API boolean.""" if value is None: return None - return bool(value) + return bool( + value + ) -def add_labels_and_scores( +def parse_heading_paths( + item: dict[str, Any], +) -> list: + try: + return json.loads( + item.pop( + "heading_paths_json" + ) + or "[]" + ) + + except json.JSONDecodeError: + return [] + + +def load_result_labels( conn: sqlite3.Connection, - query: str, - candidates: list[dict[str, Any]], -) -> list[dict[str, Any]]: + candidates: list[ + dict[str, Any] + ], +) -> tuple[ + dict[str, list[str]], + dict[str, list[str]], +]: chunk_ids = [ item["chunk_id"] for item in candidates ] - tags_by_chunk = load_labels( + tags = load_labels( conn, "chunk_tags", "tag", chunk_ids, ) - categories_by_chunk = load_labels( + categories = load_labels( conn, "chunk_categories", "category", chunk_ids, ) - results: list[dict[str, Any]] = [] + return ( + tags, + categories, + ) + + +def add_fts_metadata( + conn: sqlite3.Connection, + query: str, + candidates: list[ + dict[str, Any] + ], +) -> list[dict[str, Any]]: + ( + tags_by_chunk, + categories_by_chunk, + ) = load_result_labels( + conn, + candidates, + ) + + results: list[ + dict[str, Any] + ] = [] for item in candidates: - chunk_id = item["chunk_id"] + chunk_id = item[ + "chunk_id" + ] tags = tags_by_chunk.get( chunk_id, [], ) - categories = categories_by_chunk.get( - chunk_id, - [], + categories = ( + categories_by_chunk.get( + chunk_id, + [], + ) ) bm25_score = float( - item.pop("bm25_score") + item.pop( + "bm25_score" + ) ) - strategy = item.pop("strategy") + strategy = item.pop( + "strategy" + ) base_score = max( 0.0, @@ -447,23 +701,22 @@ def add_labels_and_scores( ) ) - try: - heading_paths = json.loads( - item.pop( - "heading_paths_json" - ) - or "[]" + heading_paths = ( + parse_heading_paths( + item ) - - except json.JSONDecodeError: - heading_paths = [] + ) item["published"] = database_bool( - item.get("published") + item.get( + "published" + ) ) item["_strategy_priority"] = ( - STRATEGY_PRIORITY[strategy] + STRATEGY_PRIORITY[ + strategy + ] ) item.update( @@ -486,7 +739,9 @@ def add_labels_and_scores( } ) - results.append(item) + results.append( + item + ) results.sort( key=lambda item: ( @@ -507,18 +762,269 @@ def add_labels_and_scores( return results +def add_vector_metadata( + conn: sqlite3.Connection, + candidates: list[ + dict[str, Any] + ], +) -> list[dict[str, Any]]: + ( + tags_by_chunk, + categories_by_chunk, + ) = load_result_labels( + conn, + candidates, + ) + + results: list[ + dict[str, Any] + ] = [] + + for item in candidates: + chunk_id = item[ + "chunk_id" + ] + + item["published"] = database_bool( + item.get( + "published" + ) + ) + + item["heading_paths"] = ( + parse_heading_paths( + item + ) + ) + + item["tags"] = ( + tags_by_chunk.get( + chunk_id, + [], + ) + ) + + item["categories"] = ( + categories_by_chunk.get( + chunk_id, + [], + ) + ) + + item["source_url"] = make_source_url( + item["document_path"] + ) + + text = ( + item.get("text") + or "" + ) + + item["snippet"] = ( + text[:320].strip() + ) + + results.append( + item + ) + + return results + + +def fuse_hybrid_results( + fts_results: list[ + dict[str, Any] + ], + vector_results: list[ + dict[str, Any] + ], +) -> list[dict[str, Any]]: + merged: dict[ + str, + dict[str, Any], + ] = {} + + scores: dict[ + str, + float, + ] = defaultdict( + float + ) + + fts_ranks: dict[ + str, + int, + ] = {} + + vector_ranks: dict[ + str, + int, + ] = {} + + for rank, item in enumerate( + fts_results, + start=1, + ): + chunk_id = item[ + "chunk_id" + ] + + merged[ + chunk_id + ] = dict( + item + ) + + fts_ranks[ + chunk_id + ] = rank + + strategy = item.get( + "match_strategy" + ) + + fts_weight = ( + ANY_TERM_RRF_WEIGHT + if strategy == "any_term" + else FTS_RRF_WEIGHT + ) + + scores[ + chunk_id + ] += ( + fts_weight + / ( + RRF_K + + rank + ) + ) + + for rank, item in enumerate( + vector_results, + start=1, + ): + chunk_id = item[ + "chunk_id" + ] + + vector_ranks[ + chunk_id + ] = rank + + scores[ + chunk_id + ] += ( + VECTOR_RRF_WEIGHT + / ( + RRF_K + + rank + ) + ) + + if chunk_id not in merged: + merged[ + chunk_id + ] = dict( + item + ) + + else: + merged[ + chunk_id + ][ + "vector_score" + ] = item[ + "vector_score" + ] + + results: list[ + dict[str, Any] + ] = [] + + for chunk_id, item in ( + merged.items() + ): + item["fts_score"] = ( + item.get("score") + ) + + item["fts_rank"] = ( + fts_ranks.get( + chunk_id + ) + ) + + item["vector_rank"] = ( + vector_ranks.get( + chunk_id + ) + ) + + item.setdefault( + "vector_score", + None, + ) + + item.setdefault( + "bm25_score", + None, + ) + + item.setdefault( + "match_strategy", + None, + ) + + hybrid_score = scores[ + chunk_id + ] + + item["hybrid_score"] = round( + hybrid_score, + 8, + ) + + item["score"] = round( + hybrid_score, + 8, + ) + + results.append( + item + ) + + results.sort( + key=lambda item: ( + -item["hybrid_score"], + item["document_path"], + item["chunk_index"], + ) + ) + + return results + + def diversify_results( - results: list[dict[str, Any]], + results: list[ + dict[str, Any] + ], limit: int, max_per_document: int, ) -> list[dict[str, Any]]: if max_per_document <= 0: - return results[:limit] + return results[ + :limit + ] - selected: list[dict[str, Any]] = [] + selected: list[ + dict[str, Any] + ] = [] - document_counts: dict[str, int] = ( - defaultdict(int) + document_counts: dict[ + str, + int, + ] = defaultdict( + int ) for item in results: @@ -527,12 +1033,16 @@ def diversify_results( ] if ( - document_counts[document_path] + document_counts[ + document_path + ] >= max_per_document ): continue - selected.append(item) + selected.append( + item + ) document_counts[ document_path @@ -553,14 +1063,17 @@ def search_database( ) -> dict[str, Any]: if not db_file.exists(): raise FileNotFoundError( - f"Databáza neexistuje: {db_file}" + "Databáza neexistuje: " + f"{db_file}" ) clean_query = query.strip() if not clean_query: return { - "engine": "sqlite_fts5", + "engine": ( + "hybrid_fts5_embeddings" + ), "strategies": [], "results": [], } @@ -569,43 +1082,42 @@ def search_database( clean_query ) - if not match_queries: - return { - "engine": "sqlite_fts5", - "strategies": [], - "results": [], - } - candidate_limit = max( MIN_CANDIDATES, - limit * DEFAULT_CANDIDATE_MULTIPLIER, + ( + limit + * DEFAULT_CANDIDATE_MULTIPLIER + ), ) with sqlite3.connect( db_file, timeout=5.0, ) as conn: - conn.row_factory = sqlite3.Row + conn.row_factory = ( + sqlite3.Row + ) + conn.execute( "PRAGMA query_only = ON" ) - verify_search_schema(conn) + verify_search_schema( + conn + ) - candidates: list[ + fts_candidates: list[ dict[str, Any] ] = [] - used_strategies: list[str] = [] + used_strategies: list[ + str + ] = [] - # Použije sa iba prvá stratégia, - # ktorá nájde aspoň jeden výsledok: - # - # all_terms -> prefix_terms -> any_term - # - # any_term teda nedopĺňa presné - # výsledky nerelevantným obsahom. - for strategy, match_query in match_queries: + for ( + strategy, + match_query, + ) in match_queries: rows = run_fts_query( conn, match_query, @@ -617,26 +1129,71 @@ def search_database( continue for row in rows: - row["strategy"] = strategy + row[ + "strategy" + ] = strategy + + fts_candidates = rows - candidates = rows used_strategies = [ strategy ] break - results = add_labels_and_scores( + fts_results = add_fts_metadata( conn, clean_query, - candidates, + fts_candidates, + ) + + vector_candidates = run_vector_query( + conn, + clean_query, + candidate_limit, + published_only, + ) + + vector_results = add_vector_metadata( + conn, + vector_candidates, + ) + + # Presné FTS výsledky. + if ( + used_strategies + and used_strategies[0] + in { + "all_terms", + "prefix_terms", + } + ): + fts_chunk_ids = { + item["chunk_id"] + for item in fts_results + } + + vector_results = [ + item + for item in vector_results + if item["chunk_id"] + in fts_chunk_ids + ] + + hybrid_results = fuse_hybrid_results( + fts_results, + vector_results, ) return { - "engine": "sqlite_fts5", - "strategies": used_strategies, + "engine": ( + "hybrid_fts5_embeddings" + ), + "strategies": ( + used_strategies + ), "results": diversify_results( - results, + hybrid_results, limit, max_per_document, ), diff --git a/tatus b/tatus new file mode 100644 index 0000000..67f74af --- /dev/null +++ b/tatus @@ -0,0 +1,40 @@ +README.md:66:WEBHOOK_SECRET= +README.md:67:SYNC_API_KEY= +README.md:68:SEARCH_API_KEY=<ďalšia náhodná hodnota s minimálne 32 znakmi> +README.md:162: -H "X-API-Key: $SEARCH_API_KEY" \ +README.md:182: -H "X-API-Key: $SYNC_API_KEY" \ +app/main.py:41:SEARCH_API_KEY_HEADER = "X-API-Key" +app/main.py:42:SYNC_API_KEY_HEADER = "X-API-Key" +app/main.py:45:search_api_key_scheme = APIKeyHeader( +app/main.py:46: name=SEARCH_API_KEY_HEADER, +app/main.py:51:sync_api_key_scheme = APIKeyHeader( +app/main.py:52: name=SYNC_API_KEY_HEADER, +app/main.py:135: validate_secret("WEBHOOK_SECRET") +app/main.py:136: validate_secret("SYNC_API_KEY") +app/main.py:137: validate_secret("SEARCH_API_KEY") +app/main.py:158:def require_search_api_key( +app/main.py:160: search_api_key_scheme +app/main.py:164: "SEARCH_API_KEY" +app/main.py:183:def require_sync_api_key( +app/main.py:185: sync_api_key_scheme +app/main.py:189: "SYNC_API_KEY" +app/main.py:293: "WEBHOOK_SECRET", +app/main.py:294: "SYNC_API_KEY", +app/main.py:295: "SEARCH_API_KEY", +app/main.py:305: Depends(require_search_api_key) +app/main.py:356: Depends(require_sync_api_key) +app/main.py:407: "WEBHOOK_SECRET" +test/conftest.py:21: "WEBHOOK_SECRET", +test/conftest.py:26: "SYNC_API_KEY", +test/conftest.py:31: "SEARCH_API_KEY", +test/test_api.py:14:WEBHOOK_SECRET = "w" * 64 +test/test_api.py:15:SYNC_API_KEY = "s" * 64 +test/test_api.py:16:SEARCH_API_KEY = "a" * 64 +test/test_api.py:34: WEBHOOK_SECRET.encode("utf-8"), +test/test_api.py:49: monkeypatch.delenv("WEBHOOK_SECRET") +test/test_api.py:53: match="WEBHOOK_SECRET", +test/test_api.py:63: "SYNC_API_KEY", +test/test_api.py:111: "X-API-Key": SEARCH_API_KEY, +test/test_api.py:133: "X-API-Key": SEARCH_API_KEY, +test/test_api.py:216: "X-API-Key": SYNC_API_KEY, +test/test_api.py:245: "X-API-Key": SYNC_API_KEY, diff --git a/test/conftest.py b/test/conftest.py index 564d5a7..bf67c00 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,7 +5,6 @@ from pathlib import Path import pytest - PROJECT_ROOT = Path(__file__).resolve().parents[1] if str(PROJECT_ROOT) not in sys.path: @@ -17,10 +16,28 @@ def security_environment( monkeypatch: pytest.MonkeyPatch, ) -> None: """Každý test dostane platnú bezpečnostnú konfiguráciu.""" - monkeypatch.setenv("WEBHOOK_SECRET", "w" * 64) - monkeypatch.setenv("SYNC_API_KEY", "s" * 64) + + monkeypatch.setenv( + "WEBHOOK_SECRET", + "w" * 64, + ) + + monkeypatch.setenv( + "SYNC_API_KEY", + "s" * 64, + ) + + monkeypatch.setenv( + "SEARCH_API_KEY", + "a" * 64, + ) + monkeypatch.setenv( "EXPECTED_GITEA_REPOSITORY", "KEMT/zpwiki", ) - monkeypatch.setenv("WEBHOOK_PULL_GIT", "false") + + monkeypatch.setenv( + "WEBHOOK_PULL_GIT", + "false", + ) diff --git a/test/test_api.py b/test/test_api.py index bad7c22..5b9c2ab 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -13,6 +13,7 @@ from scripts.rebuild_index import ReindexInProgressError WEBHOOK_SECRET = "w" * 64 SYNC_API_KEY = "s" * 64 +SEARCH_API_KEY = "a" * 64 def fake_rebuild_result() -> dict: @@ -47,7 +48,10 @@ def test_startup_rejects_missing_secret( ) -> None: monkeypatch.delenv("WEBHOOK_SECRET") - with pytest.raises(RuntimeError, match="WEBHOOK_SECRET"): + with pytest.raises( + RuntimeError, + match="WEBHOOK_SECRET", + ): with TestClient(main.app): pass @@ -55,18 +59,28 @@ def test_startup_rejects_missing_secret( def test_startup_rejects_short_secret( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setenv("SYNC_API_KEY", "short") + monkeypatch.setenv( + "SYNC_API_KEY", + "short", + ) - with pytest.raises(RuntimeError, match="aspoň 32"): + with pytest.raises( + RuntimeError, + match="aspoň 32", + ): with TestClient(main.app): pass -def test_health_endpoint(client: TestClient) -> None: +def test_health_endpoint( + client: TestClient, +) -> None: response = client.get("/health") assert response.status_code == 200 + payload = response.json() + assert payload["status"] == "ok" assert payload["search_engine"] == "sqlite_fts5" assert payload["security_configured"] is True @@ -82,36 +96,107 @@ def test_search_endpoint_uses_shared_search_logic( lambda *args, **kwargs: { "engine": "sqlite_fts5", "strategies": ["all_terms"], - "results": [{"chunk_id": "test::0", "published": True}], + "results": [ + { + "chunk_id": "test::0", + "published": True, + } + ], }, ) response = client.post( "/search", - json={"query": "jan ptak", "limit": 5}, + headers={ + "X-API-Key": SEARCH_API_KEY, + }, + json={ + "query": "jan ptak", + "limit": 5, + }, ) assert response.status_code == 200 assert response.json()["count"] == 1 - assert response.json()["results"][0]["chunk_id"] == "test::0" + assert ( + response.json()["results"][0]["chunk_id"] + == "test::0" + ) -def test_search_rejects_empty_query(client: TestClient) -> None: - response = client.post("/search", json={"query": ""}) +def test_search_rejects_empty_query( + client: TestClient, +) -> None: + response = client.post( + "/search", + headers={ + "X-API-Key": SEARCH_API_KEY, + }, + json={ + "query": "", + }, + ) + assert response.status_code == 422 -def test_sync_rejects_missing_api_key(client: TestClient) -> None: - response = client.post("/sync", json={"pull_git": False}) +def test_search_rejects_missing_api_key( + client: TestClient, +) -> None: + response = client.post( + "/search", + json={ + "query": "jan ptak", + "limit": 5, + }, + ) + assert response.status_code == 401 -def test_sync_rejects_wrong_api_key(client: TestClient) -> None: +def test_search_rejects_wrong_api_key( + client: TestClient, +) -> None: + response = client.post( + "/search", + headers={ + "X-API-Key": "x" * 64, + }, + json={ + "query": "jan ptak", + "limit": 5, + }, + ) + + assert response.status_code == 401 + + +def test_sync_rejects_missing_api_key( + client: TestClient, +) -> None: response = client.post( "/sync", - headers={"X-API-Key": "x" * 64}, - json={"pull_git": False}, + json={ + "pull_git": False, + }, ) + + assert response.status_code == 401 + + +def test_sync_rejects_wrong_api_key( + client: TestClient, +) -> None: + response = client.post( + "/sync", + headers={ + "X-API-Key": "x" * 64, + }, + json={ + "pull_git": False, + }, + ) + assert response.status_code == 401 @@ -127,8 +212,12 @@ def test_sync_accepts_valid_api_key( response = client.post( "/sync", - headers={"X-API-Key": SYNC_API_KEY}, - json={"pull_git": False}, + headers={ + "X-API-Key": SYNC_API_KEY, + }, + json={ + "pull_git": False, + }, ) assert response.status_code == 200 @@ -140,22 +229,38 @@ def test_sync_returns_conflict_when_reindex_is_running( monkeypatch: pytest.MonkeyPatch, ) -> None: def busy(*args, **kwargs): - raise ReindexInProgressError("Reindexovanie už prebieha") + raise ReindexInProgressError( + "Reindexovanie už prebieha" + ) - monkeypatch.setattr(main, "rebuild_index", busy) + monkeypatch.setattr( + main, + "rebuild_index", + busy, + ) response = client.post( "/sync", - headers={"X-API-Key": SYNC_API_KEY}, - json={"pull_git": False}, + headers={ + "X-API-Key": SYNC_API_KEY, + }, + json={ + "pull_git": False, + }, ) assert response.status_code == 409 -def test_webhook_rejects_invalid_signature(client: TestClient) -> None: +def test_webhook_rejects_invalid_signature( + client: TestClient, +) -> None: body = json.dumps( - {"repository": {"full_name": "KEMT/zpwiki"}} + { + "repository": { + "full_name": "KEMT/zpwiki", + } + } ).encode("utf-8") response = client.post( @@ -189,9 +294,15 @@ def test_webhook_rejects_invalid_json_with_valid_signature( assert response.status_code == 400 -def test_webhook_requires_event_header(client: TestClient) -> None: +def test_webhook_requires_event_header( + client: TestClient, +) -> None: body = json.dumps( - {"repository": {"full_name": "KEMT/zpwiki"}} + { + "repository": { + "full_name": "KEMT/zpwiki", + } + } ).encode("utf-8") response = client.post( @@ -217,9 +328,18 @@ def test_webhook_ignores_non_push_event( calls += 1 return fake_rebuild_result() - monkeypatch.setattr(main, "rebuild_index", fake_rebuild) + monkeypatch.setattr( + main, + "rebuild_index", + fake_rebuild, + ) + body = json.dumps( - {"repository": {"full_name": "KEMT/zpwiki"}} + { + "repository": { + "full_name": "KEMT/zpwiki", + } + } ).encode("utf-8") response = client.post( @@ -237,9 +357,15 @@ def test_webhook_ignores_non_push_event( assert calls == 0 -def test_webhook_rejects_unexpected_repository(client: TestClient) -> None: +def test_webhook_rejects_unexpected_repository( + client: TestClient, +) -> None: body = json.dumps( - {"repository": {"full_name": "OTHER/repository"}} + { + "repository": { + "full_name": "OTHER/repository", + } + } ).encode("utf-8") response = client.post( @@ -264,8 +390,13 @@ def test_webhook_accepts_signed_push( "rebuild_index", lambda pull_git=False: fake_rebuild_result(), ) + body = json.dumps( - {"repository": {"full_name": "KEMT/zpwiki"}} + { + "repository": { + "full_name": "KEMT/zpwiki", + } + } ).encode("utf-8") response = client.post( @@ -279,8 +410,14 @@ def test_webhook_accepts_signed_push( ) assert response.status_code == 200 - assert response.json()["verified_by"] == "hmac_sha256" - assert response.json()["repository"] == "KEMT/zpwiki" + assert ( + response.json()["verified_by"] + == "hmac_sha256" + ) + assert ( + response.json()["repository"] + == "KEMT/zpwiki" + ) def test_webhook_returns_conflict_when_reindex_is_running( @@ -288,11 +425,22 @@ def test_webhook_returns_conflict_when_reindex_is_running( monkeypatch: pytest.MonkeyPatch, ) -> None: def busy(*args, **kwargs): - raise ReindexInProgressError("Reindexovanie už prebieha") + raise ReindexInProgressError( + "Reindexovanie už prebieha" + ) + + monkeypatch.setattr( + main, + "rebuild_index", + busy, + ) - monkeypatch.setattr(main, "rebuild_index", busy) body = json.dumps( - {"repository": {"full_name": "KEMT/zpwiki"}} + { + "repository": { + "full_name": "KEMT/zpwiki", + } + } ).encode("utf-8") response = client.post( diff --git a/test/test_database.py b/test/test_database.py index 63e16a2..37831cb 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -10,9 +10,16 @@ import scripts.build_sqlite_index as indexer def write_json(path: Path, value) -> None: - path.parent.mkdir(parents=True, exist_ok=True) + path.parent.mkdir( + parents=True, + exist_ok=True, + ) + path.write_text( - json.dumps(value, ensure_ascii=False), + json.dumps( + value, + ensure_ascii=False, + ), encoding="utf-8", ) @@ -25,7 +32,9 @@ def sample_documents() -> list[dict]: "author": "Autor", "published": True, "content_length": 100, - "metadata": {"published": True}, + "metadata": { + "published": True, + }, } ] @@ -33,19 +42,33 @@ def sample_documents() -> list[dict]: def sample_chunks() -> list[dict]: return [ { - "chunk_id": "pages/test/README.md::chunk-0", - "document_path": "pages/test/README.md", + "chunk_id": ( + "pages/test/README.md::chunk-0" + ), + "document_path": ( + "pages/test/README.md" + ), "title": "Strojový preklad", "author": "Autor", "published": True, "chunk_index": 0, - "heading_paths": [["Úvod"]], - "text": "Dokument: Strojový preklad. Neurónový preklad textu.", + "heading_paths": [ + ["Úvod"] + ], + "text": ( + "Dokument: Strojový preklad. " + "Neurónový preklad textu." + ), "text_length": 58, "token_count": 16, "content_hash": "abc", - "tags": ["translation", "nlp"], - "categories": ["project"], + "tags": [ + "translation", + "nlp", + ], + "categories": [ + "project", + ], } ] @@ -53,26 +76,66 @@ def sample_chunks() -> list[dict]: def configure_indexer( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, -) -> tuple[Path, Path, Path]: - documents_file = tmp_path / "documents.json" - chunks_file = tmp_path / "chunks.json" - db_file = tmp_path / "zp_index.sqlite" +) -> tuple[ + Path, + Path, + Path, +]: + documents_file = ( + tmp_path / "documents.json" + ) - write_json(documents_file, sample_documents()) - write_json(chunks_file, sample_chunks()) + chunks_file = ( + tmp_path / "chunks.json" + ) - monkeypatch.setattr(indexer, "DOCUMENTS_FILE", documents_file) - monkeypatch.setattr(indexer, "CHUNKS_FILE", chunks_file) - monkeypatch.setattr(indexer, "DB_FILE", db_file) + db_file = ( + tmp_path / "zp_index.sqlite" + ) - return documents_file, chunks_file, db_file + write_json( + documents_file, + sample_documents(), + ) + + write_json( + chunks_file, + sample_chunks(), + ) + + monkeypatch.setattr( + indexer, + "DOCUMENTS_FILE", + documents_file, + ) + + monkeypatch.setattr( + indexer, + "CHUNKS_FILE", + chunks_file, + ) + + monkeypatch.setattr( + indexer, + "DB_FILE", + db_file, + ) + + return ( + documents_file, + chunks_file, + db_file, + ) def test_database_contains_documents_chunks_metadata_and_fts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _, _, db_file = configure_indexer(monkeypatch, tmp_path) + _, _, db_file = configure_indexer( + monkeypatch, + tmp_path, + ) counts = indexer.build_database() @@ -80,55 +143,205 @@ def test_database_contains_documents_chunks_metadata_and_fts( "documents": 1, "chunks": 1, "fts_chunks": 1, + "embedding_chunks": 1, "tags": 2, "categories": 1, } - with sqlite3.connect(db_file) as conn: - assert conn.execute("PRAGMA integrity_check").fetchone()[0] == "ok" - assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] - assert conn.execute("SELECT published FROM chunks").fetchone()[0] == 1 - assert conn.execute("SELECT COUNT(*) FROM chunk_tags").fetchone()[0] == 2 - assert conn.execute("SELECT COUNT(*) FROM chunk_categories").fetchone()[0] == 1 - assert conn.execute( - "SELECT COUNT(*) FROM chunks_fts WHERE chunks_fts MATCH 'strojovy'" - ).fetchone()[0] == 1 + with sqlite3.connect( + db_file + ) as conn: + assert ( + conn.execute( + "PRAGMA integrity_check" + ).fetchone()[0] + == "ok" + ) + + assert ( + conn.execute( + "PRAGMA foreign_key_check" + ).fetchall() + == [] + ) + + assert ( + conn.execute( + """ + SELECT published + FROM chunks + """ + ).fetchone()[0] + == 1 + ) + + assert ( + conn.execute( + """ + SELECT COUNT(*) + FROM chunk_tags + """ + ).fetchone()[0] + == 2 + ) + + assert ( + conn.execute( + """ + SELECT COUNT(*) + FROM chunk_categories + """ + ).fetchone()[0] + == 1 + ) + + assert ( + conn.execute( + """ + SELECT COUNT(*) + FROM chunks_fts + WHERE chunks_fts + MATCH 'strojovy' + """ + ).fetchone()[0] + == 1 + ) + + assert ( + conn.execute( + """ + SELECT COUNT(*) + FROM chunk_embeddings + """ + ).fetchone()[0] + == 1 + ) + + embedding_row = conn.execute( + """ + SELECT + model, + dimensions, + LENGTH(embedding) + FROM chunk_embeddings + """ + ).fetchone() + + assert embedding_row is not None + + model, dimensions, blob_size = ( + embedding_row + ) + + assert model + assert dimensions > 0 + + # float32 = 4 bajty. + assert ( + blob_size + == dimensions * 4 + ) def test_database_rebuild_replaces_old_database_only_after_success( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _, _, db_file = configure_indexer(monkeypatch, tmp_path) + _, _, db_file = configure_indexer( + monkeypatch, + tmp_path, + ) + + with sqlite3.connect( + db_file + ) as conn: + conn.execute( + """ + CREATE TABLE marker( + value TEXT + ) + """ + ) + + conn.execute( + """ + INSERT INTO marker + VALUES ('old database') + """ + ) - with sqlite3.connect(db_file) as conn: - conn.execute("CREATE TABLE marker(value TEXT)") - conn.execute("INSERT INTO marker VALUES ('old database')") conn.commit() - def fail_validation(conn: sqlite3.Connection) -> None: - raise RuntimeError("úmyselná chyba validácie") + def fail_validation( + conn: sqlite3.Connection, + ) -> None: + raise RuntimeError( + "úmyselná chyba validácie" + ) - monkeypatch.setattr(indexer, "validate_database", fail_validation) + monkeypatch.setattr( + indexer, + "validate_database", + fail_validation, + ) - with pytest.raises(RuntimeError, match="úmyselná chyba"): + with pytest.raises( + RuntimeError, + match="úmyselná chyba", + ): indexer.build_database() - with sqlite3.connect(db_file) as conn: - assert conn.execute("SELECT value FROM marker").fetchone()[0] == "old database" + with sqlite3.connect( + db_file + ) as conn: + assert ( + conn.execute( + """ + SELECT value + FROM marker + """ + ).fetchone()[0] + == "old database" + ) - assert not indexer.temporary_database_path(db_file).exists() + assert not ( + indexer.temporary_database_path( + db_file + ).exists() + ) def test_database_rejects_chunk_without_chunk_id( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - documents_file, chunks_file, _ = configure_indexer(monkeypatch, tmp_path) - broken = sample_chunks() - broken[0]["chunk_id"] = "" - write_json(documents_file, sample_documents()) - write_json(chunks_file, broken) + ( + documents_file, + chunks_file, + _, + ) = configure_indexer( + monkeypatch, + tmp_path, + ) - with pytest.raises(ValueError, match="chunk_id"): + broken = sample_chunks() + + broken[0][ + "chunk_id" + ] = "" + + write_json( + documents_file, + sample_documents(), + ) + + write_json( + chunks_file, + broken, + ) + + with pytest.raises( + ValueError, + match="chunk_id", + ): indexer.build_database() diff --git a/test/test_search.py b/test/test_search.py index 11cee65..d8e21c2 100644 --- a/test/test_search.py +++ b/test/test_search.py @@ -11,7 +11,13 @@ from scripts.search_utils import search_database def write_json(path: Path, value) -> None: - path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8") + path.write_text( + json.dumps( + value, + ensure_ascii=False, + ), + encoding="utf-8", + ) def build_search_database( @@ -141,18 +147,43 @@ def build_search_database( "deep::0", "pages/topics/deep/README.md", "Neurónové siete", - "Modely hlbokého učenia a trénovanie hlbokých neurónových sietí.", + ( + "Modely hlbokého učenia a trénovanie " + "hlbokých neurónových sietí." + ), False, tags=["nn"], ), ] - write_json(documents_file, documents) - write_json(chunks_file, chunks) + write_json( + documents_file, + documents, + ) + + write_json( + chunks_file, + chunks, + ) + + monkeypatch.setattr( + indexer, + "DOCUMENTS_FILE", + documents_file, + ) + + monkeypatch.setattr( + indexer, + "CHUNKS_FILE", + chunks_file, + ) + + monkeypatch.setattr( + indexer, + "DB_FILE", + db_file, + ) - monkeypatch.setattr(indexer, "DOCUMENTS_FILE", documents_file) - monkeypatch.setattr(indexer, "CHUNKS_FILE", chunks_file) - monkeypatch.setattr(indexer, "DB_FILE", db_file) indexer.build_database() return db_file @@ -162,58 +193,124 @@ def test_person_query_does_not_add_any_term_noise( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - db_file = build_search_database(tmp_path, monkeypatch) + db_file = build_search_database( + tmp_path, + monkeypatch, + ) - response = search_database(db_file, "jan ptak", limit=5) + response = search_database( + db_file, + "jan ptak", + limit=5, + ) - assert response["strategies"] == ["all_terms"] - assert [item["title"] for item in response["results"]] == ["Ján Pták"] + assert response["strategies"] == [ + "all_terms" + ] + + assert [ + item["title"] + for item in response["results"] + ] == [ + "Ján Pták" + ] def test_topic_query_returns_best_topic_first( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - db_file = build_search_database(tmp_path, monkeypatch) + db_file = build_search_database( + tmp_path, + monkeypatch, + ) - response = search_database(db_file, "strojovy preklad", limit=5) + response = search_database( + db_file, + "strojovy preklad", + limit=5, + ) - assert response["strategies"] == ["all_terms"] - assert response["results"][0]["title"] == "Strojový preklad" + assert response["strategies"] == [ + "all_terms" + ] + + assert ( + response["results"][0]["title"] + == "Strojový preklad" + ) def test_prefix_fallback_handles_slovak_inflection( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - db_file = build_search_database(tmp_path, monkeypatch) + db_file = build_search_database( + tmp_path, + monkeypatch, + ) - response = search_database(db_file, "hlboke ucenie", limit=5) + response = search_database( + db_file, + "hlboke ucenie", + limit=5, + ) - assert response["strategies"] == ["prefix_terms"] - assert response["results"][0]["title"] == "Neurónové siete" + assert response["strategies"] == [ + "prefix_terms" + ] + + assert ( + response["results"][0]["title"] + == "Neurónové siete" + ) def test_any_term_is_used_only_when_stricter_queries_find_nothing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - db_file = build_search_database(tmp_path, monkeypatch) + db_file = build_search_database( + tmp_path, + monkeypatch, + ) - response = search_database(db_file, "nezmysel preklad", limit=5) + response = search_database( + db_file, + "nezmysel preklad", + limit=5, + ) + + assert response["strategies"] == [ + "any_term" + ] - assert response["strategies"] == ["any_term"] assert response["results"] - assert all("preklad" in item["text"].casefold() for item in response["results"]) + + assert any( + item.get("match_strategy") + == "any_term" + and "preklad" + in item["text"].casefold() + for item in response["results"] + ) def test_published_only_excludes_unpublished_chunks( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - db_file = build_search_database(tmp_path, monkeypatch) + db_file = build_search_database( + tmp_path, + monkeypatch, + ) + + all_results = search_database( + db_file, + "hlboke ucenie", + limit=5, + ) - all_results = search_database(db_file, "hlboke ucenie", limit=5) public_results = search_database( db_file, "hlboke ucenie", @@ -222,14 +319,26 @@ def test_published_only_excludes_unpublished_chunks( ) assert all_results["results"] - assert public_results["results"] == [] + + assert any( + item["published"] is False + for item in all_results["results"] + ) + + assert all( + item["published"] is True + for item in public_results["results"] + ) def test_results_are_diversified_by_document( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - db_file = build_search_database(tmp_path, monkeypatch) + db_file = build_search_database( + tmp_path, + monkeypatch, + ) response = search_database( db_file, @@ -238,48 +347,97 @@ def test_results_are_diversified_by_document( max_per_document=1, ) - paths = [item["document_path"] for item in response["results"]] - assert len(paths) == len(set(paths)) + paths = [ + item["document_path"] + for item in response["results"] + ] + + assert len(paths) == len( + set(paths) + ) def test_public_result_format_has_no_internal_id_and_uses_boolean( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - db_file = build_search_database(tmp_path, monkeypatch) + db_file = build_search_database( + tmp_path, + monkeypatch, + ) - result = search_database(db_file, "jan ptak", limit=1)["results"][0] + result = search_database( + db_file, + "jan ptak", + limit=1, + )["results"][0] assert "id" not in result assert result["published"] is True assert result["chunk_id"] == "jan-ptak::0" - assert result["source_url"].endswith("students/jan_ptak") + + assert result["source_url"].endswith( + "students/jan_ptak" + ) def test_empty_query_returns_empty_result( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - db_file = build_search_database(tmp_path, monkeypatch) + db_file = build_search_database( + tmp_path, + monkeypatch, + ) - response = search_database(db_file, " ", limit=5) + response = search_database( + db_file, + " ", + limit=5, + ) assert response == { - "engine": "sqlite_fts5", + "engine": "hybrid_fts5_embeddings", "strategies": [], "results": [], } -def test_missing_database_is_reported(tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError): - search_database(tmp_path / "missing.sqlite", "rag") +def test_missing_database_is_reported( + tmp_path: Path, +) -> None: + with pytest.raises( + FileNotFoundError + ): + search_database( + tmp_path / "missing.sqlite", + "rag", + ) -def test_missing_fts_schema_is_reported(tmp_path: Path) -> None: - db_file = tmp_path / "broken.sqlite" - with sqlite3.connect(db_file) as conn: - conn.execute("CREATE TABLE chunks(id INTEGER PRIMARY KEY)") +def test_missing_fts_schema_is_reported( + tmp_path: Path, +) -> None: + db_file = ( + tmp_path / "broken.sqlite" + ) - with pytest.raises(RuntimeError, match="FTS5 index"): - search_database(db_file, "rag") + with sqlite3.connect( + db_file + ) as conn: + conn.execute( + """ + CREATE TABLE chunks( + id INTEGER PRIMARY KEY + ) + """ + ) + + with pytest.raises( + RuntimeError, + match="FTS5 index", + ): + search_database( + db_file, + "rag", + )