From b943632fd14e86c10ef56ac1ddb9d3bd6e61211d Mon Sep 17 00:00:00 2001 From: jp170na Date: Mon, 26 Oct 2026 15:12:31 +0100 Subject: [PATCH] =?UTF-8?q?Stabiln=C3=A9=20spustenie=20projektu?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + Dockerfile | 16 ++++ README.md | 45 ++++++++++- docker-compose.yml | 23 +++++- runtime_assets/.gitkeep | 0 scripts/start_project.sh | 16 ++++ test/test_runtime_assets.py | 100 ++++++++++++++++++++++++ tools/bootstrap_indexes.py | 34 +++++++++ tools/rag/build_chunks.py | 3 + tools/rag/embedding_utils.py | 7 +- tools/runtime_assets.py | 143 +++++++++++++++++++++++++++++++++++ tools/runtime_health.py | 13 ++++ 12 files changed, 395 insertions(+), 7 deletions(-) create mode 100644 runtime_assets/.gitkeep create mode 100755 scripts/start_project.sh create mode 100644 test/test_runtime_assets.py create mode 100644 tools/bootstrap_indexes.py create mode 100644 tools/runtime_assets.py create mode 100644 tools/runtime_health.py diff --git a/.gitignore b/.gitignore index 39da762..22c65c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .env data/ docs/ +runtime_assets/* +!runtime_assets/.gitkeep .venv/ __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile index 88fbe58..18858bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,22 @@ RUN python -m pip install --no-cache-dir \ COPY app ./app COPY tools ./tools +COPY runtime_assets/ /opt/zp-assets/ + +ARG EMBEDDING_MODEL=intfloat/multilingual-e5-small +ARG EMBEDDING_REVISION="" +ARG CHUNK_TOKEN_ENCODING=cl100k_base +ARG RAG_LLM_TOKEN_ENCODING=cl100k_base + +RUN python -m tools.runtime_assets prepare --output /opt/zp-assets \ + --model "$EMBEDDING_MODEL" --revision "$EMBEDDING_REVISION" \ + --encoding "$CHUNK_TOKEN_ENCODING" --encoding "$RAG_LLM_TOKEN_ENCODING" + +ENV ZP_RUNTIME_ASSETS=/opt/zp-assets +ENV ZP_ASSETS_REQUIRED=1 +ENV HF_HUB_OFFLINE=1 +ENV TRANSFORMERS_OFFLINE=1 +ENV TIKTOKEN_CACHE_DIR=/opt/zp-assets/tiktoken RUN mkdir -p data diff --git a/README.md b/README.md index 979ff0c..ac70d04 100644 --- a/README.md +++ b/README.md @@ -619,10 +619,49 @@ CHUNK_TOKEN_ENCODING=cl100k_base ## Spustenie cez Docker +Potrebný je dostupný Docker Engine, natívny Linux Compose v2 a Buildx. +Vo WSL používaj kontext, ku ktorému sa Linux klient vie pripojiť; Windows +kontext s `npipe` tu nie je náhradou Linux socketu. Overenie: + ```bash -docker compose up -d --build +docker --context default info +docker compose version +docker buildx version ``` +Po vytvorení `.env`, klonu `../zpwiki` a SSH konfigurácie spusti: + +```bash +./scripts/start_project.sh +``` + +Skript štandardne používa kontext `default`, nemení globálny Docker kontext +a spravuje iba projekt `zp-agent`. Iný cieľ nastav cez `ZP_DOCKER_CONTEXT`. +Zostaví image, počká na zdravý Neo4j, overí existujúce indexy a podľa +potreby ich koordinovane vytvorí. Až potom spustí API a počká na readiness. +Pri opakovanom spustení ponechá konzistentné a aktuálne indexy. + +Model aj tokenizér sú súčasťou image v `/opt/zp-assets`. Bežný štart +nevyžaduje internet ani existujúcu používateľskú Hugging Face/tiktoken cache. +Prvý build potrebuje balíky a modelové súbory; build overuje aj načítanie +pripravených assets so zakázanou sieťou. Prázdny HF cache volume neskrýva +model pribalený v image. Git pripojenie je samostatná sieťová operácia; +príznaky offline zakazujú sťahovanie modelu, nie synchronizáciu wiki. + +Voliteľne môžeš pred buildom pripraviť ignorované lokálne podklady a využiť +už stiahnutý model z hostiteľa: + +```bash +.venv/bin/python -m tools.runtime_assets prepare +``` + +`runtime_assets/` obsahuje veľké modelové súbory a do Gitu nepatrí; verzovaný +je iba prázdny adresár pre čistý build. Bez pripravených lokálnych podkladov +si ich Docker build vytvorí sám. Súbory majú manifest so SHA-256 a model je +pripnutý na konkrétny commit. Iný model/revízia vyžaduje nový build s +`EMBEDDING_MODEL`/`EMBEDDING_REVISION` a zodpovedajúce pripravené assets. +API pri nesúlade neprejde potichu na sťahovanie iného modelu. + Kontrola služby: ```bash @@ -647,8 +686,8 @@ Zastavenie: docker compose down ``` -Hugging Face cache je uložená v persistentnom Docker volume, takže sa -embeddingový model pri bežnom reštarte kontajnera nemusí znova sťahovať. +Hugging Face cache volume zostáva persistentné; predvolený embeddingový +model a tokenizér však štartujú priamo z overených súborov v image. Neo4j je spustené ako samostatná služba v rovnakom Docker Compose prostredí. Stav služieb je možné skontrolovať: diff --git a/docker-compose.yml b/docker-compose.yml index f72d75b..204421d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,12 @@ services: zp-agent-api: - build: . + build: + context: . + args: + EMBEDDING_MODEL: ${EMBEDDING_MODEL:-intfloat/multilingual-e5-small} + EMBEDDING_REVISION: ${EMBEDDING_REVISION:-} + CHUNK_TOKEN_ENCODING: ${CHUNK_TOKEN_ENCODING:-cl100k_base} + RAG_LLM_TOKEN_ENCODING: ${RAG_LLM_TOKEN_ENCODING:-cl100k_base} container_name: zp-agent-api ports: - "8000:8000" @@ -13,10 +19,14 @@ services: CHUNK_MAX_TOKENS: "450" CHUNK_OVERLAP_TOKENS: "70" CHUNK_MIN_TOKENS: "80" - CHUNK_TOKEN_ENCODING: cl100k_base + CHUNK_TOKEN_ENCODING: ${CHUNK_TOKEN_ENCODING:-cl100k_base} RAG_LLM_MAX_TOKENS: ${RAG_LLM_MAX_TOKENS:-12000} RAG_LLM_TOKEN_ENCODING: ${RAG_LLM_TOKEN_ENCODING:-cl100k_base} HF_HOME: /cache/huggingface + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + ZP_RUNTIME_ASSETS: /opt/zp-assets + ZP_ASSETS_REQUIRED: "1" GIT_SSH_COMMAND: >- ssh -F /dev/null -i /run/secrets/zpwiki_ssh_key -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes @@ -45,6 +55,15 @@ services: restart: unless-stopped stop_grace_period: 5m + depends_on: + neo4j: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-m", "tools.runtime_health"] + interval: 15s + timeout: 10s + retries: 12 + start_period: 90s neo4j: image: neo4j:2026.09.0 diff --git a/runtime_assets/.gitkeep b/runtime_assets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/start_project.sh b/scripts/start_project.sh new file mode 100755 index 0000000..c8b36e0 --- /dev/null +++ b/scripts/start_project.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +project_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +docker_context="${ZP_DOCKER_CONTEXT:-default}" +compose=(docker --context "$docker_context" compose --project-name zp-agent + --project-directory "$project_root" -f "$project_root/docker-compose.yml") + +docker --context "$docker_context" info --format '{{.ServerVersion}}' >/dev/null +"${compose[@]}" version >/dev/null +"${compose[@]}" config --quiet +"${compose[@]}" build zp-agent-api +"${compose[@]}" up -d --wait neo4j +"${compose[@]}" run --rm -T --no-deps zp-agent-api python -m tools.bootstrap_indexes +"${compose[@]}" up -d --no-build --wait zp-agent-api +"${compose[@]}" ps diff --git a/test/test_runtime_assets.py b/test/test_runtime_assets.py new file mode 100644 index 0000000..ec787f3 --- /dev/null +++ b/test/test_runtime_assets.py @@ -0,0 +1,100 @@ +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from tools import runtime_assets as assets + + +@pytest.fixture +def bundle(tmp_path): + files = {'model/config.json': b'{}', 'model/modules.json': b'[]', + 'model/tokenizer.json': b'{}', 'model/model.safetensors': b'weights', + 'tiktoken/token-file': b'tokenizer'} + for name, value in files.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value) + manifest = {'schema_version': 1, 'embedding': {'model': assets.DEFAULT_MODEL, 'revision': assets.DEFAULT_REVISION}, + 'encodings': ['cl100k_base'], 'files': {name: hashlib.sha256(value).hexdigest() for name, value in files.items()}} + (tmp_path / 'manifest.json').write_text(json.dumps(manifest)) + return tmp_path, manifest + + +def test_assets_verify_integrity_and_reject_corruption(bundle): + root, manifest = bundle + assert assets.verify(root) == manifest + (root / 'model/model.safetensors').write_bytes(b'corrupted') + with pytest.raises(RuntimeError, match='poškodený'): + assets.verify(root) + + +def test_prepared_assets_override_empty_user_cache_and_pin_revision(bundle, monkeypatch, tmp_path): + root, _ = bundle + monkeypatch.setenv('ZP_RUNTIME_ASSETS', str(root)) + monkeypatch.setenv('ZP_ASSETS_REQUIRED', '1') + monkeypatch.setenv('TIKTOKEN_CACHE_DIR', str(tmp_path / 'empty')) + assets.configure_tokenizer() + assert os.environ['TIKTOKEN_CACHE_DIR'] == str(root / 'tiktoken') + assert assets.local_model(assets.DEFAULT_MODEL, None) == str(root / 'model') + with pytest.raises(RuntimeError, match='revízia'): + assets.local_model(assets.DEFAULT_MODEL, 'a' * 40) + with pytest.raises(RuntimeError, match='nie je v image'): + assets.local_model('other/model', None) + + +def test_required_assets_never_silently_fall_back_to_remote_download(tmp_path, monkeypatch): + monkeypatch.setenv('ZP_RUNTIME_ASSETS', str(tmp_path)) + monkeypatch.setenv('ZP_ASSETS_REQUIRED', '1') + with pytest.raises(RuntimeError, match='Chýbajú runtime assets'): + assets.configure_tokenizer() + + +def test_existing_assets_are_verified_without_download(bundle, monkeypatch): + root, manifest = bundle + probes = [] + monkeypatch.setattr(assets, 'offline_probe', lambda path: probes.append(path)) + assert assets.prepare(root, assets.DEFAULT_MODEL, None, ['cl100k_base']) == manifest + assert probes == [root] + + +def test_manifest_cannot_escape_asset_directory(bundle): + root, manifest = bundle + manifest['files']['../outside'] = 'a' * 64 + (root / 'manifest.json').write_text(json.dumps(manifest)) + with pytest.raises(RuntimeError, match='mimo runtime assets'): + assets.verify(root) + + +def test_fresh_api_lifespan_loads_model_and_tokenizer_without_network_or_user_cache(tmp_path): + environment = {**os.environ, 'ZP_RUNTIME_ASSETS': str(assets.asset_root().resolve()), + 'ZP_ASSETS_REQUIRED': '1', 'HF_HOME': str(tmp_path / 'empty-hf'), + 'TIKTOKEN_CACHE_DIR': str(tmp_path / 'empty-tokenizer'), + 'HF_HUB_OFFLINE': '1', 'TRANSFORMERS_OFFLINE': '1', 'TEST_JOB_DIR': str(tmp_path / 'jobs')} + code = ''' +import socket, asyncio, os +from pathlib import Path +def denied(*args, **kwargs): raise AssertionError('Unexpected network access') +socket.socket.connect = denied +socket.socket.connect_ex = denied +socket.getaddrinfo = denied +import app.main as main +from tools.rag.build_chunks import TOKEN_COUNTER +from tools.rag.embedding_utils import embed_query +main.DATA_DIR = Path(os.environ['TEST_JOB_DIR']) +async def run(): + async with main.lifespan(main.app): + assert main.app.state.embedding_ready + assert TOKEN_COUNTER.encoding is not None + assert embed_query('štart bez siete').shape[0] == 384 +asyncio.run(run()) +print('offline API lifespan OK') +''' + result = subprocess.run([sys.executable, '-c', code], env=environment, + capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr + assert 'offline API lifespan OK' in result.stdout diff --git a/tools/bootstrap_indexes.py b/tools/bootstrap_indexes.py new file mode 100644 index 0000000..f835938 --- /dev/null +++ b/tools/bootstrap_indexes.py @@ -0,0 +1,34 @@ +"""Keep a valid generation, or create both indexes before starting the API.""" +import hashlib +import json + +from app.security import validate_security_configuration +from tools.rag.common import DATA_DIR, DB_FILE, ZPWIKI_ROOT +from tools.rag.embedding_utils import embed_query +from tools.rag.rebuild_index import rebuild_index +from tools.runtime_assets import asset_root, verify +from tools.sync_indexes import git_revision, source_files +from tools.tool_health import tools_readiness + + +def main(): + validate_security_configuration() + verify(asset_root()) + embed_query('overenie pripravenosti') + tools, version = tools_readiness(DB_FILE, security_configured=True, embedding_ready=True) + source_hash = hashlib.sha256(json.dumps(source_files(ZPWIKI_ROOT), sort_keys=True).encode()).hexdigest() + current = (all(value['ready'] for value in tools.values()) and version + and version.get('git_revision') == git_revision(ZPWIKI_ROOT) + and version.get('source_sha256') == source_hash) + if not current: + result = rebuild_index(pull_git=False) + tools, version = tools_readiness(DB_FILE, security_configured=True, embedding_ready=True) + if not all(value['ready'] for value in tools.values()): + raise RuntimeError('Nové indexy nie sú pripravené; API sa nespustí.') + print(json.dumps({'bootstrap': 'rebuilt', 'counts': result['counts'], 'index_version': version})) + else: + print(json.dumps({'bootstrap': 'existing', 'index_version': version})) + + +if __name__ == '__main__': + main() diff --git a/tools/rag/build_chunks.py b/tools/rag/build_chunks.py index 7dff528..609a7a4 100644 --- a/tools/rag/build_chunks.py +++ b/tools/rag/build_chunks.py @@ -24,6 +24,9 @@ from tools.rag.common import ( write_json, ) from tools.text_budget import split_to_fit +from tools.runtime_assets import configure_tokenizer + +configure_tokenizer() MAX_TOKENS = int(os.getenv("CHUNK_MAX_TOKENS", "450")) diff --git a/tools/rag/embedding_utils.py b/tools/rag/embedding_utils.py index b84a7ee..0fdbe4f 100644 --- a/tools/rag/embedding_utils.py +++ b/tools/rag/embedding_utils.py @@ -8,6 +8,7 @@ from typing import Any import numpy as np from tools.text_budget import split_to_fit +from tools.runtime_assets import local_model DEFAULT_EMBEDDING_MODEL = ( @@ -59,9 +60,11 @@ def _load_embedding_model(model_name: str, revision: str | None): SentenceTransformer, ) + bundled = local_model(model_name, revision) return SentenceTransformer( - model_name, - revision=revision, + bundled or model_name, + revision=None if bundled else revision, + local_files_only=bool(bundled) or os.getenv('HF_HUB_OFFLINE') == '1', ) diff --git a/tools/runtime_assets.py b/tools/runtime_assets.py new file mode 100644 index 0000000..04050cc --- /dev/null +++ b/tools/runtime_assets.py @@ -0,0 +1,143 @@ +"""Prepare immutable model/tokenizer assets during build, never during startup.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +DEFAULT_MODEL = 'intfloat/multilingual-e5-small' +DEFAULT_REVISION = '614241f622f53c4eeff9890bdc4f31cfecc418b3' +DEFAULT_ROOT = Path(__file__).resolve().parents[1] / 'runtime_assets' + + +def asset_root() -> Path: + return Path(os.getenv('ZP_RUNTIME_ASSETS', str(DEFAULT_ROOT))) + + +def manifest(root: Path | None = None) -> dict | None: + path = (root or asset_root()) / 'manifest.json' + if not path.exists(): + if os.getenv('ZP_ASSETS_REQUIRED') == '1': + raise RuntimeError('Chýbajú runtime assets; zostavte image alebo spustite python -m tools.runtime_assets prepare.') + return None + return json.loads(path.read_text()) + + +def configure_tokenizer() -> None: + data = manifest() + if data is not None: + # Bundled immutable files take precedence over an empty user cache. + os.environ['TIKTOKEN_CACHE_DIR'] = str(asset_root() / 'tiktoken') + + +def local_model(model_name: str, revision: str | None) -> str | None: + data = manifest() + if data and data['embedding']['model'] == model_name: + if revision and revision != data['embedding']['revision']: + raise RuntimeError('Požadovaná revízia modelu nie je v image; vytvorte nový image s touto revíziou.') + return str(asset_root() / 'model') + if data and os.getenv('ZP_ASSETS_REQUIRED') == '1': + raise RuntimeError('Požadovaný embeddingový model nie je v image; vytvorte nový image.') + return None + + +def verify(root: Path) -> dict: + data = json.loads((root / 'manifest.json').read_text()) + for relative, digest in data['files'].items(): + path = root / relative + if not path.resolve().is_relative_to(root.resolve()): + raise RuntimeError('Manifest obsahuje cestu mimo runtime assets') + if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise RuntimeError(f'Chýbajúci alebo poškodený runtime asset: {relative}') + if not any(name.startswith('tiktoken/') for name in data['files']): + raise RuntimeError('Manifest neobsahuje tokenizér') + required = {'model/config.json', 'model/modules.json', 'model/tokenizer.json'} + if not required.issubset(data['files']) or not any(name.endswith('.safetensors') for name in data['files']): + raise RuntimeError('Manifest neobsahuje kompletný embeddingový model') + return data + + +def offline_probe(root: Path) -> None: + """Fresh interpreter, with network calls forbidden even on an online builder.""" + environment = {**os.environ, 'ZP_RUNTIME_ASSETS': str(root.resolve()), 'ZP_ASSETS_REQUIRED': '1', + 'HF_HUB_OFFLINE': '1', 'TRANSFORMERS_OFFLINE': '1', 'HF_HOME': str(root / 'unused-user-cache'), + 'TIKTOKEN_CACHE_DIR': str(root / 'unused-token-cache')} + code = ''' +import socket +def denied(*args, **kwargs): raise RuntimeError('Network access during offline asset probe') +socket.socket.connect = denied +socket.socket.connect_ex = denied +socket.getaddrinfo = denied +from tools.runtime_assets import configure_tokenizer, manifest, local_model +configure_tokenizer() +import tiktoken +for encoding in manifest()['encodings']: + assert tiktoken.get_encoding(encoding).encode('Overenie štartu bez siete.') +from sentence_transformers import SentenceTransformer +data = manifest()['embedding'] +model = SentenceTransformer(local_model(data['model'], data['revision']), local_files_only=True) +assert model.encode(['query: overenie bez siete'], show_progress_bar=False).shape[0] == 1 +print('Offline model/tokenizer probe: OK') +''' + subprocess.run([sys.executable, '-c', code], env=environment, check=True) + + +def prepare(root: Path, model: str, revision: str | None, encodings: list[str]) -> dict: + revision = revision or (DEFAULT_REVISION if model == DEFAULT_MODEL else '') + if not re.fullmatch('[0-9a-f]{40}', revision): + raise ValueError('Príprava modelu vyžaduje presnú 40-znakovú Git revíziu') + if (root / 'manifest.json').is_file(): + data = verify(root) + if data['embedding'] == {'model': model, 'revision': revision} and data['encodings'] == encodings: + offline_probe(root) + return data + raise RuntimeError('Assets patria inej konfigurácii; použite nový/prázdny pracovný adresár.') + from huggingface_hub import snapshot_download + from huggingface_hub.errors import LocalEntryNotFoundError + try: + source = snapshot_download(model, revision=revision, local_files_only=True) + except (FileNotFoundError, LocalEntryNotFoundError): + source = snapshot_download(model, revision=revision, + allow_patterns=['*.json', '*.safetensors', '*.model', '*.txt'], + ignore_patterns=['onnx/*', 'openvino/*']) + root.mkdir(parents=True, exist_ok=True) + shutil.copytree(source, root / 'model', dirs_exist_ok=True, symlinks=False, + ignore=shutil.ignore_patterns('onnx', 'openvino', '*.bin', '*.md', '.cache', '.gitattributes')) + environment = {**os.environ, 'TIKTOKEN_CACHE_DIR': str(root.resolve() / 'tiktoken')} + subprocess.run([sys.executable, '-c', + 'import sys,tiktoken; [tiktoken.get_encoding(name) for name in sys.argv[1:]]', *encodings], + env=environment, check=True) + files = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for folder in ['model', 'tiktoken'] for p in sorted((root / folder).rglob('*')) if p.is_file()} + data = {'schema_version': 1, 'embedding': {'model': model, 'revision': revision}, + 'encodings': encodings, 'files': files} + (root / 'manifest.json').write_text(json.dumps(data, sort_keys=True, indent=2) + '\n') + verify(root) + offline_probe(root) + return data + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=['prepare', 'verify']) + parser.add_argument('--output', type=Path, default=asset_root()) + parser.add_argument('--model', default=DEFAULT_MODEL) + parser.add_argument('--revision') + parser.add_argument('--encoding', action='append') + args = parser.parse_args() + if args.command == 'prepare': + data = prepare(args.output, args.model, args.revision, list(dict.fromkeys(args.encoding or ['cl100k_base']))) + else: + data = verify(args.output) + offline_probe(args.output) + print(json.dumps({'model': data['embedding'], 'encodings': data['encodings'], 'files': len(data['files'])})) + + +if __name__ == '__main__': + main() diff --git a/tools/runtime_health.py b/tools/runtime_health.py new file mode 100644 index 0000000..e0f92ab --- /dev/null +++ b/tools/runtime_health.py @@ -0,0 +1,13 @@ +"""Container readiness: HTTP 200 alone does not mean that indexes are ready.""" +import json +import urllib.request + + +def main(): + with urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=8) as response: + health = json.load(response) + raise SystemExit(0 if health.get('ready') is True else 1) + + +if __name__ == '__main__': + main()