Upratanie modulov a archivácia historických evaluácií

This commit is contained in:
Ján Pták 2026-11-07 18:16:19 +01:00
parent 3d0dfa154b
commit 507c247ea3
492 changed files with 29804 additions and 1400531 deletions

View File

@ -13,3 +13,10 @@ docs/
integrations/ integrations/
tools/git_runner/ tools/git_runner/
**/*.md **/*.md
experiments/
test/
tools/**/tests/
tools/evaluation/
tools/rag/evaluation/
tools/graphrag/evaluation/
tools/claude/

View File

@ -62,5 +62,27 @@ v Docker sieti. Dočasná verejná URL vyžaduje zodpovedajúce OAuth nastavenie
RAG, GraphRAG, kontrolovaná cesta odpovedí a pripojenie cez Claude sú funkčné. RAG, GraphRAG, kontrolovaná cesta odpovedí a pripojenie cez Claude sú funkčné.
Systém obsahuje embeddingovú cache, kontroly dostupnosti a persistentnú Systém obsahuje embeddingovú cache, kontroly dostupnosti a persistentnú
synchronizačnú frontu. Historické evaluácie sú uložené pri jednotlivých vetvách. synchronizačnú frontu. Historické evaluácie sú oddelené od prevádzkového kódu.
Samostatný hlavný agent, CRZP a spracovanie plných textov prác sú ďalšie etapy. Samostatný hlavný agent, CRZP a spracovanie plných textov prác sú ďalšie etapy.
## Organizácia projektu
- `app/` – HTTP API a zabezpečenie.
- `tools/rag/`, `tools/graphrag/`, `tools/knowledge/` – vyhľadávanie a indexovanie.
- `tools/answers/` – príprava podkladov, kontroly a zostavenie odpovede.
- `tools/retrieval/`, `tools/sync/`, `tools/runtime/` – spoločné pravidlá a prevádzka.
- `tools/evaluation/` a `tools/*/evaluation/` – offline evaluátory a diagnostika.
- `experiments/rag/`, `experiments/graphrag/` – vybrané historické vstupy a výsledky.
- `experiments/archive/` – komprimované vývojové pokusy; mapovanie a SHA-256 sú v `catalog.json`.
- `packages/claude/`, `deployment/`, `scripts/` – klienti, nasadenie a spúšťanie.
Prevádzkový image neobsahuje testy ani historické experimenty. Spúšťa sa cez
`bash scripts/start.sh`; voliteľné služby ovláda `bash scripts/compose.sh`.
Regresie spúšťa `.venv/bin/python -m pytest -q test tools/rag/tests tools/graphrag/tests`.
Integritu experimentov overí `python -m tools.evaluation.archive verify`.
Archív možno obnoviť bez prepisovania súborov pomocou
`python -m tools.evaluation.archive restore --destination /tmp/zp-experiments`.
Pôvodný kód experimentov určuje `source_commit` v katalógu a historické Git tagy.
Staré manifesty ani výsledky sa neprepisujú na odtlačky aktuálnej implementácie;
ich úspešnosť nie je novým meraním po reorganizácii.

View File

@ -19,7 +19,6 @@ from fastapi.exception_handlers import http_exception_handler, request_validatio
from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.responses import JSONResponse from starlette.responses import JSONResponse
PROJECT_ROOT = Path(__file__).resolve().parents[1] PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path: if str(PROJECT_ROOT) not in sys.path:
@ -39,13 +38,10 @@ from tools.rag.common import DATA_DIR
from tools.sync.sync_jobs import SyncJobs from tools.sync.sync_jobs import SyncJobs
from tools.rag.rebuild_index import rebuild_index from tools.rag.rebuild_index import rebuild_index
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
OPENWEBUI_ORIGIN = ( OPENWEBUI_ORIGIN = "https://ui.tukekemt.xyz"
"https://ui.tukekemt.xyz"
)
@asynccontextmanager @asynccontextmanager
@ -64,9 +60,7 @@ async def lifespan(
except Exception: except Exception:
logger.exception( logger.exception(
"Predhriatie embedding modelu " "Predhriatie embedding modelu " "zlyhalo. API pokračuje bez " "úspešného warm-upu."
"zlyhalo. API pokračuje bez "
"úspešného warm-upu."
) )
def run_sync(**kwargs): def run_sync(**kwargs):
@ -99,23 +93,36 @@ app = FastAPI(
@app.exception_handler(StarletteHTTPException) @app.exception_handler(StarletteHTTPException)
async def retrieval_http_error(request: Request, error: StarletteHTTPException): async def retrieval_http_error(request: Request, error: StarletteHTTPException):
if request.url.path not in {'/rag', '/search', '/graphrag'}: if request.url.path not in {"/rag", "/search", "/graphrag"}:
return await http_exception_handler(request, error) return await http_exception_handler(request, error)
reason = {400: 'invalid_query', 401: 'unauthorized', 403: 'forbidden', reason = {
404: 'not_found', 405: 'method_not_allowed', 422: 'invalid_request', 400: "invalid_query",
503: 'retrieval_unavailable'}.get(error.status_code, 'internal_error') 401: "unauthorized",
return JSONResponse(status_code=error.status_code, headers=error.headers, 403: "forbidden",
content={**error_result(reason=reason), 'detail': error.detail}) 404: "not_found",
405: "method_not_allowed",
422: "invalid_request",
503: "retrieval_unavailable",
}.get(error.status_code, "internal_error")
return JSONResponse(
status_code=error.status_code,
headers=error.headers,
content={**error_result(reason=reason), "detail": error.detail},
)
@app.exception_handler(RequestValidationError) @app.exception_handler(RequestValidationError)
async def retrieval_validation_error(request: Request, error: RequestValidationError): async def retrieval_validation_error(request: Request, error: RequestValidationError):
if request.url.path not in {'/rag', '/search', '/graphrag'}: if request.url.path not in {"/rag", "/search", "/graphrag"}:
return await request_validation_exception_handler(request, error) return await request_validation_exception_handler(request, error)
# Retain useful validation locations/types without echoing submitted secrets. # Retain useful validation locations/types without echoing submitted secrets.
details = [{'loc': row['loc'], 'type': row['type'], 'msg': row['msg']} for row in error.errors()] details = [
return JSONResponse(status_code=422, content=jsonable_encoder({ {"loc": row["loc"], "type": row["type"], "msg": row["msg"]} for row in error.errors()
**error_result(reason='invalid_request'), 'detail': details})) ]
return JSONResponse(
status_code=422,
content=jsonable_encoder({**error_result(reason="invalid_request"), "detail": details}),
)
app.add_middleware( app.add_middleware(
@ -149,9 +156,7 @@ app.add_middleware(
) )
@app.middleware( @app.middleware("http")
"http"
)
async def add_security_headers( async def add_security_headers(
request: Request, request: Request,
call_next, call_next,
@ -159,33 +164,27 @@ async def add_security_headers(
try: try:
response = await call_next(request) response = await call_next(request)
except Exception: except Exception:
if request.url.path not in {'/rag', '/search', '/graphrag'}: if request.url.path not in {"/rag", "/search", "/graphrag"}:
raise raise
# Serialization failures must obey the same contract as backend errors. # Serialization failures must obey the same contract as backend errors.
logger.exception('Neočakávaná chyba výstupu retrieval API.') logger.exception("Neočakávaná chyba výstupu retrieval API.")
response = JSONResponse(status_code=500, content={ response = JSONResponse(
**error_result(reason='internal_error'), status_code=500,
'detail': 'Nastala interná chyba servera.'}) content={
**error_result(reason="internal_error"),
"detail": "Nastala interná chyba servera.",
},
)
response.headers[ response.headers["Cache-Control"] = "no-store"
"Cache-Control"
] = "no-store"
response.headers[ response.headers["X-Content-Type-Options"] = "nosniff"
"X-Content-Type-Options"
] = "nosniff"
response.headers[ response.headers["Referrer-Policy"] = "no-referrer"
"Referrer-Policy"
] = "no-referrer"
response.headers[ response.headers["X-Frame-Options"] = "DENY"
"X-Frame-Options"
] = "DENY"
return response return response
app.include_router( app.include_router(router)
router
)

View File

@ -47,38 +47,33 @@ from tools.rag.search_utils import search_database
from tools.sync.wiki_git import expected_branch from tools.sync.wiki_git import expected_branch
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY
from tools.runtime.tool_health import tools_readiness from tools.runtime.tool_health import tools_readiness
from tools.retrieval.retrieval_contract import RetrievalResponse, RetrievalSource, RetrievalErrorResponse, error_reason from tools.retrieval.retrieval_contract import (
RetrievalResponse,
RetrievalSource,
RetrievalErrorResponse,
error_reason,
)
from tools.answers.answer_delivery import deliver_answer from tools.answers.answer_delivery import deliver_answer
from tools.answers.answer_errors import public_failure from tools.answers.answer_errors import public_failure
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
router = APIRouter() router = APIRouter()
MAX_WEBHOOK_BODY_BYTES = ( MAX_WEBHOOK_BODY_BYTES = 2 * 1024 * 1024
2 * 1024 * 1024
)
RETRIEVAL_UNAVAILABLE_DETAIL = ( RETRIEVAL_UNAVAILABLE_DETAIL = (
"Vyhľadávací index alebo embeddingový " "Vyhľadávací index alebo embeddingový " "model momentálne nie je dostupný."
"model momentálne nie je dostupný."
) )
INVALID_QUERY_DETAIL = ( INVALID_QUERY_DETAIL = "Neplatný vyhľadávací dotaz."
"Neplatný vyhľadávací dotaz."
)
INTERNAL_ERROR_DETAIL = ( INTERNAL_ERROR_DETAIL = "Nastala interná chyba servera."
"Nastala interná chyba servera."
)
WEBHOOK_CONFIGURATION_DETAIL = ( WEBHOOK_CONFIGURATION_DETAIL = "Webhook nie je správne nakonfigurovaný."
"Webhook nie je správne nakonfigurovaný."
)
class StrictRequestModel(BaseModel): class StrictRequestModel(BaseModel):
@ -88,9 +83,7 @@ class StrictRequestModel(BaseModel):
) )
class SearchRequest( class SearchRequest(StrictRequestModel):
StrictRequestModel
):
query: str = Field( query: str = Field(
..., ...,
min_length=1, min_length=1,
@ -112,28 +105,20 @@ class SearchRequest(
) )
class RagRequest( class RagRequest(StrictRequestModel):
StrictRequestModel
):
include_diagnostics: bool = False include_diagnostics: bool = False
query: str = Field( query: str = Field(
..., ...,
min_length=1, min_length=1,
max_length=500, max_length=500,
description=( description=("Otázka alebo vyhľadávací dotaz " "používateľa nad ZP Wiki."),
"Otázka alebo vyhľadávací dotaz "
"používateľa nad ZP Wiki."
),
) )
limit: int = Field( limit: int = Field(
default=5, default=5,
ge=1, ge=1,
le=20, le=20,
description=( description=("Maximálny počet relevantných " "zdrojov pre RAG kontext."),
"Maximálny počet relevantných "
"zdrojov pre RAG kontext."
),
) )
published_only: bool = Field( published_only: bool = Field(
@ -156,14 +141,11 @@ class RagRequest(
) )
class SyncRequest( class SyncRequest(StrictRequestModel):
StrictRequestModel
):
pull_git: bool = Field( pull_git: bool = Field(
default=False, default=False,
description=( description=(
"Pred reindexovaním stiahne očakávanú vetvu " "Pred reindexovaním stiahne očakávanú vetvu " "a aktualizuje ju iba fast-forward."
"a aktualizuje ju iba fast-forward."
), ),
) )
@ -177,34 +159,43 @@ class GraphRequest(StrictRequestModel):
class AnswerRequest(StrictRequestModel): class AnswerRequest(StrictRequestModel):
query: str = Field(min_length=1, max_length=500) query: str = Field(min_length=1, max_length=500)
engine: Literal['rag', 'graphrag'] = 'rag' engine: Literal["rag", "graphrag"] = "rag"
limit: int = Field(default=5, ge=1, le=20) limit: int = Field(default=5, ge=1, le=20)
include_diagnostics: bool = False include_diagnostics: bool = False
@router.post('/answer', dependencies=[Depends(require_search_api_key)], include_in_schema=False) @router.post("/answer", dependencies=[Depends(require_search_api_key)], include_in_schema=False)
def verified_answer(request: AnswerRequest) -> dict: def verified_answer(request: AnswerRequest) -> dict:
try: try:
result = deliver_answer(request.query, engine=request.engine, limit=request.limit, result = deliver_answer(
include_diagnostics=request.include_diagnostics) request.query,
engine=request.engine,
limit=request.limit,
include_diagnostics=request.include_diagnostics,
)
except Exception: except Exception:
logger.exception('Príprava overenej odpovede zlyhala.') logger.exception("Príprava overenej odpovede zlyhala.")
return {'status': 'error', 'reason': 'answer_unavailable', 'accepted_for_delivery': False, return {
'answer': 'Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.'} "status": "error",
if (result.get('accepted_for_delivery') is not True or "reason": "answer_unavailable",
result.get('status') not in {'ok', 'no_answer', 'ambiguous'}): "accepted_for_delivery": False,
"answer": "Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.",
}
if result.get("accepted_for_delivery") is not True or result.get("status") not in {
"ok",
"no_answer",
"ambiguous",
}:
result = public_failure(result) result = public_failure(result)
result={**result,'query':request.query.strip(),'engine':request.engine} result = {**result, "query": request.query.strip(), "engine": request.engine}
if not request.include_diagnostics: if not request.include_diagnostics:
result.pop('attempts', None) result.pop("attempts", None)
result.pop('retrieval_context', None) result.pop("retrieval_context", None)
return result return result
class HealthResponse(BaseModel): class HealthResponse(BaseModel):
status: Literal[ status: Literal["ok"]
"ok"
]
ready: bool ready: bool
@ -235,10 +226,13 @@ class RagSource(RetrievalSource):
heading_paths: list[Any] | None = None heading_paths: list[Any] | None = None
text: str | None = None text: str | None = None
retrieval: dict[ retrieval: (
dict[
str, str,
Any, Any,
] | None = None ]
| None
) = None
class RagResponse(RetrievalResponse): class RagResponse(RetrievalResponse):
@ -264,9 +258,7 @@ class RagResponse(RetrievalResponse):
context: str context: str
sources: list[ sources: list[RagSource]
RagSource
]
index_version: dict[str, Any] | None = None index_version: dict[str, Any] | None = None
context_budget: dict[str, Any] | None = None context_budget: dict[str, Any] | None = None
diagnostics: dict[str, Any] | None = None diagnostics: dict[str, Any] | None = None
@ -285,14 +277,12 @@ class SearchResponse(RetrievalResponse):
ge=0, ge=0,
) )
results: list[ results: list[dict[str, Any]]
dict[str, Any]
]
index_version: dict[str, Any] | None = None index_version: dict[str, Any] | None = None
class SyncAcceptedResponse(BaseModel): class SyncAcceptedResponse(BaseModel):
status: Literal['accepted'] status: Literal["accepted"]
job_id: str job_id: str
status_url: str status_url: str
pull_git: bool pull_git: bool
@ -300,12 +290,12 @@ class SyncAcceptedResponse(BaseModel):
duplicate: bool duplicate: bool
event: str | None = None event: str | None = None
repository: str | None = None repository: str | None = None
verified_by: Literal['hmac_sha256'] | None = None verified_by: Literal["hmac_sha256"] | None = None
class WebhookIgnoredResponse(BaseModel): class WebhookIgnoredResponse(BaseModel):
status: Literal['ignored'] status: Literal["ignored"]
reason: Literal['unsupported_event', 'unexpected_branch', 'deleted_branch'] reason: Literal["unsupported_event", "unexpected_branch", "deleted_branch"]
event: str event: str
@ -313,12 +303,17 @@ def enqueue_sync(request: Request, **kwargs) -> dict:
try: try:
accepted = request.app.state.sync_jobs.enqueue(**kwargs) accepted = request.app.state.sync_jobs.enqueue(**kwargs)
except (sqlite3.Error, OSError): except (sqlite3.Error, OSError):
logger.exception('Prijatie synchronizácie do fronty zlyhalo.') logger.exception("Prijatie synchronizácie do fronty zlyhalo.")
raise HTTPException(status_code=503, detail='Synchronizačná fronta nie je dostupná') raise HTTPException(status_code=503, detail="Synchronizačná fronta nie je dostupná")
job = accepted['job'] job = accepted["job"]
return {'status': 'accepted', 'job_id': job['id'], return {
'status_url': f"/sync/jobs/{job['id']}", 'pull_git': job['pull_git'], "status": "accepted",
'coalesced': accepted['coalesced'], 'duplicate': accepted['duplicate']} "job_id": job["id"],
"status_url": f"/sync/jobs/{job['id']}",
"pull_git": job["pull_git"],
"coalesced": accepted["coalesced"],
"duplicate": accepted["duplicate"],
}
def raise_retrieval_error( def raise_retrieval_error(
@ -334,127 +329,77 @@ def raise_retrieval_error(
) )
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_400_BAD_REQUEST),
status.HTTP_400_BAD_REQUEST detail=(INVALID_QUERY_DETAIL),
),
detail=(
INVALID_QUERY_DETAIL
),
) from error ) from error
if error_reason(error) == 'retrieval_unavailable': if error_reason(error) == "retrieval_unavailable":
logger.exception( logger.exception("Retrieval nie je dostupný.")
"Retrieval nie je dostupný."
)
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
status.HTTP_503_SERVICE_UNAVAILABLE detail=(RETRIEVAL_UNAVAILABLE_DETAIL),
),
detail=(
RETRIEVAL_UNAVAILABLE_DETAIL
),
) from error ) from error
logger.exception( logger.exception("Neočakávaná chyba retrieval API.")
"Neočakávaná chyba retrieval API."
)
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_500_INTERNAL_SERVER_ERROR),
status.HTTP_500_INTERNAL_SERVER_ERROR detail=(INTERNAL_ERROR_DETAIL),
),
detail=(
INTERNAL_ERROR_DETAIL
),
) from error ) from error
def webhook_secret() -> str: def webhook_secret() -> str:
try: try:
return validate_secret( return validate_secret("WEBHOOK_SECRET")
"WEBHOOK_SECRET"
)
except RuntimeError as error: except RuntimeError as error:
logger.exception( logger.exception("WEBHOOK_SECRET nie je " "správne nakonfigurovaný.")
"WEBHOOK_SECRET nie je "
"správne nakonfigurovaný."
)
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
status.HTTP_503_SERVICE_UNAVAILABLE detail=(WEBHOOK_CONFIGURATION_DETAIL),
),
detail=(
WEBHOOK_CONFIGURATION_DETAIL
),
) from error ) from error
def configured_repository() -> str: def configured_repository() -> str:
try: try:
return ( return expected_gitea_repository()
expected_gitea_repository()
)
except RuntimeError as error: except RuntimeError as error:
logger.exception( logger.exception("EXPECTED_GITEA_REPOSITORY " "nie je správne nakonfigurovaný.")
"EXPECTED_GITEA_REPOSITORY "
"nie je správne nakonfigurovaný."
)
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
status.HTTP_503_SERVICE_UNAVAILABLE detail=(WEBHOOK_CONFIGURATION_DETAIL),
),
detail=(
WEBHOOK_CONFIGURATION_DETAIL
),
) from error ) from error
async def read_webhook_body( async def read_webhook_body(
request: Request, request: Request,
) -> bytes: ) -> bytes:
content_length = request.headers.get( content_length = request.headers.get("content-length")
"content-length"
)
if content_length: if content_length:
try: try:
declared_length = int( declared_length = int(content_length)
content_length
)
except ValueError as error: except ValueError as error:
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail=( detail=("Neplatná Content-Length " "hlavička"),
"Neplatná Content-Length "
"hlavička"
),
) from error ) from error
if declared_length < 0: if declared_length < 0:
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail=( detail=("Neplatná Content-Length " "hlavička"),
"Neplatná Content-Length "
"hlavička"
),
) )
if ( if declared_length > MAX_WEBHOOK_BODY_BYTES:
declared_length
> MAX_WEBHOOK_BODY_BYTES
):
raise HTTPException( raise HTTPException(
status_code=413, status_code=413,
detail=( detail=("Webhook payload je " "príliš veľký"),
"Webhook payload je "
"príliš veľký"
),
) )
chunks: list[bytes] = [] chunks: list[bytes] = []
@ -462,29 +407,17 @@ async def read_webhook_body(
total_size = 0 total_size = 0
async for chunk in request.stream(): async for chunk in request.stream():
total_size += len( total_size += len(chunk)
chunk
)
if ( if total_size > MAX_WEBHOOK_BODY_BYTES:
total_size
> MAX_WEBHOOK_BODY_BYTES
):
raise HTTPException( raise HTTPException(
status_code=413, status_code=413,
detail=( detail=("Webhook payload je " "príliš veľký"),
"Webhook payload je "
"príliš veľký"
),
) )
chunks.append( chunks.append(chunk)
chunk
)
return b"".join( return b"".join(chunks)
chunks
)
@router.get( @router.get(
@ -493,13 +426,9 @@ async def read_webhook_body(
response_model=HealthResponse, response_model=HealthResponse,
) )
def health(request: Request) -> dict[str, Any]: def health(request: Request) -> dict[str, Any]:
database_exists = ( database_exists = DB_FILE.exists()
DB_FILE.exists()
)
zpwiki_exists = ( zpwiki_exists = ZPWIKI_ROOT.exists()
ZPWIKI_ROOT.exists()
)
try: try:
validate_security_configuration() validate_security_configuration()
@ -509,39 +438,30 @@ def health(request: Request) -> dict[str, Any]:
except RuntimeError: except RuntimeError:
security_configured = False security_configured = False
tools, version = tools_readiness(DB_FILE, security_configured=security_configured, tools, version = tools_readiness(
embedding_ready=getattr(request.app.state, 'embedding_ready', False)) DB_FILE,
ready = tools['rag']['ready'] and tools['graphrag']['ready'] security_configured=security_configured,
embedding_ready=getattr(request.app.state, "embedding_ready", False),
)
ready = tools["rag"]["ready"] and tools["graphrag"]["ready"]
return { return {
"status": "ok", "status": "ok",
"ready": ready, "ready": ready,
"index_version": version, "index_version": version,
"tools": tools, "tools": tools,
"database_exists": ( "database_exists": (database_exists),
database_exists "search_engine": ("hybrid_fts5_embeddings"),
),
"search_engine": (
"hybrid_fts5_embeddings"
),
"rag_enabled": True, "rag_enabled": True,
"zpwiki_exists": ( "zpwiki_exists": (zpwiki_exists),
zpwiki_exists "security_configured": (security_configured),
),
"security_configured": (
security_configured
),
} }
@router.post( @router.post(
"/rag", "/rag",
operation_id=( operation_id=("retrieve_zpwiki_context"),
"retrieve_zpwiki_context" summary=("Vyhľadaj informácie v ZP Wiki"),
),
summary=(
"Vyhľadaj informácie v ZP Wiki"
),
description=( description=(
"Použi tento nástroj pri otázkach " "Použi tento nástroj pri otázkach "
"o ZP Wiki, študentoch, autoroch, " "o ZP Wiki, študentoch, autoroch, "
@ -553,48 +473,30 @@ def health(request: Request) -> dict[str, Any]:
), ),
response_model=RagResponse, response_model=RagResponse,
response_model_exclude_unset=True, response_model_exclude_unset=True,
status_code=( status_code=(status.HTTP_200_OK),
status.HTTP_200_OK
),
responses={ responses={
400: { 400: {
"model": RetrievalErrorResponse, "model": RetrievalErrorResponse,
"description": ( "description": ("Neplatný dotaz."),
"Neplatný dotaz."
),
}, },
401: { 401: {
"model": RetrievalErrorResponse, "model": RetrievalErrorResponse,
"description": ( "description": ("Chýbajúci alebo neplatný " "API kľúč."),
"Chýbajúci alebo neplatný "
"API kľúč."
),
}, },
422: { 422: {
"model": RetrievalErrorResponse, "model": RetrievalErrorResponse,
"description": ( "description": ("Neplatná štruktúra requestu."),
"Neplatná štruktúra requestu."
),
}, },
500: { 500: {
"model": RetrievalErrorResponse, "model": RetrievalErrorResponse,
"description": ( "description": ("Interná chyba servera."),
"Interná chyba servera."
),
}, },
503: { 503: {
"model": RetrievalErrorResponse, "model": RetrievalErrorResponse,
"description": ( "description": ("Retrieval nie je momentálne " "dostupný."),
"Retrieval nie je momentálne "
"dostupný."
),
}, },
}, },
dependencies=[ dependencies=[Depends(require_search_api_key)],
Depends(
require_search_api_key
)
],
) )
def rag( def rag(
request: RagRequest, request: RagRequest,
@ -605,39 +507,33 @@ def rag(
DB_FILE, DB_FILE,
request.query, request.query,
limit=request.limit, limit=request.limit,
published_only=( published_only=(request.published_only),
request.published_only max_per_document=(request.max_per_document),
),
max_per_document=(
request.max_per_document
),
) )
http_request.app.state.embedding_ready = True http_request.app.state.embedding_ready = True
return compact_rag_payload(result, include_diagnostics=request.include_diagnostics) return compact_rag_payload(result, include_diagnostics=request.include_diagnostics)
except Exception as error: except Exception as error:
raise_retrieval_error( raise_retrieval_error(error)
error
)
@router.post('/graphrag', dependencies=[Depends(require_search_api_key)], include_in_schema=False) @router.post("/graphrag", dependencies=[Depends(require_search_api_key)], include_in_schema=False)
def graph_retrieval(request: GraphRequest) -> dict[str, Any]: def graph_retrieval(request: GraphRequest) -> dict[str, Any]:
"""Native GraphRAG Tool transport; the legacy OpenAPI schema stays stable.""" """Native GraphRAG Tool transport; the legacy OpenAPI schema stays stable."""
try: try:
return search_graph(request.query, limit=request.limit, max_entities=request.max_entities, return search_graph(
published_only=request.published_only) request.query,
limit=request.limit,
max_entities=request.max_entities,
published_only=request.published_only,
)
except Exception as error: except Exception as error:
raise_retrieval_error(error) raise_retrieval_error(error)
@router.post( @router.post(
"/search", "/search",
dependencies=[ dependencies=[Depends(require_search_api_key)],
Depends(
require_search_api_key
)
],
include_in_schema=False, include_in_schema=False,
response_model=SearchResponse, response_model=SearchResponse,
) )
@ -650,48 +546,30 @@ def search(
DB_FILE, DB_FILE,
request.query, request.query,
request.limit, request.limit,
published_only=( published_only=(request.published_only),
request.published_only max_per_document=(request.max_per_document),
),
max_per_document=(
request.max_per_document
),
) )
http_request.app.state.embedding_ready = True http_request.app.state.embedding_ready = True
except Exception as error: except Exception as error:
raise_retrieval_error( raise_retrieval_error(error)
error
)
results = response[ results = response["results"]
"results"
]
return { return {
**{key: response[key] for key in RetrievalResponse.model_fields if key in response}, **{key: response[key] for key in RetrievalResponse.model_fields if key in response},
"query": request.query, "query": request.query,
"engine": response[ "engine": response["engine"],
"engine" "strategies": response["strategies"],
], "count": len(results),
"strategies": response[
"strategies"
],
"count": len(
results
),
"results": results, "results": results,
**({'index_version': response['index_version']} if response.get('index_version') else {}), **({"index_version": response["index_version"]} if response.get("index_version") else {}),
} }
@router.post( @router.post(
"/sync", "/sync",
dependencies=[ dependencies=[Depends(require_sync_api_key)],
Depends(
require_sync_api_key
)
],
include_in_schema=False, include_in_schema=False,
response_model=SyncAcceptedResponse, response_model=SyncAcceptedResponse,
status_code=202, status_code=202,
@ -703,26 +581,25 @@ def sync(
return enqueue_sync(http_request, pull_git=request.pull_git) return enqueue_sync(http_request, pull_git=request.pull_git)
@router.get('/sync/status', dependencies=[Depends(require_sync_api_key)], include_in_schema=False) @router.get("/sync/status", dependencies=[Depends(require_sync_api_key)], include_in_schema=False)
def sync_status(request: Request) -> dict: def sync_status(request: Request) -> dict:
return request.app.state.sync_jobs.overview() return request.app.state.sync_jobs.overview()
@router.get('/sync/jobs/{job_id}', dependencies=[Depends(require_sync_api_key)], include_in_schema=False) @router.get(
"/sync/jobs/{job_id}", dependencies=[Depends(require_sync_api_key)], include_in_schema=False
)
def sync_job(job_id: str, request: Request) -> dict: def sync_job(job_id: str, request: Request) -> dict:
job = request.app.state.sync_jobs.get(job_id) job = request.app.state.sync_jobs.get(job_id)
if job is None: if job is None:
raise HTTPException(status_code=404, detail='Synchronizačná úloha neexistuje') raise HTTPException(status_code=404, detail="Synchronizačná úloha neexistuje")
return job return job
@router.post( @router.post(
"/webhook/gitea", "/webhook/gitea",
status_code=202, status_code=202,
response_model=( response_model=(SyncAcceptedResponse | WebhookIgnoredResponse),
SyncAcceptedResponse
| WebhookIgnoredResponse
),
response_model_exclude_unset=True, response_model_exclude_unset=True,
include_in_schema=False, include_in_schema=False,
) )
@ -736,15 +613,10 @@ async def gitea_webhook(
default=None, default=None,
alias="X-Gitea-Signature", alias="X-Gitea-Signature",
), ),
) -> ( ) -> dict[str, Any] | JSONResponse:
dict[str, Any]
| JSONResponse
):
secret = webhook_secret() secret = webhook_secret()
raw_body = await read_webhook_body( raw_body = await read_webhook_body(request)
request
)
# Gitea podpis overujeme nad presným raw # Gitea podpis overujeme nad presným raw
# request body ešte pred JSON parsingom. # request body ešte pred JSON parsingom.
@ -754,33 +626,20 @@ async def gitea_webhook(
secret, secret,
): ):
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_401_UNAUTHORIZED),
status.HTTP_401_UNAUTHORIZED detail=("Neplatný webhook podpis"),
),
detail=(
"Neplatný webhook podpis"
),
) )
try: try:
payload = json.loads( payload = json.loads(raw_body.decode("utf-8"))
raw_body.decode(
"utf-8"
)
)
except ( except (
UnicodeDecodeError, UnicodeDecodeError,
json.JSONDecodeError, json.JSONDecodeError,
) as error: ) as error:
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_400_BAD_REQUEST),
status.HTTP_400_BAD_REQUEST detail=("Webhook payload nie je " "platný JSON"),
),
detail=(
"Webhook payload nie je "
"platný JSON"
),
) from error ) from error
if not isinstance( if not isinstance(
@ -788,97 +647,73 @@ async def gitea_webhook(
dict, dict,
): ):
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_400_BAD_REQUEST),
status.HTTP_400_BAD_REQUEST detail=("Webhook payload musí byť " "JSON objekt"),
),
detail=(
"Webhook payload musí byť "
"JSON objekt"
),
) )
if not x_gitea_event: if not x_gitea_event:
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_400_BAD_REQUEST),
status.HTTP_400_BAD_REQUEST detail=("Chýba hlavička " "X-Gitea-Event"),
),
detail=(
"Chýba hlavička "
"X-Gitea-Event"
),
) )
if ( if x_gitea_event.casefold() != "push":
x_gitea_event.casefold()
!= "push"
):
return JSONResponse( return JSONResponse(
status_code=( status_code=(status.HTTP_202_ACCEPTED),
status.HTTP_202_ACCEPTED
),
content={ content={
"status": ( "status": ("ignored"),
"ignored" "reason": ("unsupported_event"),
), "event": (x_gitea_event),
"reason": (
"unsupported_event"
),
"event": (
x_gitea_event
),
}, },
) )
repository_name = ( repository_name = repository_name_from_payload(payload)
repository_name_from_payload(
payload
)
)
if repository_name is None: if repository_name is None:
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_400_BAD_REQUEST),
status.HTTP_400_BAD_REQUEST detail=("Webhook payload neobsahuje " "repository.full_name"),
),
detail=(
"Webhook payload neobsahuje "
"repository.full_name"
),
) )
expected_repository = ( expected_repository = configured_repository()
configured_repository()
)
if not same_repository( if not same_repository(
repository_name, repository_name,
expected_repository, expected_repository,
): ):
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_403_FORBIDDEN),
status.HTTP_403_FORBIDDEN detail=("Webhook patrí neočakávanému " "repozitáru"),
),
detail=(
"Webhook patrí neočakávanému "
"repozitáru"
),
) )
ref = payload.get('ref') ref = payload.get("ref")
if not isinstance(ref, str) or not ref.startswith('refs/heads/'): if not isinstance(ref, str) or not ref.startswith("refs/heads/"):
raise HTTPException(status_code=400, detail='Push payload musí obsahovať ref vetvy') raise HTTPException(status_code=400, detail="Push payload musí obsahovať ref vetvy")
if ref != f'refs/heads/{expected_branch()}': if ref != f"refs/heads/{expected_branch()}":
return {'status': 'ignored', 'reason': 'unexpected_branch', 'event': x_gitea_event} return {"status": "ignored", "reason": "unexpected_branch", "event": x_gitea_event}
revision = payload.get('after') revision = payload.get("after")
if payload.get('deleted') is True or (isinstance(revision, str) and revision and set(revision) == {'0'}): if payload.get("deleted") is True or (
return {'status': 'ignored', 'reason': 'deleted_branch', 'event': x_gitea_event} isinstance(revision, str) and revision and set(revision) == {"0"}
if revision is not None and (not isinstance(revision, str) or not re.fullmatch(r'[0-9a-fA-F]{40}|[0-9a-fA-F]{64}', revision)): ):
raise HTTPException(status_code=400, detail='Neplatná Git revízia v push payload') return {"status": "ignored", "reason": "deleted_branch", "event": x_gitea_event}
delivery = request.headers.get('X-Gitea-Delivery') if revision is not None and (
if delivery is not None and (not delivery.strip() or len(delivery) > 200 or any(ord(c) < 32 for c in delivery)): not isinstance(revision, str)
raise HTTPException(status_code=400, detail='Neplatné ID webhook doručenia') or not re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", revision)
):
raise HTTPException(status_code=400, detail="Neplatná Git revízia v push payload")
delivery = request.headers.get("X-Gitea-Delivery")
if delivery is not None and (
not delivery.strip() or len(delivery) > 200 or any(ord(c) < 32 for c in delivery)
):
raise HTTPException(status_code=400, detail="Neplatné ID webhook doručenia")
return { return {
**enqueue_sync(request, pull_git=webhook_should_pull_git(), requested_revision=revision, **enqueue_sync(
delivery_key=f'{repository_name.casefold()}:{delivery}' if delivery else None), request,
'event': x_gitea_event, 'repository': repository_name, 'verified_by': 'hmac_sha256', pull_git=webhook_should_pull_git(),
requested_revision=revision,
delivery_key=f"{repository_name.casefold()}:{delivery}" if delivery else None,
),
"event": x_gitea_event,
"repository": repository_name,
"verified_by": "hmac_sha256",
} }

View File

@ -17,7 +17,6 @@ from fastapi.security import (
HTTPBearer, HTTPBearer,
) )
MIN_SECRET_LENGTH = 32 MIN_SECRET_LENGTH = 32
SEARCH_API_KEY_HEADER = "X-API-Key" SEARCH_API_KEY_HEADER = "X-API-Key"
@ -43,29 +42,20 @@ search_api_key_scheme = APIKeyHeader(
name=SEARCH_API_KEY_HEADER, name=SEARCH_API_KEY_HEADER,
scheme_name="SearchApiKey", scheme_name="SearchApiKey",
auto_error=False, auto_error=False,
description=( description=("API kľúč pre vyhľadávanie v ZP Wiki " "cez hlavičku X-API-Key."),
"API kľúč pre vyhľadávanie v ZP Wiki "
"cez hlavičku X-API-Key."
),
) )
search_bearer_scheme = HTTPBearer( search_bearer_scheme = HTTPBearer(
scheme_name="SearchBearer", scheme_name="SearchBearer",
auto_error=False, auto_error=False,
description=( description=("Bearer token pre vyhľadávanie v ZP Wiki. " "Používa hodnotu SEARCH_API_KEY."),
"Bearer token pre vyhľadávanie v ZP Wiki. "
"Používa hodnotu SEARCH_API_KEY."
),
) )
sync_api_key_scheme = APIKeyHeader( sync_api_key_scheme = APIKeyHeader(
name=SYNC_API_KEY_HEADER, name=SYNC_API_KEY_HEADER,
scheme_name="SyncApiKey", scheme_name="SyncApiKey",
auto_error=False, auto_error=False,
description=( description=("API kľúč pre manuálne spustenie " "reindexovania."),
"API kľúč pre manuálne spustenie "
"reindexovania."
),
) )
@ -78,10 +68,7 @@ def required_environment_value(
).strip() ).strip()
if not value: if not value:
raise RuntimeError( raise RuntimeError("Chýba povinná environment " f"premenná {name}")
"Chýba povinná environment "
f"premenná {name}"
)
return value return value
@ -89,57 +76,39 @@ def required_environment_value(
def validate_secret( def validate_secret(
name: str, name: str,
) -> str: ) -> str:
value = required_environment_value( value = required_environment_value(name)
name
)
if len(value) < MIN_SECRET_LENGTH: if len(value) < MIN_SECRET_LENGTH:
raise RuntimeError( raise RuntimeError(f"{name} musí mať aspoň " f"{MIN_SECRET_LENGTH} znakov")
f"{name} musí mať aspoň "
f"{MIN_SECRET_LENGTH} znakov"
)
return value return value
def expected_gitea_repository() -> str: def expected_gitea_repository() -> str:
value = required_environment_value( value = required_environment_value("EXPECTED_GITEA_REPOSITORY")
"EXPECTED_GITEA_REPOSITORY"
)
parts = value.split( parts = value.split("/")
"/"
)
if len(parts) != 2: if len(parts) != 2:
raise RuntimeError( raise RuntimeError("EXPECTED_GITEA_REPOSITORY musí mať " "tvar vlastník/repozitár")
"EXPECTED_GITEA_REPOSITORY musí mať "
"tvar vlastník/repozitár"
)
owner, repository = parts owner, repository = parts
if ( if not owner or not repository or any(character.isspace() for character in value):
not owner raise RuntimeError("EXPECTED_GITEA_REPOSITORY musí mať " "tvar vlastník/repozitár")
or not repository
or any(
character.isspace()
for character in value
)
):
raise RuntimeError(
"EXPECTED_GITEA_REPOSITORY musí mať "
"tvar vlastník/repozitár"
)
return value return value
def webhook_should_pull_git() -> bool: def webhook_should_pull_git() -> bool:
value = os.getenv( value = (
os.getenv(
"WEBHOOK_PULL_GIT", "WEBHOOK_PULL_GIT",
"false", "false",
).strip().casefold() )
.strip()
.casefold()
)
if value in TRUE_ENV_VALUES: if value in TRUE_ENV_VALUES:
return True return True
@ -147,39 +116,31 @@ def webhook_should_pull_git() -> bool:
if value in FALSE_ENV_VALUES: if value in FALSE_ENV_VALUES:
return False return False
raise RuntimeError( raise RuntimeError("WEBHOOK_PULL_GIT musí byť boolean " "hodnota true/false")
"WEBHOOK_PULL_GIT musí byť boolean "
"hodnota true/false"
)
def validate_security_configuration() -> None: def validate_security_configuration() -> None:
webhook_secret = validate_secret( webhook_secret = validate_secret("WEBHOOK_SECRET")
"WEBHOOK_SECRET"
)
sync_api_key = validate_secret( sync_api_key = validate_secret("SYNC_API_KEY")
"SYNC_API_KEY"
)
search_api_key = validate_secret( search_api_key = validate_secret("SEARCH_API_KEY")
"SEARCH_API_KEY"
)
# Každá funkcia musí používať vlastný # Každá funkcia musí používať vlastný
# credential. Jeden uniknutý secret tak # credential. Jeden uniknutý secret tak
# neposkytne prístup ku všetkým operáciám. # neposkytne prístup ku všetkým operáciám.
if len( if (
len(
{ {
webhook_secret, webhook_secret,
sync_api_key, sync_api_key,
search_api_key, search_api_key,
} }
) != 3: )
!= 3
):
raise RuntimeError( raise RuntimeError(
"WEBHOOK_SECRET, SYNC_API_KEY " "WEBHOOK_SECRET, SYNC_API_KEY " "a SEARCH_API_KEY musia byť " "navzájom rozdielne"
"a SEARCH_API_KEY musia byť "
"navzájom rozdielne"
) )
expected_gitea_repository() expected_gitea_repository()
@ -191,90 +152,51 @@ def validate_security_configuration() -> None:
def require_search_api_key( def require_search_api_key(
api_key: str | None = Security( api_key: str | None = Security(search_api_key_scheme),
search_api_key_scheme bearer: HTTPAuthorizationCredentials | None = Security(search_bearer_scheme),
),
bearer: (
HTTPAuthorizationCredentials | None
) = Security(
search_bearer_scheme
),
) -> None: ) -> None:
expected = validate_secret( expected = validate_secret("SEARCH_API_KEY")
"SEARCH_API_KEY"
)
supplied_credentials: list[str] = [] supplied_credentials: list[str] = []
if api_key: if api_key:
supplied_credentials.append( supplied_credentials.append(api_key)
api_key
)
if ( if bearer is not None and bearer.scheme.casefold() == "bearer" and bearer.credentials:
bearer is not None supplied_credentials.append(bearer.credentials)
and bearer.scheme.casefold()
== "bearer"
and bearer.credentials
):
supplied_credentials.append(
bearer.credentials
)
valid = any( valid = any(
hmac.compare_digest( hmac.compare_digest(
supplied, supplied,
expected, expected,
) )
for supplied for supplied in supplied_credentials
in supplied_credentials
) )
if not valid: if not valid:
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_401_UNAUTHORIZED),
status.HTTP_401_UNAUTHORIZED detail=("Neplatný alebo chýbajúci " "API kľúč"),
),
detail=(
"Neplatný alebo chýbajúci "
"API kľúč"
),
headers={ headers={
"WWW-Authenticate": ( "WWW-Authenticate": ("Bearer"),
"Bearer"
),
}, },
) )
def require_sync_api_key( def require_sync_api_key(
api_key: str | None = Security( api_key: str | None = Security(sync_api_key_scheme),
sync_api_key_scheme
),
) -> None: ) -> None:
expected = validate_secret( expected = validate_secret("SYNC_API_KEY")
"SYNC_API_KEY"
)
if ( if not api_key or not hmac.compare_digest(
not api_key
or not hmac.compare_digest(
api_key, api_key,
expected, expected,
)
): ):
raise HTTPException( raise HTTPException(
status_code=( status_code=(status.HTTP_401_UNAUTHORIZED),
status.HTTP_401_UNAUTHORIZED detail=("Neplatný alebo chýbajúci " "API kľúč"),
),
detail=(
"Neplatný alebo chýbajúci "
"API kľúč"
),
headers={ headers={
"WWW-Authenticate": ( "WWW-Authenticate": ("ApiKey"),
"ApiKey"
),
}, },
) )
@ -287,18 +209,10 @@ def verify_gitea_signature(
if not signature: if not signature:
return False return False
supplied = ( supplied = signature.strip().casefold()
signature
.strip()
.casefold()
)
if supplied.startswith( if supplied.startswith("sha256="):
"sha256=" supplied = supplied.removeprefix("sha256=")
):
supplied = supplied.removeprefix(
"sha256="
)
if len(supplied) != 64: if len(supplied) != 64:
return False return False
@ -313,9 +227,7 @@ def verify_gitea_signature(
return False return False
expected = hmac.new( expected = hmac.new(
secret.encode( secret.encode("utf-8"),
"utf-8"
),
raw_body, raw_body,
hashlib.sha256, hashlib.sha256,
).hexdigest() ).hexdigest()
@ -329,9 +241,7 @@ def verify_gitea_signature(
def repository_name_from_payload( def repository_name_from_payload(
payload: dict[str, Any], payload: dict[str, Any],
) -> str | None: ) -> str | None:
repository = payload.get( repository = payload.get("repository")
"repository"
)
if not isinstance( if not isinstance(
repository, repository,
@ -347,9 +257,7 @@ def repository_name_from_payload(
# Samotné "name" nestačí, pretože # Samotné "name" nestačí, pretože
# rovnaký názov môže existovať pod # rovnaký názov môže existovať pod
# rôznymi vlastníkmi. # rôznymi vlastníkmi.
value = repository.get( value = repository.get("full_name")
"full_name"
)
if not isinstance( if not isinstance(
value, value,

View File

@ -24,8 +24,10 @@ COPY deployment/requirements.txt ./deployment/requirements.txt
RUN python -m pip install --no-cache-dir \ RUN python -m pip install --no-cache-dir \
-r deployment/requirements.txt -r deployment/requirements.txt
COPY app ./app # Prepare immutable assets before application code, so code-only changes do
COPY tools ./tools # not invalidate model preparation. The preparer only imports its own module.
COPY tools/__init__.py ./tools/__init__.py
COPY tools/runtime/__init__.py tools/runtime/runtime_assets.py ./tools/runtime/
COPY runtime_assets/ /opt/zp-assets/ COPY runtime_assets/ /opt/zp-assets/
ARG EMBEDDING_MODEL=intfloat/multilingual-e5-small ARG EMBEDDING_MODEL=intfloat/multilingual-e5-small
@ -37,6 +39,9 @@ RUN python -m tools.runtime.runtime_assets prepare --output /opt/zp-assets \
--model "$EMBEDDING_MODEL" --revision "$EMBEDDING_REVISION" \ --model "$EMBEDDING_MODEL" --revision "$EMBEDDING_REVISION" \
--encoding "$CHUNK_TOKEN_ENCODING" --encoding "$RAG_LLM_TOKEN_ENCODING" --encoding "$CHUNK_TOKEN_ENCODING" --encoding "$RAG_LLM_TOKEN_ENCODING"
COPY app ./app
COPY tools ./tools
ENV ZP_RUNTIME_ASSETS=/opt/zp-assets ENV ZP_RUNTIME_ASSETS=/opt/zp-assets
ENV ZP_ASSETS_REQUIRED=1 ENV ZP_ASSETS_REQUIRED=1
ENV HF_HUB_OFFLINE=1 ENV HF_HUB_OFFLINE=1

Binary file not shown.

2396
experiments/catalog.json Normal file

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More