Upratanie modulov a archivácia historických evaluácií

This commit is contained in:
Ján Pták 2026-11-07 18:16:19 +01:00
parent 3d0dfa154b
commit 507c247ea3
492 changed files with 29804 additions and 1400531 deletions

View File

@ -13,3 +13,10 @@ docs/
integrations/
tools/git_runner/
**/*.md
experiments/
test/
tools/**/tests/
tools/evaluation/
tools/rag/evaluation/
tools/graphrag/evaluation/
tools/claude/

View File

@ -62,5 +62,27 @@ v Docker sieti. Dočasná verejná URL vyžaduje zodpovedajúce OAuth nastavenie
RAG, GraphRAG, kontrolovaná cesta odpovedí a pripojenie cez Claude sú funkčné.
Systém obsahuje embeddingovú cache, kontroly dostupnosti a persistentnú
synchronizačnú frontu. Historické evaluácie sú uložené pri jednotlivých vetvách.
synchronizačnú frontu. Historické evaluácie sú oddelené od prevádzkového kódu.
Samostatný hlavný agent, CRZP a spracovanie plných textov prác sú ďalšie etapy.
## Organizácia projektu
- `app/` – HTTP API a zabezpečenie.
- `tools/rag/`, `tools/graphrag/`, `tools/knowledge/` – vyhľadávanie a indexovanie.
- `tools/answers/` – príprava podkladov, kontroly a zostavenie odpovede.
- `tools/retrieval/`, `tools/sync/`, `tools/runtime/` – spoločné pravidlá a prevádzka.
- `tools/evaluation/` a `tools/*/evaluation/` – offline evaluátory a diagnostika.
- `experiments/rag/`, `experiments/graphrag/` – vybrané historické vstupy a výsledky.
- `experiments/archive/` – komprimované vývojové pokusy; mapovanie a SHA-256 sú v `catalog.json`.
- `packages/claude/`, `deployment/`, `scripts/` – klienti, nasadenie a spúšťanie.
Prevádzkový image neobsahuje testy ani historické experimenty. Spúšťa sa cez
`bash scripts/start.sh`; voliteľné služby ovláda `bash scripts/compose.sh`.
Regresie spúšťa `.venv/bin/python -m pytest -q test tools/rag/tests tools/graphrag/tests`.
Integritu experimentov overí `python -m tools.evaluation.archive verify`.
Archív možno obnoviť bez prepisovania súborov pomocou
`python -m tools.evaluation.archive restore --destination /tmp/zp-experiments`.
Pôvodný kód experimentov určuje `source_commit` v katalógu a historické Git tagy.
Staré manifesty ani výsledky sa neprepisujú na odtlačky aktuálnej implementácie;
ich úspešnosť nie je novým meraním po reorganizácii.

View File

@ -19,7 +19,6 @@ from fastapi.exception_handlers import http_exception_handler, request_validatio
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.responses import JSONResponse
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
@ -39,13 +38,10 @@ from tools.rag.common import DATA_DIR
from tools.sync.sync_jobs import SyncJobs
from tools.rag.rebuild_index import rebuild_index
logger = logging.getLogger(__name__)
OPENWEBUI_ORIGIN = (
"https://ui.tukekemt.xyz"
)
OPENWEBUI_ORIGIN = "https://ui.tukekemt.xyz"
@asynccontextmanager
@ -64,9 +60,7 @@ async def lifespan(
except Exception:
logger.exception(
"Predhriatie embedding modelu "
"zlyhalo. API pokračuje bez "
"úspešného warm-upu."
"Predhriatie embedding modelu " "zlyhalo. API pokračuje bez " "úspešného warm-upu."
)
def run_sync(**kwargs):
@ -99,23 +93,36 @@ app = FastAPI(
@app.exception_handler(StarletteHTTPException)
async def retrieval_http_error(request: Request, error: StarletteHTTPException):
if request.url.path not in {'/rag', '/search', '/graphrag'}:
if request.url.path not in {"/rag", "/search", "/graphrag"}:
return await http_exception_handler(request, error)
reason = {400: 'invalid_query', 401: 'unauthorized', 403: 'forbidden',
404: 'not_found', 405: 'method_not_allowed', 422: 'invalid_request',
503: 'retrieval_unavailable'}.get(error.status_code, 'internal_error')
return JSONResponse(status_code=error.status_code, headers=error.headers,
content={**error_result(reason=reason), 'detail': error.detail})
reason = {
400: "invalid_query",
401: "unauthorized",
403: "forbidden",
404: "not_found",
405: "method_not_allowed",
422: "invalid_request",
503: "retrieval_unavailable",
}.get(error.status_code, "internal_error")
return JSONResponse(
status_code=error.status_code,
headers=error.headers,
content={**error_result(reason=reason), "detail": error.detail},
)
@app.exception_handler(RequestValidationError)
async def retrieval_validation_error(request: Request, error: RequestValidationError):
if request.url.path not in {'/rag', '/search', '/graphrag'}:
if request.url.path not in {"/rag", "/search", "/graphrag"}:
return await request_validation_exception_handler(request, error)
# Retain useful validation locations/types without echoing submitted secrets.
details = [{'loc': row['loc'], 'type': row['type'], 'msg': row['msg']} for row in error.errors()]
return JSONResponse(status_code=422, content=jsonable_encoder({
**error_result(reason='invalid_request'), 'detail': details}))
details = [
{"loc": row["loc"], "type": row["type"], "msg": row["msg"]} for row in error.errors()
]
return JSONResponse(
status_code=422,
content=jsonable_encoder({**error_result(reason="invalid_request"), "detail": details}),
)
app.add_middleware(
@ -149,9 +156,7 @@ app.add_middleware(
)
@app.middleware(
"http"
)
@app.middleware("http")
async def add_security_headers(
request: Request,
call_next,
@ -159,33 +164,27 @@ async def add_security_headers(
try:
response = await call_next(request)
except Exception:
if request.url.path not in {'/rag', '/search', '/graphrag'}:
if request.url.path not in {"/rag", "/search", "/graphrag"}:
raise
# Serialization failures must obey the same contract as backend errors.
logger.exception('Neočakávaná chyba výstupu retrieval API.')
response = JSONResponse(status_code=500, content={
**error_result(reason='internal_error'),
'detail': 'Nastala interná chyba servera.'})
logger.exception("Neočakávaná chyba výstupu retrieval API.")
response = JSONResponse(
status_code=500,
content={
**error_result(reason="internal_error"),
"detail": "Nastala interná chyba servera.",
},
)
response.headers[
"Cache-Control"
] = "no-store"
response.headers["Cache-Control"] = "no-store"
response.headers[
"X-Content-Type-Options"
] = "nosniff"
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers[
"Referrer-Policy"
] = "no-referrer"
response.headers["Referrer-Policy"] = "no-referrer"
response.headers[
"X-Frame-Options"
] = "DENY"
response.headers["X-Frame-Options"] = "DENY"
return response
app.include_router(
router
)
app.include_router(router)

View File

@ -47,38 +47,33 @@ from tools.rag.search_utils import search_database
from tools.sync.wiki_git import expected_branch
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY
from tools.runtime.tool_health import tools_readiness
from tools.retrieval.retrieval_contract import RetrievalResponse, RetrievalSource, RetrievalErrorResponse, error_reason
from tools.retrieval.retrieval_contract import (
RetrievalResponse,
RetrievalSource,
RetrievalErrorResponse,
error_reason,
)
from tools.answers.answer_delivery import deliver_answer
from tools.answers.answer_errors import public_failure
logger = logging.getLogger(__name__)
router = APIRouter()
MAX_WEBHOOK_BODY_BYTES = (
2 * 1024 * 1024
)
MAX_WEBHOOK_BODY_BYTES = 2 * 1024 * 1024
RETRIEVAL_UNAVAILABLE_DETAIL = (
"Vyhľadávací index alebo embeddingový "
"model momentálne nie je dostupný."
"Vyhľadávací index alebo embeddingový " "model momentálne nie je dostupný."
)
INVALID_QUERY_DETAIL = (
"Neplatný vyhľadávací dotaz."
)
INVALID_QUERY_DETAIL = "Neplatný vyhľadávací dotaz."
INTERNAL_ERROR_DETAIL = (
"Nastala interná chyba servera."
)
INTERNAL_ERROR_DETAIL = "Nastala interná chyba servera."
WEBHOOK_CONFIGURATION_DETAIL = (
"Webhook nie je správne nakonfigurovaný."
)
WEBHOOK_CONFIGURATION_DETAIL = "Webhook nie je správne nakonfigurovaný."
class StrictRequestModel(BaseModel):
@ -88,9 +83,7 @@ class StrictRequestModel(BaseModel):
)
class SearchRequest(
StrictRequestModel
):
class SearchRequest(StrictRequestModel):
query: str = Field(
...,
min_length=1,
@ -112,28 +105,20 @@ class SearchRequest(
)
class RagRequest(
StrictRequestModel
):
class RagRequest(StrictRequestModel):
include_diagnostics: bool = False
query: str = Field(
...,
min_length=1,
max_length=500,
description=(
"Otázka alebo vyhľadávací dotaz "
"používateľa nad ZP Wiki."
),
description=("Otázka alebo vyhľadávací dotaz " "používateľa nad ZP Wiki."),
)
limit: int = Field(
default=5,
ge=1,
le=20,
description=(
"Maximálny počet relevantných "
"zdrojov pre RAG kontext."
),
description=("Maximálny počet relevantných " "zdrojov pre RAG kontext."),
)
published_only: bool = Field(
@ -156,14 +141,11 @@ class RagRequest(
)
class SyncRequest(
StrictRequestModel
):
class SyncRequest(StrictRequestModel):
pull_git: bool = Field(
default=False,
description=(
"Pred reindexovaním stiahne očakávanú vetvu "
"a aktualizuje ju iba fast-forward."
"Pred reindexovaním stiahne očakávanú vetvu " "a aktualizuje ju iba fast-forward."
),
)
@ -177,34 +159,43 @@ class GraphRequest(StrictRequestModel):
class AnswerRequest(StrictRequestModel):
query: str = Field(min_length=1, max_length=500)
engine: Literal['rag', 'graphrag'] = 'rag'
engine: Literal["rag", "graphrag"] = "rag"
limit: int = Field(default=5, ge=1, le=20)
include_diagnostics: bool = False
@router.post('/answer', dependencies=[Depends(require_search_api_key)], include_in_schema=False)
@router.post("/answer", dependencies=[Depends(require_search_api_key)], include_in_schema=False)
def verified_answer(request: AnswerRequest) -> dict:
try:
result = deliver_answer(request.query, engine=request.engine, limit=request.limit,
include_diagnostics=request.include_diagnostics)
result = deliver_answer(
request.query,
engine=request.engine,
limit=request.limit,
include_diagnostics=request.include_diagnostics,
)
except Exception:
logger.exception('Príprava overenej odpovede zlyhala.')
return {'status': 'error', 'reason': 'answer_unavailable', 'accepted_for_delivery': False,
'answer': 'Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.'}
if (result.get('accepted_for_delivery') is not True or
result.get('status') not in {'ok', 'no_answer', 'ambiguous'}):
logger.exception("Príprava overenej odpovede zlyhala.")
return {
"status": "error",
"reason": "answer_unavailable",
"accepted_for_delivery": False,
"answer": "Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.",
}
if result.get("accepted_for_delivery") is not True or result.get("status") not in {
"ok",
"no_answer",
"ambiguous",
}:
result = public_failure(result)
result={**result,'query':request.query.strip(),'engine':request.engine}
result = {**result, "query": request.query.strip(), "engine": request.engine}
if not request.include_diagnostics:
result.pop('attempts', None)
result.pop('retrieval_context', None)
result.pop("attempts", None)
result.pop("retrieval_context", None)
return result
class HealthResponse(BaseModel):
status: Literal[
"ok"
]
status: Literal["ok"]
ready: bool
@ -235,10 +226,13 @@ class RagSource(RetrievalSource):
heading_paths: list[Any] | None = None
text: str | None = None
retrieval: dict[
str,
Any,
] | None = None
retrieval: (
dict[
str,
Any,
]
| None
) = None
class RagResponse(RetrievalResponse):
@ -264,9 +258,7 @@ class RagResponse(RetrievalResponse):
context: str
sources: list[
RagSource
]
sources: list[RagSource]
index_version: dict[str, Any] | None = None
context_budget: dict[str, Any] | None = None
diagnostics: dict[str, Any] | None = None
@ -285,14 +277,12 @@ class SearchResponse(RetrievalResponse):
ge=0,
)
results: list[
dict[str, Any]
]
results: list[dict[str, Any]]
index_version: dict[str, Any] | None = None
class SyncAcceptedResponse(BaseModel):
status: Literal['accepted']
status: Literal["accepted"]
job_id: str
status_url: str
pull_git: bool
@ -300,12 +290,12 @@ class SyncAcceptedResponse(BaseModel):
duplicate: bool
event: str | None = None
repository: str | None = None
verified_by: Literal['hmac_sha256'] | None = None
verified_by: Literal["hmac_sha256"] | None = None
class WebhookIgnoredResponse(BaseModel):
status: Literal['ignored']
reason: Literal['unsupported_event', 'unexpected_branch', 'deleted_branch']
status: Literal["ignored"]
reason: Literal["unsupported_event", "unexpected_branch", "deleted_branch"]
event: str
@ -313,12 +303,17 @@ def enqueue_sync(request: Request, **kwargs) -> dict:
try:
accepted = request.app.state.sync_jobs.enqueue(**kwargs)
except (sqlite3.Error, OSError):
logger.exception('Prijatie synchronizácie do fronty zlyhalo.')
raise HTTPException(status_code=503, detail='Synchronizačná fronta nie je dostupná')
job = accepted['job']
return {'status': 'accepted', 'job_id': job['id'],
'status_url': f"/sync/jobs/{job['id']}", 'pull_git': job['pull_git'],
'coalesced': accepted['coalesced'], 'duplicate': accepted['duplicate']}
logger.exception("Prijatie synchronizácie do fronty zlyhalo.")
raise HTTPException(status_code=503, detail="Synchronizačná fronta nie je dostupná")
job = accepted["job"]
return {
"status": "accepted",
"job_id": job["id"],
"status_url": f"/sync/jobs/{job['id']}",
"pull_git": job["pull_git"],
"coalesced": accepted["coalesced"],
"duplicate": accepted["duplicate"],
}
def raise_retrieval_error(
@ -334,127 +329,77 @@ def raise_retrieval_error(
)
raise HTTPException(
status_code=(
status.HTTP_400_BAD_REQUEST
),
detail=(
INVALID_QUERY_DETAIL
),
status_code=(status.HTTP_400_BAD_REQUEST),
detail=(INVALID_QUERY_DETAIL),
) from error
if error_reason(error) == 'retrieval_unavailable':
logger.exception(
"Retrieval nie je dostupný."
)
if error_reason(error) == "retrieval_unavailable":
logger.exception("Retrieval nie je dostupný.")
raise HTTPException(
status_code=(
status.HTTP_503_SERVICE_UNAVAILABLE
),
detail=(
RETRIEVAL_UNAVAILABLE_DETAIL
),
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
detail=(RETRIEVAL_UNAVAILABLE_DETAIL),
) from error
logger.exception(
"Neočakávaná chyba retrieval API."
)
logger.exception("Neočakávaná chyba retrieval API.")
raise HTTPException(
status_code=(
status.HTTP_500_INTERNAL_SERVER_ERROR
),
detail=(
INTERNAL_ERROR_DETAIL
),
status_code=(status.HTTP_500_INTERNAL_SERVER_ERROR),
detail=(INTERNAL_ERROR_DETAIL),
) from error
def webhook_secret() -> str:
try:
return validate_secret(
"WEBHOOK_SECRET"
)
return validate_secret("WEBHOOK_SECRET")
except RuntimeError as error:
logger.exception(
"WEBHOOK_SECRET nie je "
"správne nakonfigurovaný."
)
logger.exception("WEBHOOK_SECRET nie je " "správne nakonfigurovaný.")
raise HTTPException(
status_code=(
status.HTTP_503_SERVICE_UNAVAILABLE
),
detail=(
WEBHOOK_CONFIGURATION_DETAIL
),
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
detail=(WEBHOOK_CONFIGURATION_DETAIL),
) from error
def configured_repository() -> str:
try:
return (
expected_gitea_repository()
)
return expected_gitea_repository()
except RuntimeError as error:
logger.exception(
"EXPECTED_GITEA_REPOSITORY "
"nie je správne nakonfigurovaný."
)
logger.exception("EXPECTED_GITEA_REPOSITORY " "nie je správne nakonfigurovaný.")
raise HTTPException(
status_code=(
status.HTTP_503_SERVICE_UNAVAILABLE
),
detail=(
WEBHOOK_CONFIGURATION_DETAIL
),
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
detail=(WEBHOOK_CONFIGURATION_DETAIL),
) from error
async def read_webhook_body(
request: Request,
) -> bytes:
content_length = request.headers.get(
"content-length"
)
content_length = request.headers.get("content-length")
if content_length:
try:
declared_length = int(
content_length
)
declared_length = int(content_length)
except ValueError as error:
raise HTTPException(
status_code=400,
detail=(
"Neplatná Content-Length "
"hlavička"
),
detail=("Neplatná Content-Length " "hlavička"),
) from error
if declared_length < 0:
raise HTTPException(
status_code=400,
detail=(
"Neplatná Content-Length "
"hlavička"
),
detail=("Neplatná Content-Length " "hlavička"),
)
if (
declared_length
> MAX_WEBHOOK_BODY_BYTES
):
if declared_length > MAX_WEBHOOK_BODY_BYTES:
raise HTTPException(
status_code=413,
detail=(
"Webhook payload je "
"príliš veľký"
),
detail=("Webhook payload je " "príliš veľký"),
)
chunks: list[bytes] = []
@ -462,29 +407,17 @@ async def read_webhook_body(
total_size = 0
async for chunk in request.stream():
total_size += len(
chunk
)
total_size += len(chunk)
if (
total_size
> MAX_WEBHOOK_BODY_BYTES
):
if total_size > MAX_WEBHOOK_BODY_BYTES:
raise HTTPException(
status_code=413,
detail=(
"Webhook payload je "
"príliš veľký"
),
detail=("Webhook payload je " "príliš veľký"),
)
chunks.append(
chunk
)
chunks.append(chunk)
return b"".join(
chunks
)
return b"".join(chunks)
@router.get(
@ -493,13 +426,9 @@ async def read_webhook_body(
response_model=HealthResponse,
)
def health(request: Request) -> dict[str, Any]:
database_exists = (
DB_FILE.exists()
)
database_exists = DB_FILE.exists()
zpwiki_exists = (
ZPWIKI_ROOT.exists()
)
zpwiki_exists = ZPWIKI_ROOT.exists()
try:
validate_security_configuration()
@ -509,39 +438,30 @@ def health(request: Request) -> dict[str, Any]:
except RuntimeError:
security_configured = False
tools, version = tools_readiness(DB_FILE, security_configured=security_configured,
embedding_ready=getattr(request.app.state, 'embedding_ready', False))
ready = tools['rag']['ready'] and tools['graphrag']['ready']
tools, version = tools_readiness(
DB_FILE,
security_configured=security_configured,
embedding_ready=getattr(request.app.state, "embedding_ready", False),
)
ready = tools["rag"]["ready"] and tools["graphrag"]["ready"]
return {
"status": "ok",
"ready": ready,
"index_version": version,
"tools": tools,
"database_exists": (
database_exists
),
"search_engine": (
"hybrid_fts5_embeddings"
),
"database_exists": (database_exists),
"search_engine": ("hybrid_fts5_embeddings"),
"rag_enabled": True,
"zpwiki_exists": (
zpwiki_exists
),
"security_configured": (
security_configured
),
"zpwiki_exists": (zpwiki_exists),
"security_configured": (security_configured),
}
@router.post(
"/rag",
operation_id=(
"retrieve_zpwiki_context"
),
summary=(
"Vyhľadaj informácie v ZP Wiki"
),
operation_id=("retrieve_zpwiki_context"),
summary=("Vyhľadaj informácie v ZP Wiki"),
description=(
"Použi tento nástroj pri otázkach "
"o ZP Wiki, študentoch, autoroch, "
@ -553,48 +473,30 @@ def health(request: Request) -> dict[str, Any]:
),
response_model=RagResponse,
response_model_exclude_unset=True,
status_code=(
status.HTTP_200_OK
),
status_code=(status.HTTP_200_OK),
responses={
400: {
"model": RetrievalErrorResponse,
"description": (
"Neplatný dotaz."
),
"description": ("Neplatný dotaz."),
},
401: {
"model": RetrievalErrorResponse,
"description": (
"Chýbajúci alebo neplatný "
"API kľúč."
),
"description": ("Chýbajúci alebo neplatný " "API kľúč."),
},
422: {
"model": RetrievalErrorResponse,
"description": (
"Neplatná štruktúra requestu."
),
"description": ("Neplatná štruktúra requestu."),
},
500: {
"model": RetrievalErrorResponse,
"description": (
"Interná chyba servera."
),
"description": ("Interná chyba servera."),
},
503: {
"model": RetrievalErrorResponse,
"description": (
"Retrieval nie je momentálne "
"dostupný."
),
"description": ("Retrieval nie je momentálne " "dostupný."),
},
},
dependencies=[
Depends(
require_search_api_key
)
],
dependencies=[Depends(require_search_api_key)],
)
def rag(
request: RagRequest,
@ -605,39 +507,33 @@ def rag(
DB_FILE,
request.query,
limit=request.limit,
published_only=(
request.published_only
),
max_per_document=(
request.max_per_document
),
published_only=(request.published_only),
max_per_document=(request.max_per_document),
)
http_request.app.state.embedding_ready = True
return compact_rag_payload(result, include_diagnostics=request.include_diagnostics)
except Exception as error:
raise_retrieval_error(
error
)
raise_retrieval_error(error)
@router.post('/graphrag', dependencies=[Depends(require_search_api_key)], include_in_schema=False)
@router.post("/graphrag", dependencies=[Depends(require_search_api_key)], include_in_schema=False)
def graph_retrieval(request: GraphRequest) -> dict[str, Any]:
"""Native GraphRAG Tool transport; the legacy OpenAPI schema stays stable."""
try:
return search_graph(request.query, limit=request.limit, max_entities=request.max_entities,
published_only=request.published_only)
return search_graph(
request.query,
limit=request.limit,
max_entities=request.max_entities,
published_only=request.published_only,
)
except Exception as error:
raise_retrieval_error(error)
@router.post(
"/search",
dependencies=[
Depends(
require_search_api_key
)
],
dependencies=[Depends(require_search_api_key)],
include_in_schema=False,
response_model=SearchResponse,
)
@ -650,48 +546,30 @@ def search(
DB_FILE,
request.query,
request.limit,
published_only=(
request.published_only
),
max_per_document=(
request.max_per_document
),
published_only=(request.published_only),
max_per_document=(request.max_per_document),
)
http_request.app.state.embedding_ready = True
except Exception as error:
raise_retrieval_error(
error
)
raise_retrieval_error(error)
results = response[
"results"
]
results = response["results"]
return {
**{key: response[key] for key in RetrievalResponse.model_fields if key in response},
"query": request.query,
"engine": response[
"engine"
],
"strategies": response[
"strategies"
],
"count": len(
results
),
"engine": response["engine"],
"strategies": response["strategies"],
"count": len(results),
"results": results,
**({'index_version': response['index_version']} if response.get('index_version') else {}),
**({"index_version": response["index_version"]} if response.get("index_version") else {}),
}
@router.post(
"/sync",
dependencies=[
Depends(
require_sync_api_key
)
],
dependencies=[Depends(require_sync_api_key)],
include_in_schema=False,
response_model=SyncAcceptedResponse,
status_code=202,
@ -703,26 +581,25 @@ def sync(
return enqueue_sync(http_request, pull_git=request.pull_git)
@router.get('/sync/status', dependencies=[Depends(require_sync_api_key)], include_in_schema=False)
@router.get("/sync/status", dependencies=[Depends(require_sync_api_key)], include_in_schema=False)
def sync_status(request: Request) -> dict:
return request.app.state.sync_jobs.overview()
@router.get('/sync/jobs/{job_id}', dependencies=[Depends(require_sync_api_key)], include_in_schema=False)
@router.get(
"/sync/jobs/{job_id}", dependencies=[Depends(require_sync_api_key)], include_in_schema=False
)
def sync_job(job_id: str, request: Request) -> dict:
job = request.app.state.sync_jobs.get(job_id)
if job is None:
raise HTTPException(status_code=404, detail='Synchronizačná úloha neexistuje')
raise HTTPException(status_code=404, detail="Synchronizačná úloha neexistuje")
return job
@router.post(
"/webhook/gitea",
status_code=202,
response_model=(
SyncAcceptedResponse
| WebhookIgnoredResponse
),
response_model=(SyncAcceptedResponse | WebhookIgnoredResponse),
response_model_exclude_unset=True,
include_in_schema=False,
)
@ -736,15 +613,10 @@ async def gitea_webhook(
default=None,
alias="X-Gitea-Signature",
),
) -> (
dict[str, Any]
| JSONResponse
):
) -> dict[str, Any] | JSONResponse:
secret = webhook_secret()
raw_body = await read_webhook_body(
request
)
raw_body = await read_webhook_body(request)
# Gitea podpis overujeme nad presným raw
# request body ešte pred JSON parsingom.
@ -754,33 +626,20 @@ async def gitea_webhook(
secret,
):
raise HTTPException(
status_code=(
status.HTTP_401_UNAUTHORIZED
),
detail=(
"Neplatný webhook podpis"
),
status_code=(status.HTTP_401_UNAUTHORIZED),
detail=("Neplatný webhook podpis"),
)
try:
payload = json.loads(
raw_body.decode(
"utf-8"
)
)
payload = json.loads(raw_body.decode("utf-8"))
except (
UnicodeDecodeError,
json.JSONDecodeError,
) as error:
raise HTTPException(
status_code=(
status.HTTP_400_BAD_REQUEST
),
detail=(
"Webhook payload nie je "
"platný JSON"
),
status_code=(status.HTTP_400_BAD_REQUEST),
detail=("Webhook payload nie je " "platný JSON"),
) from error
if not isinstance(
@ -788,97 +647,73 @@ async def gitea_webhook(
dict,
):
raise HTTPException(
status_code=(
status.HTTP_400_BAD_REQUEST
),
detail=(
"Webhook payload musí byť "
"JSON objekt"
),
status_code=(status.HTTP_400_BAD_REQUEST),
detail=("Webhook payload musí byť " "JSON objekt"),
)
if not x_gitea_event:
raise HTTPException(
status_code=(
status.HTTP_400_BAD_REQUEST
),
detail=(
"Chýba hlavička "
"X-Gitea-Event"
),
status_code=(status.HTTP_400_BAD_REQUEST),
detail=("Chýba hlavička " "X-Gitea-Event"),
)
if (
x_gitea_event.casefold()
!= "push"
):
if x_gitea_event.casefold() != "push":
return JSONResponse(
status_code=(
status.HTTP_202_ACCEPTED
),
status_code=(status.HTTP_202_ACCEPTED),
content={
"status": (
"ignored"
),
"reason": (
"unsupported_event"
),
"event": (
x_gitea_event
),
"status": ("ignored"),
"reason": ("unsupported_event"),
"event": (x_gitea_event),
},
)
repository_name = (
repository_name_from_payload(
payload
)
)
repository_name = repository_name_from_payload(payload)
if repository_name is None:
raise HTTPException(
status_code=(
status.HTTP_400_BAD_REQUEST
),
detail=(
"Webhook payload neobsahuje "
"repository.full_name"
),
status_code=(status.HTTP_400_BAD_REQUEST),
detail=("Webhook payload neobsahuje " "repository.full_name"),
)
expected_repository = (
configured_repository()
)
expected_repository = configured_repository()
if not same_repository(
repository_name,
expected_repository,
):
raise HTTPException(
status_code=(
status.HTTP_403_FORBIDDEN
),
detail=(
"Webhook patrí neočakávanému "
"repozitáru"
),
status_code=(status.HTTP_403_FORBIDDEN),
detail=("Webhook patrí neočakávanému " "repozitáru"),
)
ref = payload.get('ref')
if not isinstance(ref, str) or not ref.startswith('refs/heads/'):
raise HTTPException(status_code=400, detail='Push payload musí obsahovať ref vetvy')
if ref != f'refs/heads/{expected_branch()}':
return {'status': 'ignored', 'reason': 'unexpected_branch', 'event': x_gitea_event}
revision = payload.get('after')
if payload.get('deleted') is True or (isinstance(revision, str) and revision and set(revision) == {'0'}):
return {'status': 'ignored', 'reason': 'deleted_branch', 'event': x_gitea_event}
if revision is not None and (not isinstance(revision, str) or not re.fullmatch(r'[0-9a-fA-F]{40}|[0-9a-fA-F]{64}', revision)):
raise HTTPException(status_code=400, detail='Neplatná Git revízia v push payload')
delivery = request.headers.get('X-Gitea-Delivery')
if delivery is not None and (not delivery.strip() or len(delivery) > 200 or any(ord(c) < 32 for c in delivery)):
raise HTTPException(status_code=400, detail='Neplatné ID webhook doručenia')
ref = payload.get("ref")
if not isinstance(ref, str) or not ref.startswith("refs/heads/"):
raise HTTPException(status_code=400, detail="Push payload musí obsahovať ref vetvy")
if ref != f"refs/heads/{expected_branch()}":
return {"status": "ignored", "reason": "unexpected_branch", "event": x_gitea_event}
revision = payload.get("after")
if payload.get("deleted") is True or (
isinstance(revision, str) and revision and set(revision) == {"0"}
):
return {"status": "ignored", "reason": "deleted_branch", "event": x_gitea_event}
if revision is not None and (
not isinstance(revision, str)
or not re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", revision)
):
raise HTTPException(status_code=400, detail="Neplatná Git revízia v push payload")
delivery = request.headers.get("X-Gitea-Delivery")
if delivery is not None and (
not delivery.strip() or len(delivery) > 200 or any(ord(c) < 32 for c in delivery)
):
raise HTTPException(status_code=400, detail="Neplatné ID webhook doručenia")
return {
**enqueue_sync(request, pull_git=webhook_should_pull_git(), requested_revision=revision,
delivery_key=f'{repository_name.casefold()}:{delivery}' if delivery else None),
'event': x_gitea_event, 'repository': repository_name, 'verified_by': 'hmac_sha256',
**enqueue_sync(
request,
pull_git=webhook_should_pull_git(),
requested_revision=revision,
delivery_key=f"{repository_name.casefold()}:{delivery}" if delivery else None,
),
"event": x_gitea_event,
"repository": repository_name,
"verified_by": "hmac_sha256",
}

View File

@ -17,7 +17,6 @@ from fastapi.security import (
HTTPBearer,
)
MIN_SECRET_LENGTH = 32
SEARCH_API_KEY_HEADER = "X-API-Key"
@ -43,29 +42,20 @@ search_api_key_scheme = APIKeyHeader(
name=SEARCH_API_KEY_HEADER,
scheme_name="SearchApiKey",
auto_error=False,
description=(
"API kľúč pre vyhľadávanie v ZP Wiki "
"cez hlavičku X-API-Key."
),
description=("API kľúč pre vyhľadávanie v ZP Wiki " "cez hlavičku X-API-Key."),
)
search_bearer_scheme = HTTPBearer(
scheme_name="SearchBearer",
auto_error=False,
description=(
"Bearer token pre vyhľadávanie v ZP Wiki. "
"Používa hodnotu SEARCH_API_KEY."
),
description=("Bearer token pre vyhľadávanie v ZP Wiki. " "Používa hodnotu SEARCH_API_KEY."),
)
sync_api_key_scheme = APIKeyHeader(
name=SYNC_API_KEY_HEADER,
scheme_name="SyncApiKey",
auto_error=False,
description=(
"API kľúč pre manuálne spustenie "
"reindexovania."
),
description=("API kľúč pre manuálne spustenie " "reindexovania."),
)
@ -78,10 +68,7 @@ def required_environment_value(
).strip()
if not value:
raise RuntimeError(
"Chýba povinná environment "
f"premenná {name}"
)
raise RuntimeError("Chýba povinná environment " f"premenná {name}")
return value
@ -89,57 +76,39 @@ def required_environment_value(
def validate_secret(
name: str,
) -> str:
value = required_environment_value(
name
)
value = required_environment_value(name)
if len(value) < MIN_SECRET_LENGTH:
raise RuntimeError(
f"{name} musí mať aspoň "
f"{MIN_SECRET_LENGTH} znakov"
)
raise RuntimeError(f"{name} musí mať aspoň " f"{MIN_SECRET_LENGTH} znakov")
return value
def expected_gitea_repository() -> str:
value = required_environment_value(
"EXPECTED_GITEA_REPOSITORY"
)
value = required_environment_value("EXPECTED_GITEA_REPOSITORY")
parts = value.split(
"/"
)
parts = value.split("/")
if len(parts) != 2:
raise RuntimeError(
"EXPECTED_GITEA_REPOSITORY musí mať "
"tvar vlastník/repozitár"
)
raise RuntimeError("EXPECTED_GITEA_REPOSITORY musí mať " "tvar vlastník/repozitár")
owner, repository = parts
if (
not owner
or not repository
or any(
character.isspace()
for character in value
)
):
raise RuntimeError(
"EXPECTED_GITEA_REPOSITORY musí mať "
"tvar vlastník/repozitár"
)
if not owner or not repository or any(character.isspace() for character in value):
raise RuntimeError("EXPECTED_GITEA_REPOSITORY musí mať " "tvar vlastník/repozitár")
return value
def webhook_should_pull_git() -> bool:
value = os.getenv(
"WEBHOOK_PULL_GIT",
"false",
).strip().casefold()
value = (
os.getenv(
"WEBHOOK_PULL_GIT",
"false",
)
.strip()
.casefold()
)
if value in TRUE_ENV_VALUES:
return True
@ -147,39 +116,31 @@ def webhook_should_pull_git() -> bool:
if value in FALSE_ENV_VALUES:
return False
raise RuntimeError(
"WEBHOOK_PULL_GIT musí byť boolean "
"hodnota true/false"
)
raise RuntimeError("WEBHOOK_PULL_GIT musí byť boolean " "hodnota true/false")
def validate_security_configuration() -> None:
webhook_secret = validate_secret(
"WEBHOOK_SECRET"
)
webhook_secret = validate_secret("WEBHOOK_SECRET")
sync_api_key = validate_secret(
"SYNC_API_KEY"
)
sync_api_key = validate_secret("SYNC_API_KEY")
search_api_key = validate_secret(
"SEARCH_API_KEY"
)
search_api_key = validate_secret("SEARCH_API_KEY")
# Každá funkcia musí používať vlastný
# credential. Jeden uniknutý secret tak
# neposkytne prístup ku všetkým operáciám.
if len(
{
webhook_secret,
sync_api_key,
search_api_key,
}
) != 3:
if (
len(
{
webhook_secret,
sync_api_key,
search_api_key,
}
)
!= 3
):
raise RuntimeError(
"WEBHOOK_SECRET, SYNC_API_KEY "
"a SEARCH_API_KEY musia byť "
"navzájom rozdielne"
"WEBHOOK_SECRET, SYNC_API_KEY " "a SEARCH_API_KEY musia byť " "navzájom rozdielne"
)
expected_gitea_repository()
@ -191,90 +152,51 @@ def validate_security_configuration() -> None:
def require_search_api_key(
api_key: str | None = Security(
search_api_key_scheme
),
bearer: (
HTTPAuthorizationCredentials | None
) = Security(
search_bearer_scheme
),
api_key: str | None = Security(search_api_key_scheme),
bearer: HTTPAuthorizationCredentials | None = Security(search_bearer_scheme),
) -> None:
expected = validate_secret(
"SEARCH_API_KEY"
)
expected = validate_secret("SEARCH_API_KEY")
supplied_credentials: list[str] = []
if api_key:
supplied_credentials.append(
api_key
)
supplied_credentials.append(api_key)
if (
bearer is not None
and bearer.scheme.casefold()
== "bearer"
and bearer.credentials
):
supplied_credentials.append(
bearer.credentials
)
if bearer is not None and bearer.scheme.casefold() == "bearer" and bearer.credentials:
supplied_credentials.append(bearer.credentials)
valid = any(
hmac.compare_digest(
supplied,
expected,
)
for supplied
in supplied_credentials
for supplied in supplied_credentials
)
if not valid:
raise HTTPException(
status_code=(
status.HTTP_401_UNAUTHORIZED
),
detail=(
"Neplatný alebo chýbajúci "
"API kľúč"
),
status_code=(status.HTTP_401_UNAUTHORIZED),
detail=("Neplatný alebo chýbajúci " "API kľúč"),
headers={
"WWW-Authenticate": (
"Bearer"
),
"WWW-Authenticate": ("Bearer"),
},
)
def require_sync_api_key(
api_key: str | None = Security(
sync_api_key_scheme
),
api_key: str | None = Security(sync_api_key_scheme),
) -> None:
expected = validate_secret(
"SYNC_API_KEY"
)
expected = validate_secret("SYNC_API_KEY")
if (
not api_key
or not hmac.compare_digest(
api_key,
expected,
)
if not api_key or not hmac.compare_digest(
api_key,
expected,
):
raise HTTPException(
status_code=(
status.HTTP_401_UNAUTHORIZED
),
detail=(
"Neplatný alebo chýbajúci "
"API kľúč"
),
status_code=(status.HTTP_401_UNAUTHORIZED),
detail=("Neplatný alebo chýbajúci " "API kľúč"),
headers={
"WWW-Authenticate": (
"ApiKey"
),
"WWW-Authenticate": ("ApiKey"),
},
)
@ -287,18 +209,10 @@ def verify_gitea_signature(
if not signature:
return False
supplied = (
signature
.strip()
.casefold()
)
supplied = signature.strip().casefold()
if supplied.startswith(
"sha256="
):
supplied = supplied.removeprefix(
"sha256="
)
if supplied.startswith("sha256="):
supplied = supplied.removeprefix("sha256=")
if len(supplied) != 64:
return False
@ -313,9 +227,7 @@ def verify_gitea_signature(
return False
expected = hmac.new(
secret.encode(
"utf-8"
),
secret.encode("utf-8"),
raw_body,
hashlib.sha256,
).hexdigest()
@ -329,9 +241,7 @@ def verify_gitea_signature(
def repository_name_from_payload(
payload: dict[str, Any],
) -> str | None:
repository = payload.get(
"repository"
)
repository = payload.get("repository")
if not isinstance(
repository,
@ -347,9 +257,7 @@ def repository_name_from_payload(
# Samotné "name" nestačí, pretože
# rovnaký názov môže existovať pod
# rôznymi vlastníkmi.
value = repository.get(
"full_name"
)
value = repository.get("full_name")
if not isinstance(
value,

View File

@ -24,8 +24,10 @@ COPY deployment/requirements.txt ./deployment/requirements.txt
RUN python -m pip install --no-cache-dir \
-r deployment/requirements.txt
COPY app ./app
COPY tools ./tools
# Prepare immutable assets before application code, so code-only changes do
# not invalidate model preparation. The preparer only imports its own module.
COPY tools/__init__.py ./tools/__init__.py
COPY tools/runtime/__init__.py tools/runtime/runtime_assets.py ./tools/runtime/
COPY runtime_assets/ /opt/zp-assets/
ARG EMBEDDING_MODEL=intfloat/multilingual-e5-small
@ -37,6 +39,9 @@ RUN python -m tools.runtime.runtime_assets prepare --output /opt/zp-assets \
--model "$EMBEDDING_MODEL" --revision "$EMBEDDING_REVISION" \
--encoding "$CHUNK_TOKEN_ENCODING" --encoding "$RAG_LLM_TOKEN_ENCODING"
COPY app ./app
COPY tools ./tools
ENV ZP_RUNTIME_ASSETS=/opt/zp-assets
ENV ZP_ASSETS_REQUIRED=1
ENV HF_HUB_OFFLINE=1

Binary file not shown.

2396
experiments/catalog.json Normal file

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More