Upratanie modulov a archivácia historických evaluácií
This commit is contained in:
parent
3d0dfa154b
commit
507c247ea3
@ -13,3 +13,10 @@ docs/
|
|||||||
integrations/
|
integrations/
|
||||||
tools/git_runner/
|
tools/git_runner/
|
||||||
**/*.md
|
**/*.md
|
||||||
|
experiments/
|
||||||
|
test/
|
||||||
|
tools/**/tests/
|
||||||
|
tools/evaluation/
|
||||||
|
tools/rag/evaluation/
|
||||||
|
tools/graphrag/evaluation/
|
||||||
|
tools/claude/
|
||||||
|
|||||||
24
README.md
24
README.md
@ -62,5 +62,27 @@ v Docker sieti. Dočasná verejná URL vyžaduje zodpovedajúce OAuth nastavenie
|
|||||||
|
|
||||||
RAG, GraphRAG, kontrolovaná cesta odpovedí a pripojenie cez Claude sú funkčné.
|
RAG, GraphRAG, kontrolovaná cesta odpovedí a pripojenie cez Claude sú funkčné.
|
||||||
Systém obsahuje embeddingovú cache, kontroly dostupnosti a persistentnú
|
Systém obsahuje embeddingovú cache, kontroly dostupnosti a persistentnú
|
||||||
synchronizačnú frontu. Historické evaluácie sú uložené pri jednotlivých vetvách.
|
synchronizačnú frontu. Historické evaluácie sú oddelené od prevádzkového kódu.
|
||||||
Samostatný hlavný agent, CRZP a spracovanie plných textov prác sú ďalšie etapy.
|
Samostatný hlavný agent, CRZP a spracovanie plných textov prác sú ďalšie etapy.
|
||||||
|
|
||||||
|
## Organizácia projektu
|
||||||
|
|
||||||
|
- `app/` – HTTP API a zabezpečenie.
|
||||||
|
- `tools/rag/`, `tools/graphrag/`, `tools/knowledge/` – vyhľadávanie a indexovanie.
|
||||||
|
- `tools/answers/` – príprava podkladov, kontroly a zostavenie odpovede.
|
||||||
|
- `tools/retrieval/`, `tools/sync/`, `tools/runtime/` – spoločné pravidlá a prevádzka.
|
||||||
|
- `tools/evaluation/` a `tools/*/evaluation/` – offline evaluátory a diagnostika.
|
||||||
|
- `experiments/rag/`, `experiments/graphrag/` – vybrané historické vstupy a výsledky.
|
||||||
|
- `experiments/archive/` – komprimované vývojové pokusy; mapovanie a SHA-256 sú v `catalog.json`.
|
||||||
|
- `packages/claude/`, `deployment/`, `scripts/` – klienti, nasadenie a spúšťanie.
|
||||||
|
|
||||||
|
Prevádzkový image neobsahuje testy ani historické experimenty. Spúšťa sa cez
|
||||||
|
`bash scripts/start.sh`; voliteľné služby ovláda `bash scripts/compose.sh`.
|
||||||
|
Regresie spúšťa `.venv/bin/python -m pytest -q test tools/rag/tests tools/graphrag/tests`.
|
||||||
|
|
||||||
|
Integritu experimentov overí `python -m tools.evaluation.archive verify`.
|
||||||
|
Archív možno obnoviť bez prepisovania súborov pomocou
|
||||||
|
`python -m tools.evaluation.archive restore --destination /tmp/zp-experiments`.
|
||||||
|
Pôvodný kód experimentov určuje `source_commit` v katalógu a historické Git tagy.
|
||||||
|
Staré manifesty ani výsledky sa neprepisujú na odtlačky aktuálnej implementácie;
|
||||||
|
ich úspešnosť nie je novým meraním po reorganizácii.
|
||||||
|
|||||||
81
app/main.py
81
app/main.py
@ -19,7 +19,6 @@ from fastapi.exception_handlers import http_exception_handler, request_validatio
|
|||||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||||
from starlette.responses import JSONResponse
|
from starlette.responses import JSONResponse
|
||||||
|
|
||||||
|
|
||||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
if str(PROJECT_ROOT) not in sys.path:
|
if str(PROJECT_ROOT) not in sys.path:
|
||||||
@ -39,13 +38,10 @@ from tools.rag.common import DATA_DIR
|
|||||||
from tools.sync.sync_jobs import SyncJobs
|
from tools.sync.sync_jobs import SyncJobs
|
||||||
from tools.rag.rebuild_index import rebuild_index
|
from tools.rag.rebuild_index import rebuild_index
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
OPENWEBUI_ORIGIN = (
|
OPENWEBUI_ORIGIN = "https://ui.tukekemt.xyz"
|
||||||
"https://ui.tukekemt.xyz"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
@ -64,9 +60,7 @@ async def lifespan(
|
|||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Predhriatie embedding modelu "
|
"Predhriatie embedding modelu " "zlyhalo. API pokračuje bez " "úspešného warm-upu."
|
||||||
"zlyhalo. API pokračuje bez "
|
|
||||||
"úspešného warm-upu."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def run_sync(**kwargs):
|
def run_sync(**kwargs):
|
||||||
@ -99,23 +93,36 @@ app = FastAPI(
|
|||||||
|
|
||||||
@app.exception_handler(StarletteHTTPException)
|
@app.exception_handler(StarletteHTTPException)
|
||||||
async def retrieval_http_error(request: Request, error: StarletteHTTPException):
|
async def retrieval_http_error(request: Request, error: StarletteHTTPException):
|
||||||
if request.url.path not in {'/rag', '/search', '/graphrag'}:
|
if request.url.path not in {"/rag", "/search", "/graphrag"}:
|
||||||
return await http_exception_handler(request, error)
|
return await http_exception_handler(request, error)
|
||||||
reason = {400: 'invalid_query', 401: 'unauthorized', 403: 'forbidden',
|
reason = {
|
||||||
404: 'not_found', 405: 'method_not_allowed', 422: 'invalid_request',
|
400: "invalid_query",
|
||||||
503: 'retrieval_unavailable'}.get(error.status_code, 'internal_error')
|
401: "unauthorized",
|
||||||
return JSONResponse(status_code=error.status_code, headers=error.headers,
|
403: "forbidden",
|
||||||
content={**error_result(reason=reason), 'detail': error.detail})
|
404: "not_found",
|
||||||
|
405: "method_not_allowed",
|
||||||
|
422: "invalid_request",
|
||||||
|
503: "retrieval_unavailable",
|
||||||
|
}.get(error.status_code, "internal_error")
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=error.status_code,
|
||||||
|
headers=error.headers,
|
||||||
|
content={**error_result(reason=reason), "detail": error.detail},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.exception_handler(RequestValidationError)
|
@app.exception_handler(RequestValidationError)
|
||||||
async def retrieval_validation_error(request: Request, error: RequestValidationError):
|
async def retrieval_validation_error(request: Request, error: RequestValidationError):
|
||||||
if request.url.path not in {'/rag', '/search', '/graphrag'}:
|
if request.url.path not in {"/rag", "/search", "/graphrag"}:
|
||||||
return await request_validation_exception_handler(request, error)
|
return await request_validation_exception_handler(request, error)
|
||||||
# Retain useful validation locations/types without echoing submitted secrets.
|
# Retain useful validation locations/types without echoing submitted secrets.
|
||||||
details = [{'loc': row['loc'], 'type': row['type'], 'msg': row['msg']} for row in error.errors()]
|
details = [
|
||||||
return JSONResponse(status_code=422, content=jsonable_encoder({
|
{"loc": row["loc"], "type": row["type"], "msg": row["msg"]} for row in error.errors()
|
||||||
**error_result(reason='invalid_request'), 'detail': details}))
|
]
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=422,
|
||||||
|
content=jsonable_encoder({**error_result(reason="invalid_request"), "detail": details}),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
@ -149,9 +156,7 @@ app.add_middleware(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.middleware(
|
@app.middleware("http")
|
||||||
"http"
|
|
||||||
)
|
|
||||||
async def add_security_headers(
|
async def add_security_headers(
|
||||||
request: Request,
|
request: Request,
|
||||||
call_next,
|
call_next,
|
||||||
@ -159,33 +164,27 @@ async def add_security_headers(
|
|||||||
try:
|
try:
|
||||||
response = await call_next(request)
|
response = await call_next(request)
|
||||||
except Exception:
|
except Exception:
|
||||||
if request.url.path not in {'/rag', '/search', '/graphrag'}:
|
if request.url.path not in {"/rag", "/search", "/graphrag"}:
|
||||||
raise
|
raise
|
||||||
# Serialization failures must obey the same contract as backend errors.
|
# Serialization failures must obey the same contract as backend errors.
|
||||||
logger.exception('Neočakávaná chyba výstupu retrieval API.')
|
logger.exception("Neočakávaná chyba výstupu retrieval API.")
|
||||||
response = JSONResponse(status_code=500, content={
|
response = JSONResponse(
|
||||||
**error_result(reason='internal_error'),
|
status_code=500,
|
||||||
'detail': 'Nastala interná chyba servera.'})
|
content={
|
||||||
|
**error_result(reason="internal_error"),
|
||||||
|
"detail": "Nastala interná chyba servera.",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
response.headers[
|
response.headers["Cache-Control"] = "no-store"
|
||||||
"Cache-Control"
|
|
||||||
] = "no-store"
|
|
||||||
|
|
||||||
response.headers[
|
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||||
"X-Content-Type-Options"
|
|
||||||
] = "nosniff"
|
|
||||||
|
|
||||||
response.headers[
|
response.headers["Referrer-Policy"] = "no-referrer"
|
||||||
"Referrer-Policy"
|
|
||||||
] = "no-referrer"
|
|
||||||
|
|
||||||
response.headers[
|
response.headers["X-Frame-Options"] = "DENY"
|
||||||
"X-Frame-Options"
|
|
||||||
] = "DENY"
|
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
app.include_router(
|
app.include_router(router)
|
||||||
router
|
|
||||||
)
|
|
||||||
|
|||||||
547
app/routes.py
547
app/routes.py
@ -47,38 +47,33 @@ from tools.rag.search_utils import search_database
|
|||||||
from tools.sync.wiki_git import expected_branch
|
from tools.sync.wiki_git import expected_branch
|
||||||
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY
|
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY
|
||||||
from tools.runtime.tool_health import tools_readiness
|
from tools.runtime.tool_health import tools_readiness
|
||||||
from tools.retrieval.retrieval_contract import RetrievalResponse, RetrievalSource, RetrievalErrorResponse, error_reason
|
from tools.retrieval.retrieval_contract import (
|
||||||
|
RetrievalResponse,
|
||||||
|
RetrievalSource,
|
||||||
|
RetrievalErrorResponse,
|
||||||
|
error_reason,
|
||||||
|
)
|
||||||
from tools.answers.answer_delivery import deliver_answer
|
from tools.answers.answer_delivery import deliver_answer
|
||||||
from tools.answers.answer_errors import public_failure
|
from tools.answers.answer_errors import public_failure
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
MAX_WEBHOOK_BODY_BYTES = (
|
MAX_WEBHOOK_BODY_BYTES = 2 * 1024 * 1024
|
||||||
2 * 1024 * 1024
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
RETRIEVAL_UNAVAILABLE_DETAIL = (
|
RETRIEVAL_UNAVAILABLE_DETAIL = (
|
||||||
"Vyhľadávací index alebo embeddingový "
|
"Vyhľadávací index alebo embeddingový " "model momentálne nie je dostupný."
|
||||||
"model momentálne nie je dostupný."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
INVALID_QUERY_DETAIL = (
|
INVALID_QUERY_DETAIL = "Neplatný vyhľadávací dotaz."
|
||||||
"Neplatný vyhľadávací dotaz."
|
|
||||||
)
|
|
||||||
|
|
||||||
INTERNAL_ERROR_DETAIL = (
|
INTERNAL_ERROR_DETAIL = "Nastala interná chyba servera."
|
||||||
"Nastala interná chyba servera."
|
|
||||||
)
|
|
||||||
|
|
||||||
WEBHOOK_CONFIGURATION_DETAIL = (
|
WEBHOOK_CONFIGURATION_DETAIL = "Webhook nie je správne nakonfigurovaný."
|
||||||
"Webhook nie je správne nakonfigurovaný."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class StrictRequestModel(BaseModel):
|
class StrictRequestModel(BaseModel):
|
||||||
@ -88,9 +83,7 @@ class StrictRequestModel(BaseModel):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class SearchRequest(
|
class SearchRequest(StrictRequestModel):
|
||||||
StrictRequestModel
|
|
||||||
):
|
|
||||||
query: str = Field(
|
query: str = Field(
|
||||||
...,
|
...,
|
||||||
min_length=1,
|
min_length=1,
|
||||||
@ -112,28 +105,20 @@ class SearchRequest(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class RagRequest(
|
class RagRequest(StrictRequestModel):
|
||||||
StrictRequestModel
|
|
||||||
):
|
|
||||||
include_diagnostics: bool = False
|
include_diagnostics: bool = False
|
||||||
query: str = Field(
|
query: str = Field(
|
||||||
...,
|
...,
|
||||||
min_length=1,
|
min_length=1,
|
||||||
max_length=500,
|
max_length=500,
|
||||||
description=(
|
description=("Otázka alebo vyhľadávací dotaz " "používateľa nad ZP Wiki."),
|
||||||
"Otázka alebo vyhľadávací dotaz "
|
|
||||||
"používateľa nad ZP Wiki."
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
limit: int = Field(
|
limit: int = Field(
|
||||||
default=5,
|
default=5,
|
||||||
ge=1,
|
ge=1,
|
||||||
le=20,
|
le=20,
|
||||||
description=(
|
description=("Maximálny počet relevantných " "zdrojov pre RAG kontext."),
|
||||||
"Maximálny počet relevantných "
|
|
||||||
"zdrojov pre RAG kontext."
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
published_only: bool = Field(
|
published_only: bool = Field(
|
||||||
@ -156,14 +141,11 @@ class RagRequest(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class SyncRequest(
|
class SyncRequest(StrictRequestModel):
|
||||||
StrictRequestModel
|
|
||||||
):
|
|
||||||
pull_git: bool = Field(
|
pull_git: bool = Field(
|
||||||
default=False,
|
default=False,
|
||||||
description=(
|
description=(
|
||||||
"Pred reindexovaním stiahne očakávanú vetvu "
|
"Pred reindexovaním stiahne očakávanú vetvu " "a aktualizuje ju iba fast-forward."
|
||||||
"a aktualizuje ju iba fast-forward."
|
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -177,34 +159,43 @@ class GraphRequest(StrictRequestModel):
|
|||||||
|
|
||||||
class AnswerRequest(StrictRequestModel):
|
class AnswerRequest(StrictRequestModel):
|
||||||
query: str = Field(min_length=1, max_length=500)
|
query: str = Field(min_length=1, max_length=500)
|
||||||
engine: Literal['rag', 'graphrag'] = 'rag'
|
engine: Literal["rag", "graphrag"] = "rag"
|
||||||
limit: int = Field(default=5, ge=1, le=20)
|
limit: int = Field(default=5, ge=1, le=20)
|
||||||
include_diagnostics: bool = False
|
include_diagnostics: bool = False
|
||||||
|
|
||||||
|
|
||||||
@router.post('/answer', dependencies=[Depends(require_search_api_key)], include_in_schema=False)
|
@router.post("/answer", dependencies=[Depends(require_search_api_key)], include_in_schema=False)
|
||||||
def verified_answer(request: AnswerRequest) -> dict:
|
def verified_answer(request: AnswerRequest) -> dict:
|
||||||
try:
|
try:
|
||||||
result = deliver_answer(request.query, engine=request.engine, limit=request.limit,
|
result = deliver_answer(
|
||||||
include_diagnostics=request.include_diagnostics)
|
request.query,
|
||||||
|
engine=request.engine,
|
||||||
|
limit=request.limit,
|
||||||
|
include_diagnostics=request.include_diagnostics,
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception('Príprava overenej odpovede zlyhala.')
|
logger.exception("Príprava overenej odpovede zlyhala.")
|
||||||
return {'status': 'error', 'reason': 'answer_unavailable', 'accepted_for_delivery': False,
|
return {
|
||||||
'answer': 'Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.'}
|
"status": "error",
|
||||||
if (result.get('accepted_for_delivery') is not True or
|
"reason": "answer_unavailable",
|
||||||
result.get('status') not in {'ok', 'no_answer', 'ambiguous'}):
|
"accepted_for_delivery": False,
|
||||||
|
"answer": "Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.",
|
||||||
|
}
|
||||||
|
if result.get("accepted_for_delivery") is not True or result.get("status") not in {
|
||||||
|
"ok",
|
||||||
|
"no_answer",
|
||||||
|
"ambiguous",
|
||||||
|
}:
|
||||||
result = public_failure(result)
|
result = public_failure(result)
|
||||||
result={**result,'query':request.query.strip(),'engine':request.engine}
|
result = {**result, "query": request.query.strip(), "engine": request.engine}
|
||||||
if not request.include_diagnostics:
|
if not request.include_diagnostics:
|
||||||
result.pop('attempts', None)
|
result.pop("attempts", None)
|
||||||
result.pop('retrieval_context', None)
|
result.pop("retrieval_context", None)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
class HealthResponse(BaseModel):
|
class HealthResponse(BaseModel):
|
||||||
status: Literal[
|
status: Literal["ok"]
|
||||||
"ok"
|
|
||||||
]
|
|
||||||
|
|
||||||
ready: bool
|
ready: bool
|
||||||
|
|
||||||
@ -235,10 +226,13 @@ class RagSource(RetrievalSource):
|
|||||||
heading_paths: list[Any] | None = None
|
heading_paths: list[Any] | None = None
|
||||||
text: str | None = None
|
text: str | None = None
|
||||||
|
|
||||||
retrieval: dict[
|
retrieval: (
|
||||||
str,
|
dict[
|
||||||
Any,
|
str,
|
||||||
] | None = None
|
Any,
|
||||||
|
]
|
||||||
|
| None
|
||||||
|
) = None
|
||||||
|
|
||||||
|
|
||||||
class RagResponse(RetrievalResponse):
|
class RagResponse(RetrievalResponse):
|
||||||
@ -264,9 +258,7 @@ class RagResponse(RetrievalResponse):
|
|||||||
|
|
||||||
context: str
|
context: str
|
||||||
|
|
||||||
sources: list[
|
sources: list[RagSource]
|
||||||
RagSource
|
|
||||||
]
|
|
||||||
index_version: dict[str, Any] | None = None
|
index_version: dict[str, Any] | None = None
|
||||||
context_budget: dict[str, Any] | None = None
|
context_budget: dict[str, Any] | None = None
|
||||||
diagnostics: dict[str, Any] | None = None
|
diagnostics: dict[str, Any] | None = None
|
||||||
@ -285,14 +277,12 @@ class SearchResponse(RetrievalResponse):
|
|||||||
ge=0,
|
ge=0,
|
||||||
)
|
)
|
||||||
|
|
||||||
results: list[
|
results: list[dict[str, Any]]
|
||||||
dict[str, Any]
|
|
||||||
]
|
|
||||||
index_version: dict[str, Any] | None = None
|
index_version: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
|
||||||
class SyncAcceptedResponse(BaseModel):
|
class SyncAcceptedResponse(BaseModel):
|
||||||
status: Literal['accepted']
|
status: Literal["accepted"]
|
||||||
job_id: str
|
job_id: str
|
||||||
status_url: str
|
status_url: str
|
||||||
pull_git: bool
|
pull_git: bool
|
||||||
@ -300,12 +290,12 @@ class SyncAcceptedResponse(BaseModel):
|
|||||||
duplicate: bool
|
duplicate: bool
|
||||||
event: str | None = None
|
event: str | None = None
|
||||||
repository: str | None = None
|
repository: str | None = None
|
||||||
verified_by: Literal['hmac_sha256'] | None = None
|
verified_by: Literal["hmac_sha256"] | None = None
|
||||||
|
|
||||||
|
|
||||||
class WebhookIgnoredResponse(BaseModel):
|
class WebhookIgnoredResponse(BaseModel):
|
||||||
status: Literal['ignored']
|
status: Literal["ignored"]
|
||||||
reason: Literal['unsupported_event', 'unexpected_branch', 'deleted_branch']
|
reason: Literal["unsupported_event", "unexpected_branch", "deleted_branch"]
|
||||||
event: str
|
event: str
|
||||||
|
|
||||||
|
|
||||||
@ -313,12 +303,17 @@ def enqueue_sync(request: Request, **kwargs) -> dict:
|
|||||||
try:
|
try:
|
||||||
accepted = request.app.state.sync_jobs.enqueue(**kwargs)
|
accepted = request.app.state.sync_jobs.enqueue(**kwargs)
|
||||||
except (sqlite3.Error, OSError):
|
except (sqlite3.Error, OSError):
|
||||||
logger.exception('Prijatie synchronizácie do fronty zlyhalo.')
|
logger.exception("Prijatie synchronizácie do fronty zlyhalo.")
|
||||||
raise HTTPException(status_code=503, detail='Synchronizačná fronta nie je dostupná')
|
raise HTTPException(status_code=503, detail="Synchronizačná fronta nie je dostupná")
|
||||||
job = accepted['job']
|
job = accepted["job"]
|
||||||
return {'status': 'accepted', 'job_id': job['id'],
|
return {
|
||||||
'status_url': f"/sync/jobs/{job['id']}", 'pull_git': job['pull_git'],
|
"status": "accepted",
|
||||||
'coalesced': accepted['coalesced'], 'duplicate': accepted['duplicate']}
|
"job_id": job["id"],
|
||||||
|
"status_url": f"/sync/jobs/{job['id']}",
|
||||||
|
"pull_git": job["pull_git"],
|
||||||
|
"coalesced": accepted["coalesced"],
|
||||||
|
"duplicate": accepted["duplicate"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def raise_retrieval_error(
|
def raise_retrieval_error(
|
||||||
@ -334,127 +329,77 @@ def raise_retrieval_error(
|
|||||||
)
|
)
|
||||||
|
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
||||||
status.HTTP_400_BAD_REQUEST
|
detail=(INVALID_QUERY_DETAIL),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
INVALID_QUERY_DETAIL
|
|
||||||
),
|
|
||||||
) from error
|
) from error
|
||||||
|
|
||||||
if error_reason(error) == 'retrieval_unavailable':
|
if error_reason(error) == "retrieval_unavailable":
|
||||||
logger.exception(
|
logger.exception("Retrieval nie je dostupný.")
|
||||||
"Retrieval nie je dostupný."
|
|
||||||
)
|
|
||||||
|
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
|
||||||
status.HTTP_503_SERVICE_UNAVAILABLE
|
detail=(RETRIEVAL_UNAVAILABLE_DETAIL),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
RETRIEVAL_UNAVAILABLE_DETAIL
|
|
||||||
),
|
|
||||||
) from error
|
) from error
|
||||||
|
|
||||||
logger.exception(
|
logger.exception("Neočakávaná chyba retrieval API.")
|
||||||
"Neočakávaná chyba retrieval API."
|
|
||||||
)
|
|
||||||
|
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||||
status.HTTP_500_INTERNAL_SERVER_ERROR
|
detail=(INTERNAL_ERROR_DETAIL),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
INTERNAL_ERROR_DETAIL
|
|
||||||
),
|
|
||||||
) from error
|
) from error
|
||||||
|
|
||||||
|
|
||||||
def webhook_secret() -> str:
|
def webhook_secret() -> str:
|
||||||
try:
|
try:
|
||||||
return validate_secret(
|
return validate_secret("WEBHOOK_SECRET")
|
||||||
"WEBHOOK_SECRET"
|
|
||||||
)
|
|
||||||
|
|
||||||
except RuntimeError as error:
|
except RuntimeError as error:
|
||||||
logger.exception(
|
logger.exception("WEBHOOK_SECRET nie je " "správne nakonfigurovaný.")
|
||||||
"WEBHOOK_SECRET nie je "
|
|
||||||
"správne nakonfigurovaný."
|
|
||||||
)
|
|
||||||
|
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
|
||||||
status.HTTP_503_SERVICE_UNAVAILABLE
|
detail=(WEBHOOK_CONFIGURATION_DETAIL),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
WEBHOOK_CONFIGURATION_DETAIL
|
|
||||||
),
|
|
||||||
) from error
|
) from error
|
||||||
|
|
||||||
|
|
||||||
def configured_repository() -> str:
|
def configured_repository() -> str:
|
||||||
try:
|
try:
|
||||||
return (
|
return expected_gitea_repository()
|
||||||
expected_gitea_repository()
|
|
||||||
)
|
|
||||||
|
|
||||||
except RuntimeError as error:
|
except RuntimeError as error:
|
||||||
logger.exception(
|
logger.exception("EXPECTED_GITEA_REPOSITORY " "nie je správne nakonfigurovaný.")
|
||||||
"EXPECTED_GITEA_REPOSITORY "
|
|
||||||
"nie je správne nakonfigurovaný."
|
|
||||||
)
|
|
||||||
|
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
|
||||||
status.HTTP_503_SERVICE_UNAVAILABLE
|
detail=(WEBHOOK_CONFIGURATION_DETAIL),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
WEBHOOK_CONFIGURATION_DETAIL
|
|
||||||
),
|
|
||||||
) from error
|
) from error
|
||||||
|
|
||||||
|
|
||||||
async def read_webhook_body(
|
async def read_webhook_body(
|
||||||
request: Request,
|
request: Request,
|
||||||
) -> bytes:
|
) -> bytes:
|
||||||
content_length = request.headers.get(
|
content_length = request.headers.get("content-length")
|
||||||
"content-length"
|
|
||||||
)
|
|
||||||
|
|
||||||
if content_length:
|
if content_length:
|
||||||
try:
|
try:
|
||||||
declared_length = int(
|
declared_length = int(content_length)
|
||||||
content_length
|
|
||||||
)
|
|
||||||
|
|
||||||
except ValueError as error:
|
except ValueError as error:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
detail=(
|
detail=("Neplatná Content-Length " "hlavička"),
|
||||||
"Neplatná Content-Length "
|
|
||||||
"hlavička"
|
|
||||||
),
|
|
||||||
) from error
|
) from error
|
||||||
|
|
||||||
if declared_length < 0:
|
if declared_length < 0:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
detail=(
|
detail=("Neplatná Content-Length " "hlavička"),
|
||||||
"Neplatná Content-Length "
|
|
||||||
"hlavička"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if (
|
if declared_length > MAX_WEBHOOK_BODY_BYTES:
|
||||||
declared_length
|
|
||||||
> MAX_WEBHOOK_BODY_BYTES
|
|
||||||
):
|
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=413,
|
status_code=413,
|
||||||
detail=(
|
detail=("Webhook payload je " "príliš veľký"),
|
||||||
"Webhook payload je "
|
|
||||||
"príliš veľký"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
chunks: list[bytes] = []
|
chunks: list[bytes] = []
|
||||||
@ -462,29 +407,17 @@ async def read_webhook_body(
|
|||||||
total_size = 0
|
total_size = 0
|
||||||
|
|
||||||
async for chunk in request.stream():
|
async for chunk in request.stream():
|
||||||
total_size += len(
|
total_size += len(chunk)
|
||||||
chunk
|
|
||||||
)
|
|
||||||
|
|
||||||
if (
|
if total_size > MAX_WEBHOOK_BODY_BYTES:
|
||||||
total_size
|
|
||||||
> MAX_WEBHOOK_BODY_BYTES
|
|
||||||
):
|
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=413,
|
status_code=413,
|
||||||
detail=(
|
detail=("Webhook payload je " "príliš veľký"),
|
||||||
"Webhook payload je "
|
|
||||||
"príliš veľký"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
chunks.append(
|
chunks.append(chunk)
|
||||||
chunk
|
|
||||||
)
|
|
||||||
|
|
||||||
return b"".join(
|
return b"".join(chunks)
|
||||||
chunks
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
@ -493,13 +426,9 @@ async def read_webhook_body(
|
|||||||
response_model=HealthResponse,
|
response_model=HealthResponse,
|
||||||
)
|
)
|
||||||
def health(request: Request) -> dict[str, Any]:
|
def health(request: Request) -> dict[str, Any]:
|
||||||
database_exists = (
|
database_exists = DB_FILE.exists()
|
||||||
DB_FILE.exists()
|
|
||||||
)
|
|
||||||
|
|
||||||
zpwiki_exists = (
|
zpwiki_exists = ZPWIKI_ROOT.exists()
|
||||||
ZPWIKI_ROOT.exists()
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
validate_security_configuration()
|
validate_security_configuration()
|
||||||
@ -509,39 +438,30 @@ def health(request: Request) -> dict[str, Any]:
|
|||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
security_configured = False
|
security_configured = False
|
||||||
|
|
||||||
tools, version = tools_readiness(DB_FILE, security_configured=security_configured,
|
tools, version = tools_readiness(
|
||||||
embedding_ready=getattr(request.app.state, 'embedding_ready', False))
|
DB_FILE,
|
||||||
ready = tools['rag']['ready'] and tools['graphrag']['ready']
|
security_configured=security_configured,
|
||||||
|
embedding_ready=getattr(request.app.state, "embedding_ready", False),
|
||||||
|
)
|
||||||
|
ready = tools["rag"]["ready"] and tools["graphrag"]["ready"]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"ready": ready,
|
"ready": ready,
|
||||||
"index_version": version,
|
"index_version": version,
|
||||||
"tools": tools,
|
"tools": tools,
|
||||||
"database_exists": (
|
"database_exists": (database_exists),
|
||||||
database_exists
|
"search_engine": ("hybrid_fts5_embeddings"),
|
||||||
),
|
|
||||||
"search_engine": (
|
|
||||||
"hybrid_fts5_embeddings"
|
|
||||||
),
|
|
||||||
"rag_enabled": True,
|
"rag_enabled": True,
|
||||||
"zpwiki_exists": (
|
"zpwiki_exists": (zpwiki_exists),
|
||||||
zpwiki_exists
|
"security_configured": (security_configured),
|
||||||
),
|
|
||||||
"security_configured": (
|
|
||||||
security_configured
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/rag",
|
"/rag",
|
||||||
operation_id=(
|
operation_id=("retrieve_zpwiki_context"),
|
||||||
"retrieve_zpwiki_context"
|
summary=("Vyhľadaj informácie v ZP Wiki"),
|
||||||
),
|
|
||||||
summary=(
|
|
||||||
"Vyhľadaj informácie v ZP Wiki"
|
|
||||||
),
|
|
||||||
description=(
|
description=(
|
||||||
"Použi tento nástroj pri otázkach "
|
"Použi tento nástroj pri otázkach "
|
||||||
"o ZP Wiki, študentoch, autoroch, "
|
"o ZP Wiki, študentoch, autoroch, "
|
||||||
@ -553,48 +473,30 @@ def health(request: Request) -> dict[str, Any]:
|
|||||||
),
|
),
|
||||||
response_model=RagResponse,
|
response_model=RagResponse,
|
||||||
response_model_exclude_unset=True,
|
response_model_exclude_unset=True,
|
||||||
status_code=(
|
status_code=(status.HTTP_200_OK),
|
||||||
status.HTTP_200_OK
|
|
||||||
),
|
|
||||||
responses={
|
responses={
|
||||||
400: {
|
400: {
|
||||||
"model": RetrievalErrorResponse,
|
"model": RetrievalErrorResponse,
|
||||||
"description": (
|
"description": ("Neplatný dotaz."),
|
||||||
"Neplatný dotaz."
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
401: {
|
401: {
|
||||||
"model": RetrievalErrorResponse,
|
"model": RetrievalErrorResponse,
|
||||||
"description": (
|
"description": ("Chýbajúci alebo neplatný " "API kľúč."),
|
||||||
"Chýbajúci alebo neplatný "
|
|
||||||
"API kľúč."
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
422: {
|
422: {
|
||||||
"model": RetrievalErrorResponse,
|
"model": RetrievalErrorResponse,
|
||||||
"description": (
|
"description": ("Neplatná štruktúra requestu."),
|
||||||
"Neplatná štruktúra requestu."
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
500: {
|
500: {
|
||||||
"model": RetrievalErrorResponse,
|
"model": RetrievalErrorResponse,
|
||||||
"description": (
|
"description": ("Interná chyba servera."),
|
||||||
"Interná chyba servera."
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
503: {
|
503: {
|
||||||
"model": RetrievalErrorResponse,
|
"model": RetrievalErrorResponse,
|
||||||
"description": (
|
"description": ("Retrieval nie je momentálne " "dostupný."),
|
||||||
"Retrieval nie je momentálne "
|
|
||||||
"dostupný."
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
dependencies=[
|
dependencies=[Depends(require_search_api_key)],
|
||||||
Depends(
|
|
||||||
require_search_api_key
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
def rag(
|
def rag(
|
||||||
request: RagRequest,
|
request: RagRequest,
|
||||||
@ -605,39 +507,33 @@ def rag(
|
|||||||
DB_FILE,
|
DB_FILE,
|
||||||
request.query,
|
request.query,
|
||||||
limit=request.limit,
|
limit=request.limit,
|
||||||
published_only=(
|
published_only=(request.published_only),
|
||||||
request.published_only
|
max_per_document=(request.max_per_document),
|
||||||
),
|
|
||||||
max_per_document=(
|
|
||||||
request.max_per_document
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
http_request.app.state.embedding_ready = True
|
http_request.app.state.embedding_ready = True
|
||||||
return compact_rag_payload(result, include_diagnostics=request.include_diagnostics)
|
return compact_rag_payload(result, include_diagnostics=request.include_diagnostics)
|
||||||
|
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
raise_retrieval_error(
|
raise_retrieval_error(error)
|
||||||
error
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post('/graphrag', dependencies=[Depends(require_search_api_key)], include_in_schema=False)
|
@router.post("/graphrag", dependencies=[Depends(require_search_api_key)], include_in_schema=False)
|
||||||
def graph_retrieval(request: GraphRequest) -> dict[str, Any]:
|
def graph_retrieval(request: GraphRequest) -> dict[str, Any]:
|
||||||
"""Native GraphRAG Tool transport; the legacy OpenAPI schema stays stable."""
|
"""Native GraphRAG Tool transport; the legacy OpenAPI schema stays stable."""
|
||||||
try:
|
try:
|
||||||
return search_graph(request.query, limit=request.limit, max_entities=request.max_entities,
|
return search_graph(
|
||||||
published_only=request.published_only)
|
request.query,
|
||||||
|
limit=request.limit,
|
||||||
|
max_entities=request.max_entities,
|
||||||
|
published_only=request.published_only,
|
||||||
|
)
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
raise_retrieval_error(error)
|
raise_retrieval_error(error)
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/search",
|
"/search",
|
||||||
dependencies=[
|
dependencies=[Depends(require_search_api_key)],
|
||||||
Depends(
|
|
||||||
require_search_api_key
|
|
||||||
)
|
|
||||||
],
|
|
||||||
include_in_schema=False,
|
include_in_schema=False,
|
||||||
response_model=SearchResponse,
|
response_model=SearchResponse,
|
||||||
)
|
)
|
||||||
@ -650,48 +546,30 @@ def search(
|
|||||||
DB_FILE,
|
DB_FILE,
|
||||||
request.query,
|
request.query,
|
||||||
request.limit,
|
request.limit,
|
||||||
published_only=(
|
published_only=(request.published_only),
|
||||||
request.published_only
|
max_per_document=(request.max_per_document),
|
||||||
),
|
|
||||||
max_per_document=(
|
|
||||||
request.max_per_document
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
http_request.app.state.embedding_ready = True
|
http_request.app.state.embedding_ready = True
|
||||||
|
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
raise_retrieval_error(
|
raise_retrieval_error(error)
|
||||||
error
|
|
||||||
)
|
|
||||||
|
|
||||||
results = response[
|
results = response["results"]
|
||||||
"results"
|
|
||||||
]
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
**{key: response[key] for key in RetrievalResponse.model_fields if key in response},
|
**{key: response[key] for key in RetrievalResponse.model_fields if key in response},
|
||||||
"query": request.query,
|
"query": request.query,
|
||||||
"engine": response[
|
"engine": response["engine"],
|
||||||
"engine"
|
"strategies": response["strategies"],
|
||||||
],
|
"count": len(results),
|
||||||
"strategies": response[
|
|
||||||
"strategies"
|
|
||||||
],
|
|
||||||
"count": len(
|
|
||||||
results
|
|
||||||
),
|
|
||||||
"results": results,
|
"results": results,
|
||||||
**({'index_version': response['index_version']} if response.get('index_version') else {}),
|
**({"index_version": response["index_version"]} if response.get("index_version") else {}),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/sync",
|
"/sync",
|
||||||
dependencies=[
|
dependencies=[Depends(require_sync_api_key)],
|
||||||
Depends(
|
|
||||||
require_sync_api_key
|
|
||||||
)
|
|
||||||
],
|
|
||||||
include_in_schema=False,
|
include_in_schema=False,
|
||||||
response_model=SyncAcceptedResponse,
|
response_model=SyncAcceptedResponse,
|
||||||
status_code=202,
|
status_code=202,
|
||||||
@ -703,26 +581,25 @@ def sync(
|
|||||||
return enqueue_sync(http_request, pull_git=request.pull_git)
|
return enqueue_sync(http_request, pull_git=request.pull_git)
|
||||||
|
|
||||||
|
|
||||||
@router.get('/sync/status', dependencies=[Depends(require_sync_api_key)], include_in_schema=False)
|
@router.get("/sync/status", dependencies=[Depends(require_sync_api_key)], include_in_schema=False)
|
||||||
def sync_status(request: Request) -> dict:
|
def sync_status(request: Request) -> dict:
|
||||||
return request.app.state.sync_jobs.overview()
|
return request.app.state.sync_jobs.overview()
|
||||||
|
|
||||||
|
|
||||||
@router.get('/sync/jobs/{job_id}', dependencies=[Depends(require_sync_api_key)], include_in_schema=False)
|
@router.get(
|
||||||
|
"/sync/jobs/{job_id}", dependencies=[Depends(require_sync_api_key)], include_in_schema=False
|
||||||
|
)
|
||||||
def sync_job(job_id: str, request: Request) -> dict:
|
def sync_job(job_id: str, request: Request) -> dict:
|
||||||
job = request.app.state.sync_jobs.get(job_id)
|
job = request.app.state.sync_jobs.get(job_id)
|
||||||
if job is None:
|
if job is None:
|
||||||
raise HTTPException(status_code=404, detail='Synchronizačná úloha neexistuje')
|
raise HTTPException(status_code=404, detail="Synchronizačná úloha neexistuje")
|
||||||
return job
|
return job
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/webhook/gitea",
|
"/webhook/gitea",
|
||||||
status_code=202,
|
status_code=202,
|
||||||
response_model=(
|
response_model=(SyncAcceptedResponse | WebhookIgnoredResponse),
|
||||||
SyncAcceptedResponse
|
|
||||||
| WebhookIgnoredResponse
|
|
||||||
),
|
|
||||||
response_model_exclude_unset=True,
|
response_model_exclude_unset=True,
|
||||||
include_in_schema=False,
|
include_in_schema=False,
|
||||||
)
|
)
|
||||||
@ -736,15 +613,10 @@ async def gitea_webhook(
|
|||||||
default=None,
|
default=None,
|
||||||
alias="X-Gitea-Signature",
|
alias="X-Gitea-Signature",
|
||||||
),
|
),
|
||||||
) -> (
|
) -> dict[str, Any] | JSONResponse:
|
||||||
dict[str, Any]
|
|
||||||
| JSONResponse
|
|
||||||
):
|
|
||||||
secret = webhook_secret()
|
secret = webhook_secret()
|
||||||
|
|
||||||
raw_body = await read_webhook_body(
|
raw_body = await read_webhook_body(request)
|
||||||
request
|
|
||||||
)
|
|
||||||
|
|
||||||
# Gitea podpis overujeme nad presným raw
|
# Gitea podpis overujeme nad presným raw
|
||||||
# request body ešte pred JSON parsingom.
|
# request body ešte pred JSON parsingom.
|
||||||
@ -754,33 +626,20 @@ async def gitea_webhook(
|
|||||||
secret,
|
secret,
|
||||||
):
|
):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_401_UNAUTHORIZED),
|
||||||
status.HTTP_401_UNAUTHORIZED
|
detail=("Neplatný webhook podpis"),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
"Neplatný webhook podpis"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
payload = json.loads(
|
payload = json.loads(raw_body.decode("utf-8"))
|
||||||
raw_body.decode(
|
|
||||||
"utf-8"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
except (
|
except (
|
||||||
UnicodeDecodeError,
|
UnicodeDecodeError,
|
||||||
json.JSONDecodeError,
|
json.JSONDecodeError,
|
||||||
) as error:
|
) as error:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
||||||
status.HTTP_400_BAD_REQUEST
|
detail=("Webhook payload nie je " "platný JSON"),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
"Webhook payload nie je "
|
|
||||||
"platný JSON"
|
|
||||||
),
|
|
||||||
) from error
|
) from error
|
||||||
|
|
||||||
if not isinstance(
|
if not isinstance(
|
||||||
@ -788,97 +647,73 @@ async def gitea_webhook(
|
|||||||
dict,
|
dict,
|
||||||
):
|
):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
||||||
status.HTTP_400_BAD_REQUEST
|
detail=("Webhook payload musí byť " "JSON objekt"),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
"Webhook payload musí byť "
|
|
||||||
"JSON objekt"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if not x_gitea_event:
|
if not x_gitea_event:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
||||||
status.HTTP_400_BAD_REQUEST
|
detail=("Chýba hlavička " "X-Gitea-Event"),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
"Chýba hlavička "
|
|
||||||
"X-Gitea-Event"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if (
|
if x_gitea_event.casefold() != "push":
|
||||||
x_gitea_event.casefold()
|
|
||||||
!= "push"
|
|
||||||
):
|
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
status_code=(
|
status_code=(status.HTTP_202_ACCEPTED),
|
||||||
status.HTTP_202_ACCEPTED
|
|
||||||
),
|
|
||||||
content={
|
content={
|
||||||
"status": (
|
"status": ("ignored"),
|
||||||
"ignored"
|
"reason": ("unsupported_event"),
|
||||||
),
|
"event": (x_gitea_event),
|
||||||
"reason": (
|
|
||||||
"unsupported_event"
|
|
||||||
),
|
|
||||||
"event": (
|
|
||||||
x_gitea_event
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
repository_name = (
|
repository_name = repository_name_from_payload(payload)
|
||||||
repository_name_from_payload(
|
|
||||||
payload
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if repository_name is None:
|
if repository_name is None:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
||||||
status.HTTP_400_BAD_REQUEST
|
detail=("Webhook payload neobsahuje " "repository.full_name"),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
"Webhook payload neobsahuje "
|
|
||||||
"repository.full_name"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
expected_repository = (
|
expected_repository = configured_repository()
|
||||||
configured_repository()
|
|
||||||
)
|
|
||||||
|
|
||||||
if not same_repository(
|
if not same_repository(
|
||||||
repository_name,
|
repository_name,
|
||||||
expected_repository,
|
expected_repository,
|
||||||
):
|
):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_403_FORBIDDEN),
|
||||||
status.HTTP_403_FORBIDDEN
|
detail=("Webhook patrí neočakávanému " "repozitáru"),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
"Webhook patrí neočakávanému "
|
|
||||||
"repozitáru"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
ref = payload.get('ref')
|
ref = payload.get("ref")
|
||||||
if not isinstance(ref, str) or not ref.startswith('refs/heads/'):
|
if not isinstance(ref, str) or not ref.startswith("refs/heads/"):
|
||||||
raise HTTPException(status_code=400, detail='Push payload musí obsahovať ref vetvy')
|
raise HTTPException(status_code=400, detail="Push payload musí obsahovať ref vetvy")
|
||||||
if ref != f'refs/heads/{expected_branch()}':
|
if ref != f"refs/heads/{expected_branch()}":
|
||||||
return {'status': 'ignored', 'reason': 'unexpected_branch', 'event': x_gitea_event}
|
return {"status": "ignored", "reason": "unexpected_branch", "event": x_gitea_event}
|
||||||
revision = payload.get('after')
|
revision = payload.get("after")
|
||||||
if payload.get('deleted') is True or (isinstance(revision, str) and revision and set(revision) == {'0'}):
|
if payload.get("deleted") is True or (
|
||||||
return {'status': 'ignored', 'reason': 'deleted_branch', 'event': x_gitea_event}
|
isinstance(revision, str) and revision and set(revision) == {"0"}
|
||||||
if revision is not None and (not isinstance(revision, str) or not re.fullmatch(r'[0-9a-fA-F]{40}|[0-9a-fA-F]{64}', revision)):
|
):
|
||||||
raise HTTPException(status_code=400, detail='Neplatná Git revízia v push payload')
|
return {"status": "ignored", "reason": "deleted_branch", "event": x_gitea_event}
|
||||||
delivery = request.headers.get('X-Gitea-Delivery')
|
if revision is not None and (
|
||||||
if delivery is not None and (not delivery.strip() or len(delivery) > 200 or any(ord(c) < 32 for c in delivery)):
|
not isinstance(revision, str)
|
||||||
raise HTTPException(status_code=400, detail='Neplatné ID webhook doručenia')
|
or not re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", revision)
|
||||||
|
):
|
||||||
|
raise HTTPException(status_code=400, detail="Neplatná Git revízia v push payload")
|
||||||
|
delivery = request.headers.get("X-Gitea-Delivery")
|
||||||
|
if delivery is not None and (
|
||||||
|
not delivery.strip() or len(delivery) > 200 or any(ord(c) < 32 for c in delivery)
|
||||||
|
):
|
||||||
|
raise HTTPException(status_code=400, detail="Neplatné ID webhook doručenia")
|
||||||
return {
|
return {
|
||||||
**enqueue_sync(request, pull_git=webhook_should_pull_git(), requested_revision=revision,
|
**enqueue_sync(
|
||||||
delivery_key=f'{repository_name.casefold()}:{delivery}' if delivery else None),
|
request,
|
||||||
'event': x_gitea_event, 'repository': repository_name, 'verified_by': 'hmac_sha256',
|
pull_git=webhook_should_pull_git(),
|
||||||
|
requested_revision=revision,
|
||||||
|
delivery_key=f"{repository_name.casefold()}:{delivery}" if delivery else None,
|
||||||
|
),
|
||||||
|
"event": x_gitea_event,
|
||||||
|
"repository": repository_name,
|
||||||
|
"verified_by": "hmac_sha256",
|
||||||
}
|
}
|
||||||
|
|||||||
208
app/security.py
208
app/security.py
@ -17,7 +17,6 @@ from fastapi.security import (
|
|||||||
HTTPBearer,
|
HTTPBearer,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
MIN_SECRET_LENGTH = 32
|
MIN_SECRET_LENGTH = 32
|
||||||
|
|
||||||
SEARCH_API_KEY_HEADER = "X-API-Key"
|
SEARCH_API_KEY_HEADER = "X-API-Key"
|
||||||
@ -43,29 +42,20 @@ search_api_key_scheme = APIKeyHeader(
|
|||||||
name=SEARCH_API_KEY_HEADER,
|
name=SEARCH_API_KEY_HEADER,
|
||||||
scheme_name="SearchApiKey",
|
scheme_name="SearchApiKey",
|
||||||
auto_error=False,
|
auto_error=False,
|
||||||
description=(
|
description=("API kľúč pre vyhľadávanie v ZP Wiki " "cez hlavičku X-API-Key."),
|
||||||
"API kľúč pre vyhľadávanie v ZP Wiki "
|
|
||||||
"cez hlavičku X-API-Key."
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
search_bearer_scheme = HTTPBearer(
|
search_bearer_scheme = HTTPBearer(
|
||||||
scheme_name="SearchBearer",
|
scheme_name="SearchBearer",
|
||||||
auto_error=False,
|
auto_error=False,
|
||||||
description=(
|
description=("Bearer token pre vyhľadávanie v ZP Wiki. " "Používa hodnotu SEARCH_API_KEY."),
|
||||||
"Bearer token pre vyhľadávanie v ZP Wiki. "
|
|
||||||
"Používa hodnotu SEARCH_API_KEY."
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
sync_api_key_scheme = APIKeyHeader(
|
sync_api_key_scheme = APIKeyHeader(
|
||||||
name=SYNC_API_KEY_HEADER,
|
name=SYNC_API_KEY_HEADER,
|
||||||
scheme_name="SyncApiKey",
|
scheme_name="SyncApiKey",
|
||||||
auto_error=False,
|
auto_error=False,
|
||||||
description=(
|
description=("API kľúč pre manuálne spustenie " "reindexovania."),
|
||||||
"API kľúč pre manuálne spustenie "
|
|
||||||
"reindexovania."
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -78,10 +68,7 @@ def required_environment_value(
|
|||||||
).strip()
|
).strip()
|
||||||
|
|
||||||
if not value:
|
if not value:
|
||||||
raise RuntimeError(
|
raise RuntimeError("Chýba povinná environment " f"premenná {name}")
|
||||||
"Chýba povinná environment "
|
|
||||||
f"premenná {name}"
|
|
||||||
)
|
|
||||||
|
|
||||||
return value
|
return value
|
||||||
|
|
||||||
@ -89,57 +76,39 @@ def required_environment_value(
|
|||||||
def validate_secret(
|
def validate_secret(
|
||||||
name: str,
|
name: str,
|
||||||
) -> str:
|
) -> str:
|
||||||
value = required_environment_value(
|
value = required_environment_value(name)
|
||||||
name
|
|
||||||
)
|
|
||||||
|
|
||||||
if len(value) < MIN_SECRET_LENGTH:
|
if len(value) < MIN_SECRET_LENGTH:
|
||||||
raise RuntimeError(
|
raise RuntimeError(f"{name} musí mať aspoň " f"{MIN_SECRET_LENGTH} znakov")
|
||||||
f"{name} musí mať aspoň "
|
|
||||||
f"{MIN_SECRET_LENGTH} znakov"
|
|
||||||
)
|
|
||||||
|
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
def expected_gitea_repository() -> str:
|
def expected_gitea_repository() -> str:
|
||||||
value = required_environment_value(
|
value = required_environment_value("EXPECTED_GITEA_REPOSITORY")
|
||||||
"EXPECTED_GITEA_REPOSITORY"
|
|
||||||
)
|
|
||||||
|
|
||||||
parts = value.split(
|
parts = value.split("/")
|
||||||
"/"
|
|
||||||
)
|
|
||||||
|
|
||||||
if len(parts) != 2:
|
if len(parts) != 2:
|
||||||
raise RuntimeError(
|
raise RuntimeError("EXPECTED_GITEA_REPOSITORY musí mať " "tvar vlastník/repozitár")
|
||||||
"EXPECTED_GITEA_REPOSITORY musí mať "
|
|
||||||
"tvar vlastník/repozitár"
|
|
||||||
)
|
|
||||||
|
|
||||||
owner, repository = parts
|
owner, repository = parts
|
||||||
|
|
||||||
if (
|
if not owner or not repository or any(character.isspace() for character in value):
|
||||||
not owner
|
raise RuntimeError("EXPECTED_GITEA_REPOSITORY musí mať " "tvar vlastník/repozitár")
|
||||||
or not repository
|
|
||||||
or any(
|
|
||||||
character.isspace()
|
|
||||||
for character in value
|
|
||||||
)
|
|
||||||
):
|
|
||||||
raise RuntimeError(
|
|
||||||
"EXPECTED_GITEA_REPOSITORY musí mať "
|
|
||||||
"tvar vlastník/repozitár"
|
|
||||||
)
|
|
||||||
|
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
def webhook_should_pull_git() -> bool:
|
def webhook_should_pull_git() -> bool:
|
||||||
value = os.getenv(
|
value = (
|
||||||
"WEBHOOK_PULL_GIT",
|
os.getenv(
|
||||||
"false",
|
"WEBHOOK_PULL_GIT",
|
||||||
).strip().casefold()
|
"false",
|
||||||
|
)
|
||||||
|
.strip()
|
||||||
|
.casefold()
|
||||||
|
)
|
||||||
|
|
||||||
if value in TRUE_ENV_VALUES:
|
if value in TRUE_ENV_VALUES:
|
||||||
return True
|
return True
|
||||||
@ -147,39 +116,31 @@ def webhook_should_pull_git() -> bool:
|
|||||||
if value in FALSE_ENV_VALUES:
|
if value in FALSE_ENV_VALUES:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
raise RuntimeError(
|
raise RuntimeError("WEBHOOK_PULL_GIT musí byť boolean " "hodnota true/false")
|
||||||
"WEBHOOK_PULL_GIT musí byť boolean "
|
|
||||||
"hodnota true/false"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_security_configuration() -> None:
|
def validate_security_configuration() -> None:
|
||||||
webhook_secret = validate_secret(
|
webhook_secret = validate_secret("WEBHOOK_SECRET")
|
||||||
"WEBHOOK_SECRET"
|
|
||||||
)
|
|
||||||
|
|
||||||
sync_api_key = validate_secret(
|
sync_api_key = validate_secret("SYNC_API_KEY")
|
||||||
"SYNC_API_KEY"
|
|
||||||
)
|
|
||||||
|
|
||||||
search_api_key = validate_secret(
|
search_api_key = validate_secret("SEARCH_API_KEY")
|
||||||
"SEARCH_API_KEY"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Každá funkcia musí používať vlastný
|
# Každá funkcia musí používať vlastný
|
||||||
# credential. Jeden uniknutý secret tak
|
# credential. Jeden uniknutý secret tak
|
||||||
# neposkytne prístup ku všetkým operáciám.
|
# neposkytne prístup ku všetkým operáciám.
|
||||||
if len(
|
if (
|
||||||
{
|
len(
|
||||||
webhook_secret,
|
{
|
||||||
sync_api_key,
|
webhook_secret,
|
||||||
search_api_key,
|
sync_api_key,
|
||||||
}
|
search_api_key,
|
||||||
) != 3:
|
}
|
||||||
|
)
|
||||||
|
!= 3
|
||||||
|
):
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"WEBHOOK_SECRET, SYNC_API_KEY "
|
"WEBHOOK_SECRET, SYNC_API_KEY " "a SEARCH_API_KEY musia byť " "navzájom rozdielne"
|
||||||
"a SEARCH_API_KEY musia byť "
|
|
||||||
"navzájom rozdielne"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
expected_gitea_repository()
|
expected_gitea_repository()
|
||||||
@ -191,90 +152,51 @@ def validate_security_configuration() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def require_search_api_key(
|
def require_search_api_key(
|
||||||
api_key: str | None = Security(
|
api_key: str | None = Security(search_api_key_scheme),
|
||||||
search_api_key_scheme
|
bearer: HTTPAuthorizationCredentials | None = Security(search_bearer_scheme),
|
||||||
),
|
|
||||||
bearer: (
|
|
||||||
HTTPAuthorizationCredentials | None
|
|
||||||
) = Security(
|
|
||||||
search_bearer_scheme
|
|
||||||
),
|
|
||||||
) -> None:
|
) -> None:
|
||||||
expected = validate_secret(
|
expected = validate_secret("SEARCH_API_KEY")
|
||||||
"SEARCH_API_KEY"
|
|
||||||
)
|
|
||||||
|
|
||||||
supplied_credentials: list[str] = []
|
supplied_credentials: list[str] = []
|
||||||
|
|
||||||
if api_key:
|
if api_key:
|
||||||
supplied_credentials.append(
|
supplied_credentials.append(api_key)
|
||||||
api_key
|
|
||||||
)
|
|
||||||
|
|
||||||
if (
|
if bearer is not None and bearer.scheme.casefold() == "bearer" and bearer.credentials:
|
||||||
bearer is not None
|
supplied_credentials.append(bearer.credentials)
|
||||||
and bearer.scheme.casefold()
|
|
||||||
== "bearer"
|
|
||||||
and bearer.credentials
|
|
||||||
):
|
|
||||||
supplied_credentials.append(
|
|
||||||
bearer.credentials
|
|
||||||
)
|
|
||||||
|
|
||||||
valid = any(
|
valid = any(
|
||||||
hmac.compare_digest(
|
hmac.compare_digest(
|
||||||
supplied,
|
supplied,
|
||||||
expected,
|
expected,
|
||||||
)
|
)
|
||||||
for supplied
|
for supplied in supplied_credentials
|
||||||
in supplied_credentials
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if not valid:
|
if not valid:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_401_UNAUTHORIZED),
|
||||||
status.HTTP_401_UNAUTHORIZED
|
detail=("Neplatný alebo chýbajúci " "API kľúč"),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
"Neplatný alebo chýbajúci "
|
|
||||||
"API kľúč"
|
|
||||||
),
|
|
||||||
headers={
|
headers={
|
||||||
"WWW-Authenticate": (
|
"WWW-Authenticate": ("Bearer"),
|
||||||
"Bearer"
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def require_sync_api_key(
|
def require_sync_api_key(
|
||||||
api_key: str | None = Security(
|
api_key: str | None = Security(sync_api_key_scheme),
|
||||||
sync_api_key_scheme
|
|
||||||
),
|
|
||||||
) -> None:
|
) -> None:
|
||||||
expected = validate_secret(
|
expected = validate_secret("SYNC_API_KEY")
|
||||||
"SYNC_API_KEY"
|
|
||||||
)
|
|
||||||
|
|
||||||
if (
|
if not api_key or not hmac.compare_digest(
|
||||||
not api_key
|
api_key,
|
||||||
or not hmac.compare_digest(
|
expected,
|
||||||
api_key,
|
|
||||||
expected,
|
|
||||||
)
|
|
||||||
):
|
):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=(
|
status_code=(status.HTTP_401_UNAUTHORIZED),
|
||||||
status.HTTP_401_UNAUTHORIZED
|
detail=("Neplatný alebo chýbajúci " "API kľúč"),
|
||||||
),
|
|
||||||
detail=(
|
|
||||||
"Neplatný alebo chýbajúci "
|
|
||||||
"API kľúč"
|
|
||||||
),
|
|
||||||
headers={
|
headers={
|
||||||
"WWW-Authenticate": (
|
"WWW-Authenticate": ("ApiKey"),
|
||||||
"ApiKey"
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -287,18 +209,10 @@ def verify_gitea_signature(
|
|||||||
if not signature:
|
if not signature:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
supplied = (
|
supplied = signature.strip().casefold()
|
||||||
signature
|
|
||||||
.strip()
|
|
||||||
.casefold()
|
|
||||||
)
|
|
||||||
|
|
||||||
if supplied.startswith(
|
if supplied.startswith("sha256="):
|
||||||
"sha256="
|
supplied = supplied.removeprefix("sha256=")
|
||||||
):
|
|
||||||
supplied = supplied.removeprefix(
|
|
||||||
"sha256="
|
|
||||||
)
|
|
||||||
|
|
||||||
if len(supplied) != 64:
|
if len(supplied) != 64:
|
||||||
return False
|
return False
|
||||||
@ -313,9 +227,7 @@ def verify_gitea_signature(
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
expected = hmac.new(
|
expected = hmac.new(
|
||||||
secret.encode(
|
secret.encode("utf-8"),
|
||||||
"utf-8"
|
|
||||||
),
|
|
||||||
raw_body,
|
raw_body,
|
||||||
hashlib.sha256,
|
hashlib.sha256,
|
||||||
).hexdigest()
|
).hexdigest()
|
||||||
@ -329,9 +241,7 @@ def verify_gitea_signature(
|
|||||||
def repository_name_from_payload(
|
def repository_name_from_payload(
|
||||||
payload: dict[str, Any],
|
payload: dict[str, Any],
|
||||||
) -> str | None:
|
) -> str | None:
|
||||||
repository = payload.get(
|
repository = payload.get("repository")
|
||||||
"repository"
|
|
||||||
)
|
|
||||||
|
|
||||||
if not isinstance(
|
if not isinstance(
|
||||||
repository,
|
repository,
|
||||||
@ -347,9 +257,7 @@ def repository_name_from_payload(
|
|||||||
# Samotné "name" nestačí, pretože
|
# Samotné "name" nestačí, pretože
|
||||||
# rovnaký názov môže existovať pod
|
# rovnaký názov môže existovať pod
|
||||||
# rôznymi vlastníkmi.
|
# rôznymi vlastníkmi.
|
||||||
value = repository.get(
|
value = repository.get("full_name")
|
||||||
"full_name"
|
|
||||||
)
|
|
||||||
|
|
||||||
if not isinstance(
|
if not isinstance(
|
||||||
value,
|
value,
|
||||||
|
|||||||
@ -24,8 +24,10 @@ COPY deployment/requirements.txt ./deployment/requirements.txt
|
|||||||
RUN python -m pip install --no-cache-dir \
|
RUN python -m pip install --no-cache-dir \
|
||||||
-r deployment/requirements.txt
|
-r deployment/requirements.txt
|
||||||
|
|
||||||
COPY app ./app
|
# Prepare immutable assets before application code, so code-only changes do
|
||||||
COPY tools ./tools
|
# not invalidate model preparation. The preparer only imports its own module.
|
||||||
|
COPY tools/__init__.py ./tools/__init__.py
|
||||||
|
COPY tools/runtime/__init__.py tools/runtime/runtime_assets.py ./tools/runtime/
|
||||||
COPY runtime_assets/ /opt/zp-assets/
|
COPY runtime_assets/ /opt/zp-assets/
|
||||||
|
|
||||||
ARG EMBEDDING_MODEL=intfloat/multilingual-e5-small
|
ARG EMBEDDING_MODEL=intfloat/multilingual-e5-small
|
||||||
@ -37,6 +39,9 @@ RUN python -m tools.runtime.runtime_assets prepare --output /opt/zp-assets \
|
|||||||
--model "$EMBEDDING_MODEL" --revision "$EMBEDDING_REVISION" \
|
--model "$EMBEDDING_MODEL" --revision "$EMBEDDING_REVISION" \
|
||||||
--encoding "$CHUNK_TOKEN_ENCODING" --encoding "$RAG_LLM_TOKEN_ENCODING"
|
--encoding "$CHUNK_TOKEN_ENCODING" --encoding "$RAG_LLM_TOKEN_ENCODING"
|
||||||
|
|
||||||
|
COPY app ./app
|
||||||
|
COPY tools ./tools
|
||||||
|
|
||||||
ENV ZP_RUNTIME_ASSETS=/opt/zp-assets
|
ENV ZP_RUNTIME_ASSETS=/opt/zp-assets
|
||||||
ENV ZP_ASSETS_REQUIRED=1
|
ENV ZP_ASSETS_REQUIRED=1
|
||||||
ENV HF_HUB_OFFLINE=1
|
ENV HF_HUB_OFFLINE=1
|
||||||
|
|||||||
BIN
experiments/archive/development-runs.tar.gz
Normal file
BIN
experiments/archive/development-runs.tar.gz
Normal file
Binary file not shown.
2396
experiments/catalog.json
Normal file
2396
experiments/catalog.json
Normal file
File diff suppressed because it is too large
Load Diff
|
Can't render this file because it is too large.
|
|
Can't render this file because it is too large.
|
|
Can't render this file because it is too large.
|
|
Can't render this file because it is too large.
|
|
Can't render this file because it is too large.
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user