720 lines
19 KiB
Python
720 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import sqlite3
|
|
from typing import (
|
|
Any,
|
|
Literal,
|
|
NoReturn,
|
|
)
|
|
|
|
from fastapi import (
|
|
APIRouter,
|
|
Depends,
|
|
Header,
|
|
HTTPException,
|
|
Request,
|
|
status,
|
|
)
|
|
from fastapi.responses import JSONResponse
|
|
from pydantic import (
|
|
BaseModel,
|
|
ConfigDict,
|
|
Field,
|
|
)
|
|
|
|
from app.security import (
|
|
expected_gitea_repository,
|
|
repository_name_from_payload,
|
|
require_search_api_key,
|
|
require_sync_api_key,
|
|
same_repository,
|
|
validate_secret,
|
|
validate_security_configuration,
|
|
verify_gitea_signature,
|
|
webhook_should_pull_git,
|
|
)
|
|
from tools.rag.common import (
|
|
DB_FILE,
|
|
ZPWIKI_ROOT,
|
|
)
|
|
from tools.rag.rag_utils import build_rag_context
|
|
from tools.rag.llm_context import compact_rag_payload
|
|
from tools.graphrag.graph_search import search_graph, DEFAULT_LIMIT, DEFAULT_MAX_ENTITIES
|
|
from tools.rag.search_utils import search_database
|
|
from tools.sync.wiki_git import expected_branch
|
|
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY
|
|
from tools.runtime.tool_health import tools_readiness
|
|
from tools.retrieval.retrieval_contract import (
|
|
RetrievalResponse,
|
|
RetrievalSource,
|
|
RetrievalErrorResponse,
|
|
error_reason,
|
|
)
|
|
from tools.answers.answer_delivery import deliver_answer
|
|
from tools.answers.answer_errors import public_failure
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
MAX_WEBHOOK_BODY_BYTES = 2 * 1024 * 1024
|
|
|
|
|
|
RETRIEVAL_UNAVAILABLE_DETAIL = (
|
|
"Vyhľadávací index alebo embeddingový " "model momentálne nie je dostupný."
|
|
)
|
|
|
|
INVALID_QUERY_DETAIL = "Neplatný vyhľadávací dotaz."
|
|
|
|
INTERNAL_ERROR_DETAIL = "Nastala interná chyba servera."
|
|
|
|
WEBHOOK_CONFIGURATION_DETAIL = "Webhook nie je správne nakonfigurovaný."
|
|
|
|
|
|
class StrictRequestModel(BaseModel):
|
|
model_config = ConfigDict(
|
|
extra="forbid",
|
|
str_strip_whitespace=True,
|
|
)
|
|
|
|
|
|
class SearchRequest(StrictRequestModel):
|
|
query: str = Field(
|
|
...,
|
|
min_length=1,
|
|
max_length=500,
|
|
)
|
|
|
|
limit: int = Field(
|
|
default=10,
|
|
ge=1,
|
|
le=50,
|
|
)
|
|
|
|
published_only: bool = DEFAULT_PUBLISHED_ONLY
|
|
|
|
max_per_document: int = Field(
|
|
default=1,
|
|
ge=0,
|
|
le=10,
|
|
)
|
|
|
|
|
|
class RagRequest(StrictRequestModel):
|
|
include_diagnostics: bool = False
|
|
query: str = Field(
|
|
...,
|
|
min_length=1,
|
|
max_length=500,
|
|
description=("Otázka alebo vyhľadávací dotaz " "používateľa nad ZP Wiki."),
|
|
)
|
|
|
|
limit: int = Field(
|
|
default=5,
|
|
ge=1,
|
|
le=20,
|
|
description=("Maximálny počet relevantných " "zdrojov pre RAG kontext."),
|
|
)
|
|
|
|
published_only: bool = Field(
|
|
default=DEFAULT_PUBLISHED_ONLY,
|
|
description=(
|
|
"Ak je true, použijú sa iba "
|
|
"dokumenty okrem explicitného published=false; chýbajúca hodnota sa zahrnie."
|
|
),
|
|
)
|
|
|
|
max_per_document: int = Field(
|
|
default=1,
|
|
ge=0,
|
|
le=10,
|
|
description=(
|
|
"Maximálny počet chunkov z jedného "
|
|
"dokumentu. Hodnota 1 preferuje "
|
|
"rôzne dokumenty."
|
|
),
|
|
)
|
|
|
|
|
|
class SyncRequest(StrictRequestModel):
|
|
pull_git: bool = Field(
|
|
default=False,
|
|
description=(
|
|
"Pred reindexovaním stiahne očakávanú vetvu " "a aktualizuje ju iba fast-forward."
|
|
),
|
|
)
|
|
|
|
|
|
class GraphRequest(StrictRequestModel):
|
|
query: str = Field(min_length=1, max_length=500)
|
|
limit: int = Field(default=DEFAULT_LIMIT, ge=1, le=100)
|
|
max_entities: int = Field(default=DEFAULT_MAX_ENTITIES, ge=1, le=20)
|
|
published_only: bool = DEFAULT_PUBLISHED_ONLY
|
|
|
|
|
|
class AnswerRequest(StrictRequestModel):
|
|
query: str = Field(min_length=1, max_length=500)
|
|
engine: Literal["rag", "graphrag"] = "rag"
|
|
limit: int = Field(default=5, ge=1, le=20)
|
|
include_diagnostics: bool = False
|
|
|
|
|
|
@router.post("/answer", dependencies=[Depends(require_search_api_key)], include_in_schema=False)
|
|
def verified_answer(request: AnswerRequest) -> dict:
|
|
try:
|
|
result = deliver_answer(
|
|
request.query,
|
|
engine=request.engine,
|
|
limit=request.limit,
|
|
include_diagnostics=request.include_diagnostics,
|
|
)
|
|
except Exception:
|
|
logger.exception("Príprava overenej odpovede zlyhala.")
|
|
return {
|
|
"status": "error",
|
|
"reason": "answer_unavailable",
|
|
"accepted_for_delivery": False,
|
|
"answer": "Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.",
|
|
}
|
|
if result.get("accepted_for_delivery") is not True or result.get("status") not in {
|
|
"ok",
|
|
"no_answer",
|
|
"ambiguous",
|
|
}:
|
|
result = public_failure(result)
|
|
result = {**result, "query": request.query.strip(), "engine": request.engine}
|
|
if not request.include_diagnostics:
|
|
result.pop("attempts", None)
|
|
result.pop("retrieval_context", None)
|
|
return result
|
|
|
|
|
|
class HealthResponse(BaseModel):
|
|
status: Literal["ok"]
|
|
|
|
ready: bool
|
|
|
|
database_exists: bool
|
|
|
|
search_engine: str
|
|
rag_enabled: bool
|
|
|
|
zpwiki_exists: bool
|
|
|
|
security_configured: bool
|
|
index_version: dict[str, Any] | None = None
|
|
tools: dict[str, Any]
|
|
|
|
|
|
class RagSource(RetrievalSource):
|
|
model_config = ConfigDict(
|
|
extra="allow",
|
|
)
|
|
|
|
source_id: str
|
|
title: str
|
|
source_url: str
|
|
|
|
author: str | None = None
|
|
published: bool | None = None
|
|
|
|
heading_paths: list[Any] | None = None
|
|
text: str | None = None
|
|
|
|
retrieval: (
|
|
dict[
|
|
str,
|
|
Any,
|
|
]
|
|
| None
|
|
) = None
|
|
|
|
|
|
class RagResponse(RetrievalResponse):
|
|
model_config = ConfigDict(
|
|
extra="forbid",
|
|
)
|
|
|
|
query: str
|
|
engine: str
|
|
|
|
strategies: list[str]
|
|
|
|
source_count: int = Field(
|
|
ge=0,
|
|
)
|
|
|
|
instructions: list[str]
|
|
|
|
answer_format: dict[
|
|
str,
|
|
Any,
|
|
]
|
|
|
|
context: str
|
|
|
|
sources: list[RagSource]
|
|
index_version: dict[str, Any] | None = None
|
|
context_budget: dict[str, Any] | None = None
|
|
diagnostics: dict[str, Any] | None = None
|
|
resolved_query: str | None = None
|
|
name_resolution: dict[str, Any] | None = None
|
|
clarification: str | None = None
|
|
|
|
|
|
class SearchResponse(RetrievalResponse):
|
|
query: str
|
|
engine: str
|
|
|
|
strategies: list[str]
|
|
|
|
count: int = Field(
|
|
ge=0,
|
|
)
|
|
|
|
results: list[dict[str, Any]]
|
|
index_version: dict[str, Any] | None = None
|
|
|
|
|
|
class SyncAcceptedResponse(BaseModel):
|
|
status: Literal["accepted"]
|
|
job_id: str
|
|
status_url: str
|
|
pull_git: bool
|
|
coalesced: bool
|
|
duplicate: bool
|
|
event: str | None = None
|
|
repository: str | None = None
|
|
verified_by: Literal["hmac_sha256"] | None = None
|
|
|
|
|
|
class WebhookIgnoredResponse(BaseModel):
|
|
status: Literal["ignored"]
|
|
reason: Literal["unsupported_event", "unexpected_branch", "deleted_branch"]
|
|
event: str
|
|
|
|
|
|
def enqueue_sync(request: Request, **kwargs) -> dict:
|
|
try:
|
|
accepted = request.app.state.sync_jobs.enqueue(**kwargs)
|
|
except (sqlite3.Error, OSError):
|
|
logger.exception("Prijatie synchronizácie do fronty zlyhalo.")
|
|
raise HTTPException(status_code=503, detail="Synchronizačná fronta nie je dostupná")
|
|
job = accepted["job"]
|
|
return {
|
|
"status": "accepted",
|
|
"job_id": job["id"],
|
|
"status_url": f"/sync/jobs/{job['id']}",
|
|
"pull_git": job["pull_git"],
|
|
"coalesced": accepted["coalesced"],
|
|
"duplicate": accepted["duplicate"],
|
|
}
|
|
|
|
|
|
def raise_retrieval_error(
|
|
error: Exception,
|
|
) -> NoReturn:
|
|
if isinstance(
|
|
error,
|
|
ValueError,
|
|
):
|
|
logger.warning(
|
|
"Neplatný retrieval dotaz: %s",
|
|
error,
|
|
)
|
|
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
|
detail=(INVALID_QUERY_DETAIL),
|
|
) from error
|
|
|
|
if error_reason(error) == "retrieval_unavailable":
|
|
logger.exception("Retrieval nie je dostupný.")
|
|
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
|
|
detail=(RETRIEVAL_UNAVAILABLE_DETAIL),
|
|
) from error
|
|
|
|
logger.exception("Neočakávaná chyba retrieval API.")
|
|
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_500_INTERNAL_SERVER_ERROR),
|
|
detail=(INTERNAL_ERROR_DETAIL),
|
|
) from error
|
|
|
|
|
|
def webhook_secret() -> str:
|
|
try:
|
|
return validate_secret("WEBHOOK_SECRET")
|
|
|
|
except RuntimeError as error:
|
|
logger.exception("WEBHOOK_SECRET nie je " "správne nakonfigurovaný.")
|
|
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
|
|
detail=(WEBHOOK_CONFIGURATION_DETAIL),
|
|
) from error
|
|
|
|
|
|
def configured_repository() -> str:
|
|
try:
|
|
return expected_gitea_repository()
|
|
|
|
except RuntimeError as error:
|
|
logger.exception("EXPECTED_GITEA_REPOSITORY " "nie je správne nakonfigurovaný.")
|
|
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_503_SERVICE_UNAVAILABLE),
|
|
detail=(WEBHOOK_CONFIGURATION_DETAIL),
|
|
) from error
|
|
|
|
|
|
async def read_webhook_body(
|
|
request: Request,
|
|
) -> bytes:
|
|
content_length = request.headers.get("content-length")
|
|
|
|
if content_length:
|
|
try:
|
|
declared_length = int(content_length)
|
|
|
|
except ValueError as error:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=("Neplatná Content-Length " "hlavička"),
|
|
) from error
|
|
|
|
if declared_length < 0:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=("Neplatná Content-Length " "hlavička"),
|
|
)
|
|
|
|
if declared_length > MAX_WEBHOOK_BODY_BYTES:
|
|
raise HTTPException(
|
|
status_code=413,
|
|
detail=("Webhook payload je " "príliš veľký"),
|
|
)
|
|
|
|
chunks: list[bytes] = []
|
|
|
|
total_size = 0
|
|
|
|
async for chunk in request.stream():
|
|
total_size += len(chunk)
|
|
|
|
if total_size > MAX_WEBHOOK_BODY_BYTES:
|
|
raise HTTPException(
|
|
status_code=413,
|
|
detail=("Webhook payload je " "príliš veľký"),
|
|
)
|
|
|
|
chunks.append(chunk)
|
|
|
|
return b"".join(chunks)
|
|
|
|
|
|
@router.get(
|
|
"/health",
|
|
include_in_schema=False,
|
|
response_model=HealthResponse,
|
|
)
|
|
def health(request: Request) -> dict[str, Any]:
|
|
database_exists = DB_FILE.exists()
|
|
|
|
zpwiki_exists = ZPWIKI_ROOT.exists()
|
|
|
|
try:
|
|
validate_security_configuration()
|
|
|
|
security_configured = True
|
|
|
|
except RuntimeError:
|
|
security_configured = False
|
|
|
|
tools, version = tools_readiness(
|
|
DB_FILE,
|
|
security_configured=security_configured,
|
|
embedding_ready=getattr(request.app.state, "embedding_ready", False),
|
|
)
|
|
ready = tools["rag"]["ready"] and tools["graphrag"]["ready"]
|
|
|
|
return {
|
|
"status": "ok",
|
|
"ready": ready,
|
|
"index_version": version,
|
|
"tools": tools,
|
|
"database_exists": (database_exists),
|
|
"search_engine": ("hybrid_fts5_embeddings"),
|
|
"rag_enabled": True,
|
|
"zpwiki_exists": (zpwiki_exists),
|
|
"security_configured": (security_configured),
|
|
}
|
|
|
|
|
|
@router.post(
|
|
"/rag",
|
|
operation_id=("retrieve_zpwiki_context"),
|
|
summary=("Vyhľadaj informácie v ZP Wiki"),
|
|
description=(
|
|
"Použi tento nástroj pri otázkach "
|
|
"o ZP Wiki, študentoch, autoroch, "
|
|
"záverečných prácach, témach, rokoch, "
|
|
"projektoch alebo dokumentoch. "
|
|
"Nástroj vykoná hybridné FTS5 a "
|
|
"embeddingové vyhľadávanie a pripraví "
|
|
"zdrojovo podložený RAG kontext."
|
|
),
|
|
response_model=RagResponse,
|
|
response_model_exclude_unset=True,
|
|
status_code=(status.HTTP_200_OK),
|
|
responses={
|
|
400: {
|
|
"model": RetrievalErrorResponse,
|
|
"description": ("Neplatný dotaz."),
|
|
},
|
|
401: {
|
|
"model": RetrievalErrorResponse,
|
|
"description": ("Chýbajúci alebo neplatný " "API kľúč."),
|
|
},
|
|
422: {
|
|
"model": RetrievalErrorResponse,
|
|
"description": ("Neplatná štruktúra requestu."),
|
|
},
|
|
500: {
|
|
"model": RetrievalErrorResponse,
|
|
"description": ("Interná chyba servera."),
|
|
},
|
|
503: {
|
|
"model": RetrievalErrorResponse,
|
|
"description": ("Retrieval nie je momentálne " "dostupný."),
|
|
},
|
|
},
|
|
dependencies=[Depends(require_search_api_key)],
|
|
)
|
|
def rag(
|
|
request: RagRequest,
|
|
http_request: Request,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
result = build_rag_context(
|
|
DB_FILE,
|
|
request.query,
|
|
limit=request.limit,
|
|
published_only=(request.published_only),
|
|
max_per_document=(request.max_per_document),
|
|
)
|
|
http_request.app.state.embedding_ready = True
|
|
return compact_rag_payload(result, include_diagnostics=request.include_diagnostics)
|
|
|
|
except Exception as error:
|
|
raise_retrieval_error(error)
|
|
|
|
|
|
@router.post("/graphrag", dependencies=[Depends(require_search_api_key)], include_in_schema=False)
|
|
def graph_retrieval(request: GraphRequest) -> dict[str, Any]:
|
|
"""Native GraphRAG Tool transport; the legacy OpenAPI schema stays stable."""
|
|
try:
|
|
return search_graph(
|
|
request.query,
|
|
limit=request.limit,
|
|
max_entities=request.max_entities,
|
|
published_only=request.published_only,
|
|
)
|
|
except Exception as error:
|
|
raise_retrieval_error(error)
|
|
|
|
|
|
@router.post(
|
|
"/search",
|
|
dependencies=[Depends(require_search_api_key)],
|
|
include_in_schema=False,
|
|
response_model=SearchResponse,
|
|
)
|
|
def search(
|
|
request: SearchRequest,
|
|
http_request: Request,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
response = search_database(
|
|
DB_FILE,
|
|
request.query,
|
|
request.limit,
|
|
published_only=(request.published_only),
|
|
max_per_document=(request.max_per_document),
|
|
)
|
|
http_request.app.state.embedding_ready = True
|
|
|
|
except Exception as error:
|
|
raise_retrieval_error(error)
|
|
|
|
results = response["results"]
|
|
|
|
return {
|
|
**{key: response[key] for key in RetrievalResponse.model_fields if key in response},
|
|
"query": request.query,
|
|
"engine": response["engine"],
|
|
"strategies": response["strategies"],
|
|
"count": len(results),
|
|
"results": results,
|
|
**({"index_version": response["index_version"]} if response.get("index_version") else {}),
|
|
}
|
|
|
|
|
|
@router.post(
|
|
"/sync",
|
|
dependencies=[Depends(require_sync_api_key)],
|
|
include_in_schema=False,
|
|
response_model=SyncAcceptedResponse,
|
|
status_code=202,
|
|
)
|
|
def sync(
|
|
request: SyncRequest,
|
|
http_request: Request,
|
|
) -> dict[str, Any]:
|
|
return enqueue_sync(http_request, pull_git=request.pull_git)
|
|
|
|
|
|
@router.get("/sync/status", dependencies=[Depends(require_sync_api_key)], include_in_schema=False)
|
|
def sync_status(request: Request) -> dict:
|
|
return request.app.state.sync_jobs.overview()
|
|
|
|
|
|
@router.get(
|
|
"/sync/jobs/{job_id}", dependencies=[Depends(require_sync_api_key)], include_in_schema=False
|
|
)
|
|
def sync_job(job_id: str, request: Request) -> dict:
|
|
job = request.app.state.sync_jobs.get(job_id)
|
|
if job is None:
|
|
raise HTTPException(status_code=404, detail="Synchronizačná úloha neexistuje")
|
|
return job
|
|
|
|
|
|
@router.post(
|
|
"/webhook/gitea",
|
|
status_code=202,
|
|
response_model=(SyncAcceptedResponse | WebhookIgnoredResponse),
|
|
response_model_exclude_unset=True,
|
|
include_in_schema=False,
|
|
)
|
|
async def gitea_webhook(
|
|
request: Request,
|
|
x_gitea_event: str | None = Header(
|
|
default=None,
|
|
alias="X-Gitea-Event",
|
|
),
|
|
x_gitea_signature: str | None = Header(
|
|
default=None,
|
|
alias="X-Gitea-Signature",
|
|
),
|
|
) -> dict[str, Any] | JSONResponse:
|
|
secret = webhook_secret()
|
|
|
|
raw_body = await read_webhook_body(request)
|
|
|
|
# Gitea podpis overujeme nad presným raw
|
|
# request body ešte pred JSON parsingom.
|
|
if not verify_gitea_signature(
|
|
raw_body,
|
|
x_gitea_signature,
|
|
secret,
|
|
):
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_401_UNAUTHORIZED),
|
|
detail=("Neplatný webhook podpis"),
|
|
)
|
|
|
|
try:
|
|
payload = json.loads(raw_body.decode("utf-8"))
|
|
|
|
except (
|
|
UnicodeDecodeError,
|
|
json.JSONDecodeError,
|
|
) as error:
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
|
detail=("Webhook payload nie je " "platný JSON"),
|
|
) from error
|
|
|
|
if not isinstance(
|
|
payload,
|
|
dict,
|
|
):
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
|
detail=("Webhook payload musí byť " "JSON objekt"),
|
|
)
|
|
|
|
if not x_gitea_event:
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
|
detail=("Chýba hlavička " "X-Gitea-Event"),
|
|
)
|
|
|
|
if x_gitea_event.casefold() != "push":
|
|
return JSONResponse(
|
|
status_code=(status.HTTP_202_ACCEPTED),
|
|
content={
|
|
"status": ("ignored"),
|
|
"reason": ("unsupported_event"),
|
|
"event": (x_gitea_event),
|
|
},
|
|
)
|
|
|
|
repository_name = repository_name_from_payload(payload)
|
|
|
|
if repository_name is None:
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_400_BAD_REQUEST),
|
|
detail=("Webhook payload neobsahuje " "repository.full_name"),
|
|
)
|
|
|
|
expected_repository = configured_repository()
|
|
|
|
if not same_repository(
|
|
repository_name,
|
|
expected_repository,
|
|
):
|
|
raise HTTPException(
|
|
status_code=(status.HTTP_403_FORBIDDEN),
|
|
detail=("Webhook patrí neočakávanému " "repozitáru"),
|
|
)
|
|
|
|
ref = payload.get("ref")
|
|
if not isinstance(ref, str) or not ref.startswith("refs/heads/"):
|
|
raise HTTPException(status_code=400, detail="Push payload musí obsahovať ref vetvy")
|
|
if ref != f"refs/heads/{expected_branch()}":
|
|
return {"status": "ignored", "reason": "unexpected_branch", "event": x_gitea_event}
|
|
revision = payload.get("after")
|
|
if payload.get("deleted") is True or (
|
|
isinstance(revision, str) and revision and set(revision) == {"0"}
|
|
):
|
|
return {"status": "ignored", "reason": "deleted_branch", "event": x_gitea_event}
|
|
if revision is not None and (
|
|
not isinstance(revision, str)
|
|
or not re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", revision)
|
|
):
|
|
raise HTTPException(status_code=400, detail="Neplatná Git revízia v push payload")
|
|
delivery = request.headers.get("X-Gitea-Delivery")
|
|
if delivery is not None and (
|
|
not delivery.strip() or len(delivery) > 200 or any(ord(c) < 32 for c in delivery)
|
|
):
|
|
raise HTTPException(status_code=400, detail="Neplatné ID webhook doručenia")
|
|
return {
|
|
**enqueue_sync(
|
|
request,
|
|
pull_git=webhook_should_pull_git(),
|
|
requested_revision=revision,
|
|
delivery_key=f"{repository_name.casefold()}:{delivery}" if delivery else None,
|
|
),
|
|
"event": x_gitea_event,
|
|
"repository": repository_name,
|
|
"verified_by": "hmac_sha256",
|
|
}
|