diff --git a/README.md b/README.md index f789d3c..337766f 100644 --- a/README.md +++ b/README.md @@ -709,6 +709,12 @@ môže klient ďalej používať RAG podľa `tools.rag.ready`. ## Hybridné vyhľadávanie +RAG aj GraphRAG používajú spoločné polia `status`, `reason`, `sources`, +`index_version`, `complete`, `truncated` a `truncation_reasons`. Pôvodný +RAG kontext aj grafové fakty a cesty zostávajú dostupné. Úplnosť vyhľadania +sa odlišuje od pravdivosti odpovede; pri RAG môže byť nezaručená (`null`). +[Výstupný kontrakt a spracovanie chýb](docs/vystupny_kontrakt.md). + Vyhľadávanie kombinuje: ```text diff --git a/app/main.py b/app/main.py index 3c3aad2..45389f0 100644 --- a/app/main.py +++ b/app/main.py @@ -13,6 +13,11 @@ from fastapi import ( from fastapi.middleware.cors import ( CORSMiddleware, ) +from fastapi.exceptions import RequestValidationError +from fastapi.encoders import jsonable_encoder +from fastapi.exception_handlers import http_exception_handler, request_validation_exception_handler +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.responses import JSONResponse PROJECT_ROOT = Path(__file__).resolve().parents[1] @@ -29,6 +34,7 @@ from app.security import ( validate_security_configuration, ) from tools.rag.embedding_utils import embed_query +from tools.retrieval_contract import error_result logger = logging.getLogger(__name__) @@ -78,6 +84,27 @@ app = FastAPI( ) +@app.exception_handler(StarletteHTTPException) +async def retrieval_http_error(request: Request, error: StarletteHTTPException): + if request.url.path not in {'/rag', '/search'}: + return await http_exception_handler(request, error) + reason = {400: 'invalid_query', 401: 'unauthorized', 403: 'forbidden', + 404: 'not_found', 405: 'method_not_allowed', 422: 'invalid_request', + 503: 'retrieval_unavailable'}.get(error.status_code, 'internal_error') + return JSONResponse(status_code=error.status_code, headers=error.headers, + content={**error_result(reason=reason), 'detail': error.detail}) + + +@app.exception_handler(RequestValidationError) +async def retrieval_validation_error(request: Request, error: RequestValidationError): + if request.url.path not in {'/rag', '/search'}: + return await request_validation_exception_handler(request, error) + # Retain useful validation locations/types without echoing submitted secrets. + details = [{'loc': row['loc'], 'type': row['type'], 'msg': row['msg']} for row in error.errors()] + return JSONResponse(status_code=422, content=jsonable_encoder({ + **error_result(reason='invalid_request'), 'detail': details})) + + app.add_middleware( CORSMiddleware, allow_origins=[ @@ -116,9 +143,16 @@ async def add_security_headers( request: Request, call_next, ): - response = await call_next( - request - ) + try: + response = await call_next(request) + except Exception: + if request.url.path not in {'/rag', '/search'}: + raise + # Serialization failures must obey the same contract as backend errors. + logger.exception('Neočakávaná chyba výstupu retrieval API.') + response = JSONResponse(status_code=500, content={ + **error_result(reason='internal_error'), + 'detail': 'Nastala interná chyba servera.'}) response.headers[ "Cache-Control" diff --git a/app/routes.py b/app/routes.py index 6205fb0..a0bbccf 100644 --- a/app/routes.py +++ b/app/routes.py @@ -48,6 +48,7 @@ from tools.rag.rebuild_index import ( from tools.rag.search_utils import search_database from tools.publication import DEFAULT_PUBLISHED_ONLY from tools.tool_health import tools_readiness +from tools.retrieval_contract import RetrievalResponse, RetrievalSource, RetrievalErrorResponse logger = logging.getLogger(__name__) @@ -188,7 +189,7 @@ class HealthResponse(BaseModel): tools: dict[str, Any] -class RagSource(BaseModel): +class RagSource(RetrievalSource): model_config = ConfigDict( extra="allow", ) @@ -209,7 +210,7 @@ class RagSource(BaseModel): ] | None = None -class RagResponse(BaseModel): +class RagResponse(RetrievalResponse): model_config = ConfigDict( extra="forbid", ) @@ -238,7 +239,7 @@ class RagResponse(BaseModel): index_version: dict[str, Any] | None = None -class SearchResponse(BaseModel): +class SearchResponse(RetrievalResponse): query: str engine: str @@ -583,27 +584,32 @@ def health(request: Request) -> dict[str, Any]: ), responses={ 400: { + "model": RetrievalErrorResponse, "description": ( "Neplatný dotaz." ), }, 401: { + "model": RetrievalErrorResponse, "description": ( "Chýbajúci alebo neplatný " "API kľúč." ), }, 422: { + "model": RetrievalErrorResponse, "description": ( "Neplatná štruktúra requestu." ), }, 500: { + "model": RetrievalErrorResponse, "description": ( "Interná chyba servera." ), }, 503: { + "model": RetrievalErrorResponse, "description": ( "Retrieval nie je momentálne " "dostupný." @@ -679,6 +685,7 @@ def search( ] return { + **{key: response[key] for key in RetrievalResponse.model_fields if key in response}, "query": request.query, "engine": response[ "engine" diff --git a/docs/vystupny_kontrakt.md b/docs/vystupny_kontrakt.md new file mode 100644 index 0000000..8a82b78 --- /dev/null +++ b/docs/vystupny_kontrakt.md @@ -0,0 +1,129 @@ +# Spoločný výstupný kontrakt RAG a GraphRAG + +Kontrakt v1 zjednocuje polia pre hlavného agenta a budúce OpenWebUI nástroje. +Vyhľadávanie naďalej vracia svoje pôvodné dáta. Klasický RAG zachováva +`context`, `instructions`, `answer_format` a rozšírené zdroje; GraphRAG +zachováva `facts`, `paths`, `documents`, `aggregation` a rozpoznané entity. +Historické evaluačné súbory ani zmrazené manifesty sa neprepisujú. + +## Spoločné polia + +| Pole | Význam | +|---|---| +| `status` | `ok`, `no_answer`, `ambiguous` alebo `error` | +| `reason` | Stabilný strojový dôvod; pri úspechu `null` | +| `sources` | Zoznam zdrojov s `source_id`, `document_path`, `source_url`, `title` | +| `index_version` | Spoločná generácia indexov vrátane Git revízie; `null`, ak nie je známa | +| `complete` | `true`: úplné vyhľadanie v rozsahu nástroja; `false`: neúplné; `null`: úplnosť nemožno zaručiť | +| `truncated` | Nástroj preukázateľne narazil na limit výberu alebo výstupu | +| `truncation_reasons` | Konkrétne limity; prázdne pole, ak žiadny nebol zistený | + +`source_id` má tvar `S1`, `S2`, … v rámci jednej odpovede. URL a názov +môžu byť `null`, ak ich zdroj neposkytuje; nesmú sa domýšľať. RAG si v každom +zdroji ponecháva aj text, sekcie a retrieval metadáta. GraphRAG pridáva +`sources` popri pôvodnom `documents`, takže existujúci evaluator môže +ďalej používať pôvodné polia. Zdrojový dokument je identifikovaný cestou. + +`ok` znamená úspešné získanie dôkazov alebo výsledku agregácie. Neznamená, +že LLM už odpovedal ani že vie zo zdrojov zodpovedať každú časť otázky. +Nulová, korektne vykonaná agregácia môže mať `status=ok`, `count=0` a žiadne +zdroje. `no_answer` označuje nenájdenie požadovaných faktov alebo entity; +`ambiguous` označuje nejednoznačné rozpoznanie; `error` zlyhanie nástroja. + +## Význam úplnosti + +Úplnosť **nevyjadruje percento presnosti ani pravdivosť odpovede**. + +RAG je radený výber kandidátov. Ak sa preukáže orezanie, uvádza +`complete=false`; inak `null`, pretože nemôže zaručiť vyčerpávajúci zoznam +všetkých významovo relevantných informácií. Hlási `candidate_limit`, +`per_document_limit` a `result_limit`. Dodatočný kandidát slúži na detekciu +orezania a nezaraďuje sa do pôvodného radeného výberu. Rozšírenia kontextu +ďalej používajú svoje existujúce limity; `truncated=false` preto nie je +zárukou úplnosti kontextu ani pokrytia celej wiki. + +GraphRAG vyjadruje úplnosť v rámci podporovaných štruktúrovaných dotazov +a nakonfigurovaných grafových prechodov. Hlási `entity_limit`, +`evidence_limit`, `fact_limit` a `path_limit`. Pri prechádzaní grafu overí +aj jeden výsledok nad limitom, aby rozpoznal orezanie už na úrovni Cypher. +`complete=true` neznamená neobmedzený prechod celým grafom: všeobecné +prechody naďalej používajú pôvodnú povolenú dĺžku ciest. Pri nejednoznačnosti +alebo chybe je `complete=false`, aj keď `truncated=false`. + +`aggregation.complete` ostáva osobitným príznakom úplnosti **výpočtu**. +Príklad: presne napočítané 2 osoby, ale `limit=1` pre zobrazené dôkazy: + +```json +{ + "status": "ok", + "reason": null, + "aggregation": {"count": 2, "complete": true}, + "complete": false, + "truncated": true, + "truncation_reasons": ["fact_limit", "path_limit"] +} +``` + +Príklad je skrátený. Agent môže uviesť presný počet podľa agregácie, +ale nesmie tvrdiť, že zobrazený výpis obsahuje všetky dôkazy. + +## Nenájdenie a chyby + +Nenájdenie používa napríklad `no_matching_sources`, `no_matching_facts`, +`unknown_person`, `unknown_keyword` alebo `unknown_title`. Nejednoznačnosť +používa napríklad `ambiguous_person`. Zostávajú zachované aj podrobnejšie +grafové `reasons` a dôkazy zamietnutých kandidátov. + +API `/rag` a `/search` vracia pri chybách rovnakú obálku; pôvodné HTTP kódy +a `detail` zostávajú zachované. Dôvody zahŕňajú `invalid_query` (400), +`unauthorized` (401), `invalid_request` (422), `retrieval_unavailable` (503) +a `internal_error` (500). Validačné chyby nevracajú hodnoty zaslaných polí. + +```json +{ + "status": "error", + "reason": "retrieval_unavailable", + "sources": [], + "index_version": null, + "complete": false, + "truncated": false, + "truncation_reasons": [], + "detail": "Vyhľadávací index alebo embeddingový model momentálne nie je dostupný." +} +``` + +Po chybe sa neuvádza verzia dát, ktorú nástroj úspešne neprečítal. Chybová +obálka neobsahuje text interných výnimiek, heslá ani interné cesty. + +Python knižničné funkcie naďalej vyhadzujú výnimky, aby existujúci kód mohol +riešiť chyby pomocou `try/except`. Transportné rozhrania ich prevádzajú na +obálku cez spoločné `error_result()`: HTTP API, grafový CLI a RAG CLI v režime +`--json`. Chybový CLI skončí návratovým kódom 1. Pre budúci MCP/HTTP adaptér +grafu sa použije ten istý prevod; nový grafový HTTP endpoint tento krok nepridáva. + +## Použitie + +```bash +python -m tools.rag.search_db 'strojový preklad' --json --limit 5 +python -m tools.graphrag.graph_search 'Aký názov práce má Ján Holp?' --limit 5 +``` + +Spoločné Pydantic modely aj tvorba obálky sú v `tools/retrieval_contract.py`. +OpenAPI pre `/rag` používa spoločné polia aj model chybovej odpovede. + +## Overenie 25. 10. 2026 + +Kompletná sada projektu: **506 úspešných testov, 4 preskočené**. Nové testy +overujú spoločné polia, zachovanie rozšírených zdrojov RAG, detekciu limitov, +nejednoznačnosť, nenájdenie a chyby vrátane neplatného výstupu backendu. +Grafové testy používajú skutočný Neo4j v izolovanej transakcii s rollbackom. + +Na nasadenom API sa overili `/rag`, `/search`, chyby autentifikácie a validácie +aj OpenAPI schéma. Skutočný GraphRAG napočítal 15 osôb s prácou v roku 2024; +počet bol rovnaký pri limite 1 a 100. Pri limite 1 správne označil dôkazy +za orezané, pričom samotný počet zostal úplný. Overilo sa tiež nenájdenie +neexistujúcej osoby a JSON chybové výstupy oboch CLI s návratovým kódom 1. + +Nasadenie nevyžadovalo reindexáciu: verzia dát zostala nezmenená. Toto +overenie kontraktu nepredstavuje nové meranie presnosti LLM odpovedí. +Strojový záznam je v [vystupny_kontrakt_overenie.json](vystupny_kontrakt_overenie.json). diff --git a/docs/vystupny_kontrakt_overenie.json b/docs/vystupny_kontrakt_overenie.json new file mode 100644 index 0000000..fd6e81c --- /dev/null +++ b/docs/vystupny_kontrakt_overenie.json @@ -0,0 +1,81 @@ +{ + "checked_at_utc": "2026-10-25T15:26:47.218274+00:00", + "index_version": { + "version": "ff682d98134d406081040902f234c44f", + "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e", + "created_at_utc": "2026-10-25T15:06:03.279424+00:00", + "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363", + "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52", + "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9", + "embedding_model": "intfloat/multilingual-e5-small", + "publication_policy": "explicit-false-hidden-v1" + }, + "shared_fields": [ + "complete", + "index_version", + "reason", + "sources", + "status", + "truncated", + "truncation_reasons" + ], + "api": { + "/rag": { + "status": "ok", + "source_count": 1, + "complete": false, + "truncated": true, + "truncation_reasons": [ + "per_document_limit", + "result_limit" + ], + "unauthorized_contract": true, + "validation_error_contract": true + }, + "/search": { + "status": "ok", + "source_count": 2, + "complete": false, + "truncated": true, + "truncation_reasons": [ + "per_document_limit", + "result_limit" + ], + "unauthorized_contract": true, + "validation_error_contract": true + } + }, + "regression": { + "passed": 506, + "skipped": 4, + "failed": 0 + }, + "openapi_common_fields_and_errors": true, + "graphrag": { + "status": "ok", + "complete": false, + "truncated": true, + "truncation_reasons": [ + "fact_limit", + "path_limit" + ], + "exact_count": 15, + "count_complete": true, + "same_count_at_limit_1_and_100": true, + "no_answer_reason": "unknown_person" + }, + "cli_error_checks": { + "rag": { + "exit_code": 1, + "reason": "invalid_query", + "valid_contract": true + }, + "graphrag": { + "exit_code": 1, + "reason": "retrieval_unavailable", + "valid_contract": true + } + }, + "index_version_unchanged": true, + "scope": "Overenie výstupného kontraktu retrieval nástrojov; nejde o meranie presnosti LLM." +} diff --git a/test/test_retrieval_contract.py b/test/test_retrieval_contract.py new file mode 100644 index 0000000..68846c0 --- /dev/null +++ b/test/test_retrieval_contract.py @@ -0,0 +1,134 @@ +"""Contract tests use real SQLite retrieval and API boundaries, not an LLM.""" +import json + +import numpy as np +import pytest +from fastapi.testclient import TestClient + +import app.main as main +import app.routes as routes +import tools.rag.build_sqlite_index as indexer +import tools.rag.search_core as core +import tools.rag.search_utils as search +from tools.rag.rag_utils import build_rag_context +from tools.retrieval_contract import RetrievalResponse, error_result + + +@pytest.fixture +def index(tmp_path, monkeypatch): + documents = []; chunks = [] + for number in range(3): + path = f'pages/topics/item{number}/README.md' + doc = {'path': path, 'title': f'Contract {number}', 'published': True} + documents.append(doc) + for part in range(2): + chunks.append({'chunk_id': path+f'::{part}', 'document_path': path, + 'title': doc['title'], 'published': True, 'chunk_index': part, + 'text': f'spoločný výraz {number} úsek {part}', 'heading_paths': [], + 'tags': [], 'categories': []}) + for name, rows in [('documents.json', documents), ('chunks.json', chunks)]: + (tmp_path/name).write_text(json.dumps(rows)) + monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.ones((len(texts),4), dtype=np.float32)) + monkeypatch.setattr(core, 'embed_query', lambda *args, **kwargs: np.ones(4, dtype=np.float32)) + db = tmp_path/'zp_index.sqlite' + indexer.build_database(documents_file=tmp_path/'documents.json', chunks_file=tmp_path/'chunks.json', db_file=db) + monkeypatch.setattr(routes, 'DB_FILE', db) + return db + + +def test_rag_limits_are_reported_and_rich_sources_preserved(index): + raw = search.search_database(index, 'spoločný výraz', limit=2, max_per_document=1) + assert raw['status'] == 'ok' and raw['truncated'] and raw['complete'] is False + assert set(raw['truncation_reasons']) == {'result_limit','per_document_limit'} + assert len(raw['sources']) == len(raw['results']) == 2 + rag = build_rag_context(index, 'spoločný výraz', limit=2) + RetrievalResponse.model_validate(rag) + assert rag['truncated'] and rag['complete'] is False and rag['reason'] is None + assert rag['index_version'] is None + assert len(rag['sources']) == rag['source_count'] + assert 'retrieval' in rag['sources'][0] and 'text' in rag['sources'][0] + + +def test_candidate_cap_detected_by_extra_row(index, monkeypatch): + monkeypatch.setattr(search, 'MIN_CANDIDATES', 1) + monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1) + result = search.search_database(index, 'spoločný výraz', limit=1) + assert result['truncation_reasons'] == ['candidate_limit'] + assert result['complete'] is False and len(result['results']) == 1 + + +def test_ranked_rag_never_promises_exhaustive_answer(index): + result = build_rag_context(index, 'spoločný výraz', limit=20, max_per_document=0) + assert result['status']=='ok' and not result['truncated'] + assert result['complete'] is None + + +def test_empty_rag_result_has_reason_and_null_version(index): + result = build_rag_context(index, ' ') + assert result['status']=='no_answer' and result['sources']==[] + assert result['reason']=='empty_query' and result['index_version'] is None + + +@pytest.mark.parametrize('path', ['/rag','/search']) +def test_api_success_contains_common_fields(index, path): + with TestClient(main.app) as client: + response = client.post(path, json={'query':'spoločný výraz','limit':2}, + headers={'X-API-Key':'a'*64}) + assert response.status_code==200 + body = response.json() + assert set(RetrievalResponse.model_fields) <= body.keys() + RetrievalResponse.model_validate(body) + assert body['status']=='ok' and body['sources'] and body['complete'] is False + + +@pytest.mark.parametrize('path', ['/rag','/search']) +@pytest.mark.parametrize('code,reason,error', [ + (400,'invalid_query',ValueError('PRIVATE_ERROR')), + (503,'retrieval_unavailable',RuntimeError('PRIVATE_ERROR')), + (500,'internal_error',TypeError('PRIVATE_ERROR')), +]) +def test_api_errors_have_contract_and_preserve_http_status(index, monkeypatch, path, code, reason, error): + def fail(*args, **kwargs): + raise error + monkeypatch.setattr(routes, 'build_rag_context' if path=='/rag' else 'search_database', fail) + with TestClient(main.app) as client: + response = client.post(path, json={'query':'test'}, headers={'X-API-Key':'a'*64}) + assert response.status_code==code + body = response.json() + assert body['status']=='error' and body['reason']==reason and body['sources']==[] + assert body['complete'] is False and body['index_version'] is None + assert 'PRIVATE_ERROR' not in response.text + assert 'detail' in body + + +@pytest.mark.parametrize('path', ['/rag','/search']) +def test_auth_and_validation_errors_use_same_envelope(index, path): + with TestClient(main.app) as client: + unauthorized = client.post(path, json={'query':'test'}) + invalid = client.post(path, json={'query':'test','unknown':'PRIVATE_INPUT'}, + headers={'X-API-Key':'a'*64}) + assert unauthorized.status_code==401 and unauthorized.json()['reason']=='unauthorized' + assert invalid.status_code==422 and invalid.json()['reason']=='invalid_request' + assert 'PRIVATE_INPUT' not in invalid.text + for response in [unauthorized,invalid]: + assert set(RetrievalResponse.model_fields) <= response.json().keys() + + +def test_error_factory_does_not_invent_sources_or_active_version(): + result = error_result(RuntimeError('password=PRIVATE')) + assert result['status']=='error' and result['sources']==[] and result['index_version'] is None + assert not result['truncated'] and result['complete'] is False + assert 'PRIVATE' not in json.dumps(result) + + +@pytest.mark.parametrize('path', ['/rag','/search']) +def test_malformed_backend_output_still_has_safe_error_contract(index, monkeypatch, path): + monkeypatch.setattr(routes, 'build_rag_context' if path=='/rag' else 'search_database', + lambda *args, **kwargs: {'unexpected':'PRIVATE_BACKEND_OUTPUT'}) + with TestClient(main.app) as client: + response = client.post(path, json={'query':'test'}, headers={'X-API-Key':'a'*64}) + assert response.status_code==500 + assert set(RetrievalResponse.model_fields) <= response.json().keys() + assert response.json()['reason']=='internal_error' and response.json()['status']=='error' + assert 'PRIVATE_BACKEND_OUTPUT' not in response.text + assert response.headers['Cache-Control']=='no-store' diff --git a/tools/graphrag/graph_search.py b/tools/graphrag/graph_search.py index 0047a82..d0b4a1a 100644 --- a/tools/graphrag/graph_search.py +++ b/tools/graphrag/graph_search.py @@ -7,6 +7,7 @@ import unicodedata from typing import Any, Iterable from tools.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION from .publication import public_query +from tools.retrieval_contract import result_fields, source_references, error_result GRAPH_SCOPE = "zpwiki" @@ -2284,7 +2285,7 @@ LIMIT $limit DOCUMENT_SOURCES_QUERY = """ MATCH (document:Document) WHERE document.graph_scope = $scope AND document.path IN $paths -RETURN document.path AS document_path, document.source_url AS source_url +RETURN document.path AS document_path, document.source_url AS source_url, document.title AS title ORDER BY document_path """ @@ -2465,9 +2466,15 @@ def search_graph_with_session( blocked_reasons.append("unknown_title") constraints["unresolved_title"] = True blocked = bool(blocked_reasons) + fetch_capped = False def fetch(enabled: bool, cypher: str, **params: Any) -> list[dict[str, Any]]: - return run_rows(session, cypher, **common, **params) if enabled and not blocked else [] + nonlocal fetch_capped + if not enabled or blocked: + return [] + rows = run_rows(session, cypher, **{**common, 'limit': fetch_limit + 1}, **params) + fetch_capped = fetch_capped or len(rows) > fetch_limit + return rows[:fetch_limit] thesis_rows = fetch(thesis_specific, THESIS_FACTS_QUERY, thesis_ids=thesis_ids, years=constraints["years"], @@ -2515,6 +2522,7 @@ def search_graph_with_session( work_rows = [row for row in work_rows if row.get("year") in shared_years] paths: list[dict[str, Any]] = [] + path_capped = False for enabled, cypher, rows, kind, node_ids, types in ( (thesis_specific, THESIS_PATHS_QUERY, thesis_rows, "Thesis", thesis_ids, constraints["thesis_types"]), (work_specific, WORK_PATHS_QUERY, work_rows, "Work", work_ids, constraints["work_types"]), @@ -2529,13 +2537,14 @@ def search_graph_with_session( )) if not structured and resolved_entities and not blocked: start_ids = [str(row["element_id"]) for row in resolved_entities] - params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit, + params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit + 1, "published_only": published_only} if constraints["mentions_shared"] and len(start_ids) >= 2: paths.extend(run_rows(session, PAIR_PATH_QUERY, **params)) else: paths.extend(run_rows(session, NEIGHBORHOOD_QUERY, target_kinds=constraints["target_kinds"], **params)) + path_capped = len(paths) > limit paths = deduplicate_paths(paths) facts = build_facts( thesis_rows, work_rows, start_rows, keyword_rows, category_rows, @@ -2561,7 +2570,7 @@ def search_graph_with_session( if constraints["mentions_count"] and not blocked: matching = sorted({str(row["person"]) for row in thesis_rows + work_rows}) aggregation = {"count": len(matching), "people": matching, - "candidate_count": len(person_ids), "complete": not entity_limit_reached} + "candidate_count": len(person_ids), "complete": not (entity_limit_reached or fetch_capped)} # Sources of rejected candidates still document the comparison. Fetch their # actual stored URLs rather than manufacturing URLs from expected answers. documents = collect_documents(resolved_entities, paths, facts) @@ -2596,8 +2605,17 @@ def search_graph_with_session( reason = blocked_reasons[0] if blocked_reasons else ( "no_matching_facts" if status == "no_answer" else None ) + truncation_reasons = [] + for active, name in ((entity_limit_reached, 'entity_limit'), (fetch_capped, 'evidence_limit'), + (len(facts) > limit, 'fact_limit'), (path_capped or len(paths) > limit, 'path_limit')): + if active: + truncation_reasons.append(name) + titles = {row['document_path']: row.get('title') for row in sources} + citations = source_references([{**doc, 'title': titles.get(doc['document_path'])} for doc in documents]) return { - "status": status, "reason": reason, "reasons": blocked_reasons, + **result_fields(status=status, sources=citations, reason=reason, complete=True, + truncation_reasons=truncation_reasons), + "reasons": blocked_reasons, "query": clean_query, "strategies": strategies, "constraints": constraints, "resolved_entities": resolved_entities, "paths": paths[:limit], "facts": facts[:limit], "documents": documents, "aggregation": aggregation, @@ -2605,7 +2623,6 @@ def search_graph_with_session( "rejection_evidence": [row for row in facts if row.get("fact_type") in { "academic_year_comparison", "start_year_comparison" } and not positive_facts], - "truncated": entity_limit_reached or len(facts) > limit or len(paths) > limit, } @@ -2675,14 +2692,11 @@ def main() -> None: help='Výslovne zahrnie aj nepublikované dokumenty.') args = parser.parse_args() - result = search_graph( - args.query, - limit=args.limit, - max_entities=( - args.max_entities - ), - published_only=not args.include_unpublished, - ) + try: + result = search_graph(args.query, limit=args.limit, max_entities=args.max_entities, + published_only=not args.include_unpublished) + except Exception as error: + result = error_result(error) print( json.dumps( @@ -2691,6 +2705,8 @@ def main() -> None: indent=2, ) ) + if result['status'] == 'error': + raise SystemExit(1) if __name__ == "__main__": diff --git a/tools/graphrag/tests/test_graph_publication.py b/tools/graphrag/tests/test_graph_publication.py index 2797327..ae92edd 100644 --- a/tools/graphrag/tests/test_graph_publication.py +++ b/tools/graphrag/tests/test_graph_publication.py @@ -99,3 +99,44 @@ def test_shared_keyword_does_not_include_private_person(graph): assert public['status']=='no_answer' and not public['facts'] all_rows = gs.search_graph_with_session(tx, question, published_only=False) assert any(f['fact_type']=='shared_keyword' for f in all_rows['facts']) + + +def test_graph_contract_distinguishes_exact_count_from_trimmed_evidence(graph): + from tools.retrieval_contract import RetrievalResponse + tx, _ = graph + result = gs.search_graph_with_session(tx, 'Koľko osôb má prácu v roku 2024?', limit=1) + RetrievalResponse.model_validate(result) + assert result['status']=='ok' and result['aggregation']['count']==2 + assert result['aggregation']['complete'] is True + assert result['complete'] is False and result['truncated'] + assert 'fact_limit' in result['truncation_reasons'] + assert {r['document_path'] for r in result['sources']} == {r['document_path'] for r in result['documents']} + assert all(r['source_id'].startswith('S') for r in result['sources']) + assert result['index_version'] is None # Raw fixture session has no published snapshot. + + +def test_graph_traversal_reports_database_limit_not_just_python_slicing(graph): + tx, _ = graph + result = gs.search_graph_with_session(tx, 'Alena Nová', limit=1) + assert len(result['paths'])==1 and result['truncated'] and result['complete'] is False + assert 'path_limit' in result['truncation_reasons'] + full = gs.search_graph_with_session(tx, 'Alena Nová', limit=100) + assert full['complete'] is True and not full['truncated'] + + +def test_graph_no_answer_keeps_common_fields_and_reason(graph): + tx, _ = graph + result = gs.search_graph_with_session(tx, 'Aký názov práce má osoba Cyril Malý v roku 2024?') + assert result['status']=='no_answer' and result['reason']=='unknown_person' + assert result['sources']==[] and not result['truncated'] and result['index_version'] is None + + +def test_ambiguous_graph_response_cannot_claim_completeness(graph): + tx, scope = graph + tx.run('''CREATE (p:Person {id:$id, name:'Alena Nová', source_document:$path, graph_scope:$scope}) + CREATE (d:Document {path:$path, published:true, graph_scope:$scope}) + CREATE (p)-[:HAS_DOCUMENT]->(d)''', + id=scope+'duplicate', path=scope+'/duplicate.md', scope=scope).consume() + result = gs.search_graph_with_session(tx, 'Aký názov práce má osoba Alena Nová v roku 2024?') + assert result['status']=='ambiguous' and result['reason']=='ambiguous_person' + assert result['complete'] is False and not result['truncated'] diff --git a/tools/rag/rag_utils.py b/tools/rag/rag_utils.py index 596751f..d1317b8 100644 --- a/tools/rag/rag_utils.py +++ b/tools/rag/rag_utils.py @@ -1,6 +1,7 @@ from __future__ import annotations from tools.publication import DEFAULT_PUBLISHED_ONLY +from tools.retrieval_contract import result_fields import inspect import json @@ -752,6 +753,9 @@ def build_rag_context( context = build_context_text(sources) return { + **result_fields(status='ok' if sources else 'no_answer', sources=sources, + reason=None if sources else response.get('reason') or 'no_matching_sources', + truncation_reasons=response.get('truncation_reasons', [])), "query": query, "engine": response["engine"], "strategies": response["strategies"], diff --git a/tools/rag/search_db.py b/tools/rag/search_db.py index 8ff2e6f..9eda216 100644 --- a/tools/rag/search_db.py +++ b/tools/rag/search_db.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import json import sys from pathlib import Path @@ -15,6 +16,7 @@ if str(PROJECT_ROOT) not in sys.path: from tools.rag.common import DB_FILE from tools.rag.search_utils import search_database +from tools.retrieval_contract import error_result def print_results( @@ -155,6 +157,7 @@ def main() -> None: parser.add_argument("--include-unpublished", action="store_false", dest="published_only", help="Výslovne zahrnie aj nepublikované dokumenty.") + parser.add_argument('--json', action='store_true', help='Strojový výstup so spoločným kontraktom.') args = parser.parse_args() query = " ".join(args.query) @@ -167,14 +170,18 @@ def main() -> None: max_per_document=args.max_per_document, ) - except ( - FileNotFoundError, - RuntimeError, - ) as error: + except Exception as error: + if args.json: + sys.stdout.write(json.dumps(error_result(error), ensure_ascii=False, indent=2)+'\n') + raise SystemExit(1) from error raise SystemExit( str(error) ) from error + if args.json: + sys.stdout.write(json.dumps(response, ensure_ascii=False, indent=2)+'\n') + return + print_results( query, response, diff --git a/tools/rag/search_utils.py b/tools/rag/search_utils.py index ca1a7ca..1c9d256 100644 --- a/tools/rag/search_utils.py +++ b/tools/rag/search_utils.py @@ -6,6 +6,7 @@ import sqlite3 from pathlib import Path from typing import Any from tools.index_state import rag_snapshot +from tools.retrieval_contract import result_fields, source_references # Tieto importy sú zámerne verejné. # Zachovávajú spätnú kompatibilitu pre testy @@ -54,6 +55,8 @@ def search_database( published_only: bool = DEFAULT_PUBLISHED_ONLY, max_per_document: int = 3, ) -> dict[str, Any]: + if limit < 1 or max_per_document < 0: + raise ValueError('Limit musí byť kladný a max_per_document nezáporné.') if not db_file.exists(): raise FileNotFoundError( "Databáza neexistuje: " @@ -64,6 +67,7 @@ def search_database( if not clean_query: return { + **result_fields(status='no_answer', sources=[], reason='empty_query'), "engine": ( "hybrid_fts5_embeddings" ), @@ -108,6 +112,7 @@ def search_database( fts_candidates: list[ dict[str, Any] ] = [] + fts_capped = False used_strategies: list[ str @@ -120,13 +125,15 @@ def search_database( rows = run_fts_query( conn, match_query, - candidate_limit, + candidate_limit + 1, published_only, ) if not rows: continue + fts_capped = len(rows) > candidate_limit + rows = rows[:candidate_limit] for row in rows: row[ "strategy" @@ -158,11 +165,14 @@ def search_database( run_vector_query( conn, clean_query, - candidate_limit, + candidate_limit + 1, published_only, ) ) + vector_capped = len(vector_candidates) > candidate_limit + vector_candidates = vector_candidates[:candidate_limit] + vector_results = ( add_vector_metadata( conn, @@ -231,7 +241,22 @@ def search_database( ) ) + reasons = [] + if fts_candidates and fts_capped: + reasons.append('candidate_limit') + if vector_capped and (not used_strategies or used_strategies[0] not in {'all_terms', 'prefix_terms'}): + reasons.append('candidate_limit') + eligible = diversify_results(hybrid_results, max(1, len(hybrid_results)), max_per_document) + if len(eligible) < len(hybrid_results): + reasons.append('per_document_limit') + if len(eligible) > limit: + reasons.append('result_limit') + return { + **result_fields(status='ok' if final_results else 'no_answer', + sources=source_references(final_results), + reason=None if final_results else 'no_matching_sources', + truncation_reasons=reasons), "engine": ( "hybrid_fts5_embeddings" ), diff --git a/tools/rag/tests/test_rag.py b/tools/rag/tests/test_rag.py index 09e9ab0..fc39910 100644 --- a/tools/rag/tests/test_rag.py +++ b/tools/rag/tests/test_rag.py @@ -1132,6 +1132,7 @@ def test_rag_endpoint( "https://example.test/" "jan_holp" ), + "document_path": "pages/students/2016/jan_holp/README.md", }, ], } diff --git a/tools/rag/tests/test_search.py b/tools/rag/tests/test_search.py index 03c1c22..e822347 100644 --- a/tools/rag/tests/test_search.py +++ b/tools/rag/tests/test_search.py @@ -398,11 +398,10 @@ def test_empty_query_returns_empty_result( limit=5, ) - assert response == { - "engine": "hybrid_fts5_embeddings", - "strategies": [], - "results": [], - } + assert response['engine'] == 'hybrid_fts5_embeddings' + assert response['strategies'] == [] and response['results'] == [] + assert response['status'] == 'no_answer' and response['reason'] == 'empty_query' + assert response['sources'] == [] and response['index_version'] is None def test_missing_database_is_reported( diff --git a/tools/retrieval_contract.py b/tools/retrieval_contract.py new file mode 100644 index 0000000..e362ad5 --- /dev/null +++ b/tools/retrieval_contract.py @@ -0,0 +1,76 @@ +"""Shared machine-readable fields for RAG and GraphRAG tool responses. + +Completeness describes retrieval, not correctness or answerability. RAG is a +ranked selection and cannot promise exhaustive coverage of the wiki. +""" +from __future__ import annotations + +import sqlite3 +from typing import Any, Literal, Sequence + +from neo4j.exceptions import Neo4jError, ServiceUnavailable, SessionExpired +from pydantic import BaseModel, ConfigDict, Field + + +RetrievalStatus = Literal['ok', 'no_answer', 'ambiguous', 'error'] + + +class RetrievalSource(BaseModel): + model_config = ConfigDict(extra='allow') + + source_id: str + document_path: str + source_url: str | None = None + title: str | None = None + + +class RetrievalResponse(BaseModel): + status: RetrievalStatus = 'ok' + reason: str | None = None + sources: list[RetrievalSource] = Field(default_factory=list) + index_version: dict[str, Any] | None = None + complete: bool | None = Field(default=None, description=( + 'Úplnosť vyhľadania v rozsahu nástroja: true=úplné, false=neúplné, ' + 'null=nemožno zaručiť. Nevyjadruje pravdivosť odpovede.')) + truncated: bool = False + truncation_reasons: list[str] = Field(default_factory=list) + + +class RetrievalErrorResponse(RetrievalResponse): + status: Literal['error'] = 'error' + detail: Any = None + + +def source_references(rows: list[dict]) -> list[dict]: + """Stable citation shape; never invent a URL or a document title.""" + return [{ + 'source_id': f'S{number}', 'document_path': row['document_path'], + 'source_url': row.get('source_url'), 'title': row.get('title'), + } for number, row in enumerate(rows, 1)] + + +def result_fields(*, status: RetrievalStatus, sources: list[dict], reason: str | None = None, + complete: bool | None = None, truncation_reasons: Sequence[str] = (), + index_version: dict | None = None) -> dict: + reasons = list(dict.fromkeys(truncation_reasons)) + return { + 'status': status, 'reason': reason, 'sources': sources, + 'index_version': index_version, + 'complete': False if reasons or status in {'error', 'ambiguous'} else complete, + 'truncated': bool(reasons), 'truncation_reasons': reasons, + } + + +def error_reason(error: Exception) -> str: + if isinstance(error, ValueError): + return 'invalid_query' + if isinstance(error, (FileNotFoundError, sqlite3.Error, RuntimeError, + Neo4jError, ServiceUnavailable, SessionExpired)): + return 'retrieval_unavailable' + return 'internal_error' + + +def error_result(error: Exception | None = None, *, reason: str | None = None) -> dict: + # Raw exception messages can contain credentials, SQL or private paths. + return result_fields(status='error', sources=[], complete=False, + reason=reason or error_reason(error))