"""A bounded tool message with one copy of evidence and no retrieval diagnostics.""" from __future__ import annotations import copy import json import os import re from tools.rag.build_chunks import TokenCounter from tools.retrieval.text_budget import split_to_fit SOURCE_FIELDS = ('source_id', 'title', 'author', 'document_path', 'source_url', 'published', 'categories', 'tags', 'section') FIELDS = ('status', 'reason', 'query', 'engine', 'strategies', 'instructions', 'answer_format', 'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count', 'clarification', 'named_subjects', 'work_scopes', 'resolved_query', 'name_resolution') def compact_rag_payload(data: dict, *, max_tokens: int | None = None, include_diagnostics: bool = False) -> dict: from tools.rag.rag_utils import build_context_text limit = max_tokens if max_tokens is not None else int(os.getenv('RAG_LLM_MAX_TOKENS', '12000')) if limit < 256: raise ValueError('RAG_LLM_MAX_TOKENS musí byť aspoň 256') encoding = os.getenv('RAG_LLM_TOKEN_ENCODING', 'cl100k_base') counter = TokenCounter(encoding) if counter.encoding is None: raise RuntimeError('Kontextový rozpočet vyžaduje platný tiktoken tokenizér') base = {key: copy.deepcopy(data[key]) for key in FIELDS if key in data} base.setdefault('status', 'ok') base.setdefault('complete', None) base.setdefault('truncated', False) base.setdefault('truncation_reasons', []) input_sources = [{**source, 'source_id': source.get('source_id') or f'S{number}'} for number, source in enumerate(data.get('sources', []), 1)] references = [{key: source[key] for key in SOURCE_FIELDS if key in source} for source in input_sources] def stamp(payload): payload['context_budget'] = {'max_tokens': limit, 'used_tokens': 0, 'tokenizer': counter.name, 'tokens_are_estimate': True} for _ in range(8): used = counter.count(json.dumps(payload, ensure_ascii=False, separators=(',', ':'))) if payload['context_budget']['used_tokens'] == used: break payload['context_budget']['used_tokens'] = used return payload def render(sources, *, truncated=False, raw_context=None): payload = copy.deepcopy(base) if truncated: payload['complete'] = False payload['truncated'] = True payload['truncation_reasons'] = list(dict.fromkeys( [*payload['truncation_reasons'], 'llm_context_budget'])) payload['sources'] = [{key: source[key] for key in SOURCE_FIELDS if key in source} for source in sources] payload['source_count'] = len(sources) coverage = {key: payload.get(key) for key in ['complete', 'truncated', 'truncation_reasons', 'index_version']} context = raw_context if raw_context is not None else build_context_text(sources) payload['context'] = 'METADÁTA VYHĽADANIA\n' + json.dumps(coverage, ensure_ascii=False) + '\n\n' + context return stamp(payload) def fits(payload): return payload['context_budget']['used_tokens'] <= limit def whole_prefix(body,end): boundaries=[m.end() for m in re.finditer( r'\n\s*\n|(?=^[ \t]*(?:\d+[.)]|[-+*])\s+|^#{1,6}\s|^Sekcia:)',body,re.M) if m.end()<=end] return body[:max(boundaries,default=0)] def balanced_sources(sources): """Reserve evidence space for every requested person before expanding.""" suffix='\n[OBSAH ZDROJA OREZANÝ]' selected=[{**s,'text':suffix if s.get('text') else ''} for s in sources] if not fits(render(selected,truncated=True)): raise ValueError('Rozpočet nestačí na metadáta všetkých porovnávaných osôb') families={s.get('document_path','').removesuffix('README.md'):s['title'] for s in sources if s.get('title') in base['named_subjects'] and s.get('document_path')} groups={} for i,s in enumerate(sources): owner=next((p for path,p in families.items() if s.get('document_path','').startswith(path)),f'other:{i}') groups.setdefault(owner,[]).append(i) def used():return render(selected,truncated=True)['context_budget']['used_tokens'] def expand(index,budget): body=sources[index].get('text') or '' if not body:return ceiling=min(limit,used()+max(0,budget)) def candidate(text): replacement={**selected[index],'text':text+(suffix if len(text)1: result=balanced_sources(sources) elif not fits(result): selected = [] if not fits(render([], truncated=True)): raise ValueError('Tokenový rozpočet je menší ako povinné pokyny a metadáta') for source in sources: candidate = render([*selected, source], truncated=True) if fits(candidate): selected.append(source) continue body = source.get('text') or '' suffix = '\n[OBSAH ZDROJA OREZANÝ]' def body_fits(text): return fits(render([*selected, {**source, 'text': text + suffix}], truncated=True)) if body and body_fits(body[:1]): # Only the first fitting prefix is delivered; the complete # original remains available through the diagnostics option. low, high = 1, len(body) while low < high: middle = (low + high + 1) // 2 if body_fits(body[:middle]): low = middle else: high = middle - 1 selected.append({**source, 'text': whole_prefix(body,low) + suffix}) break result = render(selected, truncated=True) if not selected: result['status'] = 'no_answer' result['reason'] = 'llm_context_budget_exhausted' result = stamp(result) else: # Compatibility with older tool servers that expose only context + refs. context = data.get('context', '') result = render(references, raw_context=context) if not fits(result): parts = split_to_fit(context, lambda text: fits(render(references, truncated=True, raw_context=text))) result = render(references, truncated=True, raw_context=parts[0]) if not fits(result): raise ValueError('Povinné metadáta prekračujú kontextový rozpočet') if include_diagnostics: result['diagnostics'] = copy.deepcopy(data) return result