368 lines
15 KiB
Python
368 lines
15 KiB
Python
"""Provider adapter for verified delivery; guards stay independent of the model vendor."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextvars
|
|
import json
|
|
import os
|
|
import urllib.parse
|
|
import urllib.request
|
|
import threading
|
|
import time
|
|
import hashlib
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
|
|
|
|
# -- time budget of one user answer ---------------------------------------------
|
|
# One answer may call the model several times. A single deadline bounds the
|
|
# whole answer, so the client never waits longer than the tunnel/host allows.
|
|
_deadline = contextvars.ContextVar("answer_deadline", default=None)
|
|
# Identity of the answer a call belongs to (its original deadline); a reserved
|
|
# window shortens the deadline but keeps the owner for fair admission.
|
|
_owner = contextvars.ContextVar("answer_owner", default=None)
|
|
# Shortest time a call needs to be useful; the gate never sends it with less.
|
|
_call_floor = contextvars.ContextVar("call_floor_seconds", default=0.0)
|
|
|
|
|
|
class DeadlineExceeded(RuntimeError):
|
|
"""The answer's time budget is spent; callers fall back to checked evidence."""
|
|
|
|
|
|
def answer_budget_seconds() -> float:
|
|
return float(os.getenv("ANSWER_TIME_BUDGET_SECONDS", "55"))
|
|
|
|
|
|
def start_deadline(seconds: float | None = None):
|
|
"""Start a budget unless an outer caller already set one; returns a reset token."""
|
|
if _deadline.get() is not None:
|
|
return None
|
|
value = time.monotonic() + (answer_budget_seconds() if seconds is None else seconds)
|
|
return _deadline.set(value), _owner.set(value)
|
|
|
|
|
|
def reset_deadline(token) -> None:
|
|
if token is not None:
|
|
_deadline.reset(token[0])
|
|
_owner.reset(token[1])
|
|
|
|
|
|
class call_floor:
|
|
"""Declare the shortest time the enclosed model calls need to finish.
|
|
|
|
A call queued until less time is left is not sent: the endpoint would keep
|
|
generating it after this answer gave up, slowing every other answer.
|
|
"""
|
|
|
|
def __init__(self, seconds: float):
|
|
self.seconds = seconds
|
|
self.token = None
|
|
|
|
def __enter__(self):
|
|
self.token = _call_floor.set(self.seconds)
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
_call_floor.reset(self.token)
|
|
return False
|
|
|
|
|
|
def remaining_seconds() -> float | None:
|
|
value = _deadline.get()
|
|
return None if value is None else value - time.monotonic()
|
|
|
|
|
|
def has_budget(seconds: float) -> bool:
|
|
left = remaining_seconds()
|
|
return left is None or left >= seconds
|
|
|
|
|
|
class reserved_budget:
|
|
"""Temporarily end the budget earlier, keeping `seconds` for a later step.
|
|
|
|
Evidence expansion runs inside this window so the final audit is never
|
|
starved by it. Nested windows only shorten the budget; outside an answer
|
|
budget (tests, offline builds) this is a no-op.
|
|
"""
|
|
|
|
def __init__(self, seconds: float):
|
|
self.seconds = seconds
|
|
self.token = None
|
|
|
|
def __enter__(self):
|
|
value = _deadline.get()
|
|
if value is not None:
|
|
self.token = _deadline.set(value - self.seconds)
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
if self.token is not None:
|
|
_deadline.reset(self.token)
|
|
return False
|
|
|
|
|
|
class _ModelGate:
|
|
"""Fair, deadline-ordered admission to the shared model endpoint.
|
|
|
|
The school endpoint serves two requests at a time and keeps generating
|
|
after a client disconnects. Sending more requests than it serves only
|
|
slows every answer and wastes generations nobody reads. The gate keeps at
|
|
most `slots` requests in flight. While several answers share the endpoint
|
|
each holds at most its fair share of slots (one of two), so a second user
|
|
is not starved until his budget ends; within that rule the nearest
|
|
deadline goes first. A request is never sent when its answer budget would
|
|
end before it could finish (MIN_START or the caller's call_floor).
|
|
"""
|
|
|
|
MIN_START = 3.0
|
|
|
|
def __init__(self):
|
|
self.condition = threading.Condition()
|
|
self.active = 0
|
|
self.by_owner = {}
|
|
self.waiting = []
|
|
self.sequence = 0
|
|
|
|
@staticmethod
|
|
def slots() -> int:
|
|
return int(os.getenv("ANSWER_LLM_SLOTS", "2"))
|
|
|
|
def _next(self, slots):
|
|
"""The waiting entry to admit now, or None."""
|
|
if self.active >= slots or not self.waiting:
|
|
return None
|
|
owners = {owner for owner, count in self.by_owner.items() if count} | {e[2] for e in self.waiting}
|
|
share = slots if len(owners) <= 1 else max(1, slots // len(owners))
|
|
for entry in sorted(self.waiting):
|
|
if self.by_owner.get(entry[2], 0) < share:
|
|
return entry
|
|
return None
|
|
|
|
def acquire(self, deadline, owner=None, floor=0.0):
|
|
slots = self.slots()
|
|
if slots <= 0:
|
|
return None
|
|
need = max(self.MIN_START, floor)
|
|
with self.condition:
|
|
self.sequence += 1
|
|
owner = owner if owner is not None else ("call", self.sequence)
|
|
entry = (float("inf") if deadline is None else deadline, self.sequence, owner)
|
|
self.waiting.append(entry)
|
|
try:
|
|
while self._next(slots) != entry:
|
|
wait = None
|
|
if deadline is not None:
|
|
wait = deadline - time.monotonic() - need
|
|
if wait <= 0:
|
|
raise DeadlineExceeded("Answer time budget exhausted while queued")
|
|
self.condition.wait(timeout=None if wait is None else min(wait, 1.0))
|
|
if deadline is not None and deadline - time.monotonic() < need:
|
|
raise DeadlineExceeded("Answer time budget exhausted while queued")
|
|
except BaseException:
|
|
self.waiting.remove(entry)
|
|
self.condition.notify_all()
|
|
raise
|
|
self.waiting.remove(entry)
|
|
self.active += 1
|
|
self.by_owner[owner] = self.by_owner.get(owner, 0) + 1
|
|
self.condition.notify_all()
|
|
return owner
|
|
|
|
def release(self, owner):
|
|
with self.condition:
|
|
self.active -= 1
|
|
self.by_owner[owner] -= 1
|
|
if not self.by_owner[owner]:
|
|
del self.by_owner[owner]
|
|
self.condition.notify_all()
|
|
|
|
|
|
_gate = _ModelGate()
|
|
|
|
|
|
def model_busy() -> bool:
|
|
"""Whether other model calls are running or queued right now.
|
|
|
|
Called between the steps of one answer (it then has no call in flight),
|
|
so a positive value means another answer shares the endpoint.
|
|
"""
|
|
with _gate.condition:
|
|
mine = _owner.get()
|
|
return any(owner != mine for owner, count in _gate.by_owner.items() if count) or any(
|
|
entry[2] != mine for entry in _gate.waiting)
|
|
|
|
|
|
def parallel_map(function, items, *, workers: int = 4) -> list:
|
|
"""Run independent model calls concurrently, preserving order and the deadline.
|
|
|
|
The school endpoint serves parallel requests in about the time of one.
|
|
Each worker runs in a copy of the caller's context, so the deadline applies.
|
|
Exceptions propagate exactly as in a sequential loop.
|
|
"""
|
|
items = list(items)
|
|
if len(items) <= 1:
|
|
return [function(item) for item in items]
|
|
contexts = [contextvars.copy_context() for _ in items]
|
|
with ThreadPoolExecutor(max_workers=min(workers, len(items))) as pool:
|
|
futures = [pool.submit(ctx.run, function, item) for ctx, item in zip(contexts, items)]
|
|
return [future.result() for future in futures]
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
return None
|
|
|
|
|
|
def model_configuration() -> dict:
|
|
provider = os.getenv("ANSWER_PROVIDER", "openai_compatible").strip()
|
|
model = os.getenv("ANSWER_MODEL", "").strip()
|
|
if provider == "anthropic":
|
|
key = (os.getenv("ANSWER_API_KEY") or os.getenv("ANTHROPIC_API_KEY", "")).strip()
|
|
endpoint = os.getenv("ANSWER_LLM_URL", "https://api.anthropic.com/v1/messages").strip()
|
|
elif provider == "openai_compatible":
|
|
legacy = os.getenv("OPENWEBUI_API_KEY", "").strip()
|
|
key = os.getenv("ANSWER_API_KEY", "").strip() or legacy
|
|
endpoint = os.getenv("ANSWER_LLM_URL", "https://ui.tukekemt.xyz/api/chat/completions" if legacy else "").strip()
|
|
model = model or ("model120-fast" if legacy else "")
|
|
else:
|
|
raise RuntimeError("Unsupported answer provider")
|
|
parsed = urllib.parse.urlsplit(endpoint)
|
|
if (not key or not model or any(c in key for c in "\r\n") or parsed.scheme != "https"
|
|
or not parsed.hostname or parsed.username is not None or parsed.password is not None
|
|
or parsed.fragment or parsed.query or any(c.isspace() for c in endpoint)):
|
|
raise RuntimeError("Answer model is not configured")
|
|
return {'provider':provider, 'model':model, 'endpoint':endpoint, 'key':key}
|
|
|
|
|
|
_status_lock = threading.Lock()
|
|
_status = {}
|
|
HEALTH_MAX_AGE_SECONDS = 900
|
|
|
|
|
|
def configuration_key(config: dict) -> str:
|
|
return hashlib.sha256(json.dumps(config,sort_keys=True).encode()).hexdigest()
|
|
|
|
|
|
def generation_status() -> dict:
|
|
try:
|
|
config = model_configuration()
|
|
except (RuntimeError,ValueError):
|
|
return {'ready':False,'status':'unavailable','reason':'model_not_configured','configured':False}
|
|
public={'configured':True,'provider':config['provider'],'model':config['model']}
|
|
with _status_lock:
|
|
status = dict(_status)
|
|
if status.get('configuration') != configuration_key(config) or time.time()-status.get('checked_at',0)>HEALTH_MAX_AGE_SECONDS:
|
|
return {**public,'ready':False,'status':'unknown','reason':'model_availability_not_verified'}
|
|
return {**public,'ready':status['success'],'status':'ready' if status['success'] else 'unavailable',
|
|
'reason':None if status['success'] else 'model_probe_failed',
|
|
'checked_at_utc':datetime.fromtimestamp(status['checked_at'],timezone.utc).isoformat()}
|
|
|
|
|
|
def complete_with_configured_model(messages: list[dict], *, max_tokens: int | None = None, output_schema: dict | None = None) -> dict:
|
|
config = model_configuration()
|
|
try:
|
|
options={'max_tokens':max_tokens}
|
|
# Compatible endpoints do not necessarily implement constrained decoding.
|
|
# Enable only for a provider/model whose capability was configured.
|
|
if output_schema is not None and os.getenv('ANSWER_STRUCTURED_OUTPUT','prompt')=='json_schema':
|
|
options['output_schema']=strict_output_schema(output_schema)
|
|
result = _complete_request(messages,config,**options)
|
|
except DeadlineExceeded:
|
|
raise # The answer budget ended; this says nothing about model health.
|
|
except Exception:
|
|
left = remaining_seconds()
|
|
if left is not None and left <= 1:
|
|
raise DeadlineExceeded('Answer time budget exhausted') from None
|
|
with _status_lock:
|
|
_status.update(configuration=configuration_key(config), checked_at=time.time(),success=False)
|
|
raise
|
|
with _status_lock:
|
|
_status.update(configuration=configuration_key(config),checked_at=time.time(),success=True)
|
|
return result
|
|
|
|
|
|
def probe_generation() -> dict:
|
|
try:
|
|
complete_with_configured_model([{'role':'user','content':'Reply only with OK.'}],max_tokens=32)
|
|
except Exception:
|
|
pass # The caller gets only sanitized configuration/availability, never raw exceptions.
|
|
return generation_status()
|
|
|
|
|
|
def _complete_request(messages: list[dict], config: dict, *, max_tokens: int | None = None, output_schema: dict | None = None) -> dict:
|
|
if max_tokens is not None and (type(max_tokens) is not int or not 1 <= max_tokens <= 16384):
|
|
raise ValueError("Invalid model output budget")
|
|
provider,model,key,endpoint = (config[k] for k in ('provider','model','key','endpoint'))
|
|
if provider == "anthropic":
|
|
system = "\n\n".join(row["content"] for row in messages if row["role"] == "system")
|
|
body = {
|
|
"model": model,
|
|
"messages": [row for row in messages if row["role"] != "system"],
|
|
"max_tokens": max_tokens or int(os.getenv("ANSWER_MAX_TOKENS", "4096")),
|
|
"temperature": 0,
|
|
}
|
|
if system:
|
|
body["system"] = system
|
|
if output_schema is not None:
|
|
body['output_config']={'format':{'type':'json_schema','schema':output_schema}}
|
|
headers = {"x-api-key": key, "anthropic-version": "2023-06-01"}
|
|
elif provider == "openai_compatible":
|
|
body = {"model": model, "messages": messages, "temperature": 0, "stream": False}
|
|
if max_tokens is not None:
|
|
body["max_tokens"] = max_tokens
|
|
if output_schema is not None:
|
|
body['response_format']={'type':'json_schema','json_schema':{'name':'question_plan','strict':True,'schema':output_schema}}
|
|
headers = {"Authorization": "Bearer " + key}
|
|
else:
|
|
raise RuntimeError("Unsupported answer provider")
|
|
request = urllib.request.Request(
|
|
endpoint,
|
|
data=json.dumps(body, ensure_ascii=False).encode(),
|
|
headers={"Content-Type": "application/json", **headers},
|
|
)
|
|
if remaining_seconds() is not None and remaining_seconds() < 2:
|
|
raise DeadlineExceeded("Answer time budget exhausted")
|
|
admitted = _gate.acquire(_deadline.get(), _owner.get(), _call_floor.get())
|
|
try:
|
|
# The budget is measured again: waiting in the gate consumed part of it.
|
|
timeout = float(os.getenv("ANSWER_LLM_TIMEOUT_SECONDS", "45"))
|
|
left = remaining_seconds()
|
|
if left is not None:
|
|
if left < 2:
|
|
raise DeadlineExceeded("Answer time budget exhausted")
|
|
timeout = min(timeout, left)
|
|
with urllib.request.build_opener(NoRedirect()).open(request, timeout=timeout) as response:
|
|
data = response.read(2 * 1024 * 1024 + 1)
|
|
finally:
|
|
if admitted is not None:
|
|
_gate.release(admitted)
|
|
if len(data) > 2 * 1024 * 1024:
|
|
raise RuntimeError("Answer model output is too large")
|
|
value = json.loads(data)
|
|
if provider == "anthropic":
|
|
if value.get("stop_reason") != "end_turn":
|
|
raise RuntimeError("Incomplete answer model output")
|
|
text = "".join(
|
|
block["text"] for block in value.get("content", []) if block.get("type") == "text"
|
|
)
|
|
if not text:
|
|
raise RuntimeError("Missing answer model output")
|
|
return {"choices": [{"message": {"content": text}}]}
|
|
if not isinstance(value,dict) or not value.get('choices'):
|
|
raise RuntimeError('Invalid answer model response')
|
|
choice=value['choices'][0]
|
|
if choice.get('finish_reason') not in (None,'stop') or not choice.get('message',{}).get('content'):
|
|
raise RuntimeError('Incomplete answer model output')
|
|
return value
|
|
|
|
|
|
def strict_output_schema(value):
|
|
"""Provider schema subset; validate the result against full Pydantic constraints."""
|
|
if isinstance(value,list):return [strict_output_schema(v) for v in value]
|
|
if not isinstance(value,dict):return value
|
|
result={k:strict_output_schema(v) for k,v in value.items() if k not in {'default','minimum','maximum','minLength','maxLength','minItems','maxItems','format'}}
|
|
if result.get('type')=='object':
|
|
result['additionalProperties']=False
|
|
result['required']=list(result.get('properties',{}))
|
|
return result
|