Kontrola zdrojov

This commit is contained in:
Ján Pták 2026-11-07 12:11:41 +01:00
parent de5adb2c99
commit 7d07374ce9
18 changed files with 417 additions and 21 deletions

View File

@ -34,6 +34,8 @@ Obsahový graf dopĺňa pojmy a vzťahy extrahované jazykovým modelom z profil
a explicitne uvedeným dátumom. Podporuje otázky na pokrok, čas výsledkov,
podobné témy a študentov venujúcich sa určitej činnosti. Zadanie ani plán
sa nevydávajú za dokončený výsledok; chýbajúce údaje sa neodhadujú.
Pri výsledkoch sa rozlišuje dátum záznamu od skutočného dňa vykonania práce.
Repozitárové odkazy sa overujú cez verejné API; neoveriteľné odkazy sa nezobrazujú.
Extrakciu spúšťa `python -m tools.knowledge.build` s nakonfigurovaným LLM API.
Vyžaduje pripravené spoločné indexy a Neo4j. Nezmenené vstupy využívajú cache;

View File

@ -83,6 +83,7 @@ class Config:
def error_result(reason):
messages={
'reference_unavailable':'Zdrojový repozitár sa nepodarilo verejne overiť. Odpoveď založenú iba na tomto odkaze neuvádzam.',
'unauthorized':'Prístup nebol autorizovaný. Skontroluj prihlásenie konektora a jeho konfiguráciu.',
'forbidden':'Pre tento nástroj nemáš potrebné oprávnenie.',
'invalid_query':'Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.',
@ -177,7 +178,7 @@ def query_backend(query: str, engine: str, *, mode='answer', limit=5, config=Non
# Raw model output, diagnostics and rejected text must not escape.
reason = value.get('reason')
safe_reasons = {'model_unavailable', 'answer_capacity_exceeded', 'answer_rejected',
'answer_unavailable', 'retrieval_unavailable', 'content_index_unavailable'}
'answer_unavailable', 'retrieval_unavailable', 'content_index_unavailable','reference_unavailable'}
raise ClientError(reason if isinstance(reason, str) and reason in safe_reasons else
'backend_answer_error' if mode == 'answer' else 'retrieval_unavailable')
validate_metadata(value)

View File

@ -3,12 +3,12 @@
{
"archive": "zpwiki-rag.zip",
"name": "rag",
"sha256": "e4813b8da2d0943b8587d5d4e1446ff10729a35b5d480812cf6e89b0d53b61a7"
"sha256": "812764028d95fc0b9f0ebd52f3aa20f38984eb1d3477da3f55a633703d423123"
},
{
"archive": "zpwiki-graphrag.zip",
"name": "graphrag",
"sha256": "1663af0ca32ef79e43fd5cdbfb14120d4cdb8fb4c3843284ef329b41e238de1f"
"sha256": "f548b9a4ec313fd687cffb2f004435cae0f4b40a0e5f57a955fde4dff27cb5d1"
}
],
"schema_version": 1

Binary file not shown.

Binary file not shown.

View File

@ -28,6 +28,10 @@ extraction is ready. Preserve any backend guidance instead of inventing graph ev
Do not turn a task into a result, a work year into a meeting date, or shared topics into collaboration.
For multiple explicit commands in one message, handle each question separately in order;
keep all conditions belonging to each question together.
For a follow-up omitting a person or supervisor, reconstruct the full question only when
that subject is unambiguous in the immediately preceding exchange. Preserve the new year
and every remaining condition. Otherwise ask for the name. Document authorship does not
establish supervision.
1. With a ZP Wiki MCP connector, call `zpwiki_graphrag` with the complete question.
2. Otherwise, with securely configured backend access in a Python execution environment,

View File

@ -20,6 +20,9 @@ from starlette.responses import JSONResponse
# Host-model rules belong to MCP initialization and Skills, not tool-result data.
QUESTION_POLICY = (
'For a follow-up omitting a person or supervisor, inherit the subject only from an unambiguous '
'immediately preceding exchange and reconstruct one full question with the new year and all constraints. '
'Otherwise ask for the name. Never infer supervision from document authorship. '
'For a complete question, copy the latest user question verbatim after removing only its leading '
'/rag or /graphrag command. Keep names, work types, years, negations, comparisons and every condition. '
'Never reuse a previous question or an example, paraphrase it or split its conditions across searches. '

View File

@ -30,6 +30,10 @@ requested aspects. For one work, use its exact quoted heading; for "both", reque
works in one query and keep their evidence separate. Do not search the words "both" alone.
If the preceding question/options are unavailable, ask for them rather than guessing.
Compare the returned `query` against this resolved full question in that case.
For a follow-up that omits the person or supervisor (for example "but which ones in 2026?"),
carry over that subject only when it is unambiguous in the immediately preceding exchange.
Send one complete question preserving the new year and all remaining conditions. If the subject
is unclear, ask for the name; never send a subjectless search or infer supervision from authorship.
For multiple explicit commands in one message, handle each question separately in order;
keep all conditions belonging to each question together.

173
test/test_source_audit.py Normal file
View File

@ -0,0 +1,173 @@
"""Source-link availability, lifecycle qualifications and supervisor semantics."""
import io
import json
import urllib.error
import pytest
from tools.answers import reference_links as links
from tools.answer_evaluation.content_selection import reference_only,evidence_phase,facets_for
from tools.knowledge.sources import profile_units
from tools.knowledge.query import answer_from_index
from tools.retrieval.person_names import resolve_person_names
REPO='https://git.kemt.fei.tuke.sk/sh662er/Rasa'
SOURCE='https://zp.kemt.fei.tuke.sk/students/2019/samuel_horani'
def result(text):
return {'accepted_for_delivery':True,'status':'ok','answer':text,'source_urls':[SOURCE],'claims':[]}
@pytest.mark.parametrize('text',[
'- [Repozitár s chatbotom]('+REPO+')',
'Repozitár: '+REPO,
'- Výstup je tu '+REPO,
])
def test_a_bare_reference_is_not_a_completed_result(text):
assert reference_only(text)
assert evidence_phase(text,'progress','result')=='unknown'
assert facets_for(text,'progress','Aké výsledky dosiahol?','result')==[]
def test_real_implementation_with_a_reference_remains_an_implementation():
text='Implementovaný chatbot používa RASA. Kód: '+REPO
assert not reference_only(text)
assert evidence_phase(text,'progress')=='result'
@pytest.mark.parametrize('text',[
'[Repozitár]('+REPO+')',REPO,
'[Repozitár][r]\n\n[r]: '+REPO,
'<a href="'+REPO+'">Repozitár</a>',
])
def test_unverified_repository_is_not_delivered_as_a_clickable_link(text):
value=links.checked_repository_links(result(text),check=lambda _: 'not_public')
assert REPO not in value['answer'] and value['source_urls']==[SOURCE]
assert value['accepted_for_delivery'] and 'súkromný' in value['answer']
assert value['suppressed_repository_links']==1
def test_verified_repository_preserves_the_exact_answer_and_sources():
value=result('[Repozitár]('+REPO+')')
assert links.checked_repository_links(value,check=lambda _: 'verified')==value
def test_html_escaped_markdown_reference_cannot_bypass_link_suppression():
url='https://github.com/example/missing?x=1&amp;y=2'
value=links.checked_repository_links(result('[Repozitár]('+url+')'),check=lambda _: 'unverified')
assert 'github.com' not in value['answer'] and '](' not in value['answer']
def test_unverifiable_repository_cannot_be_the_only_factual_citation():
value={**result('Práca je hotová.'),'source_urls':[REPO]}
checked=links.checked_repository_links(value,check=lambda _: 'unverified')
assert not checked['accepted_for_delivery'] and checked['reason']=='reference_unavailable'
assert not checked['source_urls']
@pytest.mark.parametrize('url',[
'http://git.kemt.fei.tuke.sk/a/b','https://git.kemt.fei.tuke.sk:8443/a/b',
'https://user:password@git.kemt.fei.tuke.sk/a/b','https://git.internal/a/b',
'https://github.com/a/b?access_token=private','https://git.kemt.fei.tuke.sk/../admin',
])
def test_repository_probe_never_contacts_unapproved_destinations(monkeypatch,url):
monkeypatch.setattr(links.urllib.request,'build_opener',lambda *a:pytest.fail('Unsafe network request'))
assert links._probe.__wrapped__(url,0)=='unverified'
@pytest.mark.parametrize('status,expected',[(404,'not_public'),(401,'not_public'),(403,'not_public'),(302,'unverified'),(503,'unverified')])
def test_repository_api_failures_are_not_treated_as_success(monkeypatch,status,expected):
class Opener:
def open(self,request,**kw):
assert request.full_url.endswith('/api/v1/repos/sh662er/Rasa')
assert 'Authorization' not in request.headers
raise urllib.error.HTTPError(request.full_url,status,'Unavailable',{},None)
monkeypatch.setattr(links.urllib.request,'build_opener',lambda *a:Opener())
assert links._probe.__wrapped__(REPO,0)==expected
def test_public_repo_api_metadata_is_required(monkeypatch):
class Opener:
def open(self,request,**kw):return io.BytesIO(json.dumps({'private':False,'full_name':'sh662er/Rasa'}).encode())
monkeypatch.setattr(links.urllib.request,'build_opener',lambda *a:Opener())
assert links._probe.__wrapped__(REPO,0)=='verified'
def index():
doc={'title':'Alena Nová','path':'pages/students/2022/alena/README.md',
'content':'# Diplomová práca 2024\nNázov: Overené grafy\nVedúci: Daniel Hládek\n'
'Stav:\n- Implementovaný systém RAG.\nUkončil predčasne\n'}
rows=profile_units(doc)
return {'units':rows,'version':'fixture','index_version':{'version':'fixture'}}
def test_lifecycle_qualification_is_preserved_even_when_it_has_only_two_words():
rows=index()['units']
note=next(r for r in rows if r['kind']=='lifecycle')
assert note['text']=='Ukončil predčasne' and note['work_year']==2024
r=answer_from_index('Ako pokročila Alena Nová?',index(),engine='rag')
assert 'Ukončil predčasne' in r['answer'] and any(e['evidence_role']=='lifecycle' for e in r['evidence'])
def test_supervisor_year_is_never_satisfied_by_a_historical_work():
r=answer_from_index('Ktoré práce vedie Daniel Hladek v roku 2026?',index(),engine='rag')
assert r['status']=='no_answer' and r['reason']=='supervision_not_documented'
assert '2026' in r['answer'] and 'autor dokumentu' in r['answer'].lower()
assert not r['evidence'] and not r['source_urls']
def test_current_supervision_is_not_inferred_from_a_formal_supervisor_line():
r=answer_from_index('Ktoré práce aktuálne vedie Daniel Hladek?',index(),engine='graphrag')
assert r['status']=='no_answer' and r['reason']=='active_supervision_unknown'
assert 'historickom' in r['answer'] and not r['evidence']
def test_missing_supervisor_in_followup_requires_a_name_and_keeps_the_year():
r=answer_from_index('Ktoré práce sú teraz vedené v roku 2026?',index(),engine='rag')
assert r['status']=='ambiguous' and r['reason']=='missing_supervisor'
assert '2026' in r['answer']
def test_title_typo_is_not_misclassified_as_a_person():
catalog=[{'name':'Daniel Hládek','kind':'Person'}]
r=resolve_person_names('Ktoré práce aktuálne vedie Daniel Hladek a nazv prác?',catalog)
assert r['status']=='ok' and [m['name'] for m in r['matches']]==['Daniel Hládek']
def test_dated_status_overrides_previous_meeting_instead_of_inheriting_its_year():
from tools.knowledge.sources import meeting_date
assert meeting_date('Stav 8.11.2024:')['date']=='2024-11-08'
rows=profile_units({'title':'Alena Nová','path':'pages/students/2022/alena/README.md',
'content':'# Bakalárska práca 2025\nStretnutie 4.2.2025\nStav:\n- Implementovaný systém RAG.\n\n'
'Stav 8.11.2024:\n- Urobené poznámky o RAG.\n'})
assert next(r for r in rows if 'poznámky' in r['text'])['date']=='2024-11-08'
def test_new_lifecycle_note_does_not_reextract_unchanged_later_profiles(tmp_path):
import hashlib
from tools.knowledge.build import build
from tools.sync.sync_indexes import source_files
from tools.sync.index_state import durable_json
wiki=tmp_path/'wiki';data=tmp_path/'data';data.mkdir()
paths=[]
for slug,name in [('alena','Alena Nová'),('boris','Boris Starý')]:
p=wiki/f'pages/students/2022/{slug}/README.md';p.parent.mkdir(parents=True)
p.write_text(f'---\ntitle: {name}\n---\n# Diplomová práca 2026\nStav:\n- Implementovaný systém RAG.\n')
paths.append(p)
def version(number):
return {'version':str(number),'source_sha256':hashlib.sha256(json.dumps(source_files(wiki),sort_keys=True).encode()).hexdigest()}
def complete(messages):
return {'choices':[{'message':{'content':json.dumps({'items':[]})}}]}
durable_json(data/'index_version.json',version(1))
build(wiki,data,complete,{'model':'fixture'},graph=False,vectors=False)
old=json.loads((data/'knowledge/index.json').read_text())
old_ids={u['person']:u['id'] for u in old['units']}
paths[0].write_text(paths[0].read_text()+'\nUkončil predčasne\n')
durable_json(data/'index_version.json',version(2))
report=build(wiki,data,complete,{'model':'fixture'},graph=False,vectors=False)
new=json.loads((data/'knowledge/index.json').read_text())
assert report['llm_batches']==0
assert all(u['id']==old_ids[u['person']] for u in new['units'] if u['kind']!='lifecycle')
assert any(u['kind']=='lifecycle' for u in new['units'])

View File

@ -22,6 +22,17 @@ def administrative_only(text: str) -> bool:
return bool(re.search(ADMIN,n) and not re.search(COMPLETED,n))
def reference_only(text: str) -> bool:
"""A link alone establishes a reference, never an implemented result."""
value=re.sub(r'^\s*(?:[-*+]|\d+[.)])\s+','',text).strip()
if re.fullmatch(r'\[[^]]+\]\([^\s]+\)',value):return True
from .context_policy import text_urls
links=text_urls(value)
if not links:return False
for url in links:value=value.replace(url,'')
return bool(re.fullmatch(r'(?:repozitar(?: s \w+(?: \w+)?)?|zdrojove kody|vystup je tu|vystupy su tu|odkaz|git)?',normalized(value)))
def split_units(text: str) -> list[str]:
from .rag_evidence import descriptive_text
parts = re.split(r'\n\s*\n|(?=^[ \t]*(?:\d+[.)]|[-+*])\s+)', text, flags=re.M)
@ -51,6 +62,7 @@ def effective_role(text: str, role: str) -> str:
def evidence_phase(text: str, role: str, source_phase: str = 'unknown') -> str:
if reference_only(text):return 'unknown'
if role=='assignment':return 'assignment'
if role=='task':return 'planned'
if administrative_only(text):return 'administrative'
@ -63,6 +75,7 @@ def evidence_phase(text: str, role: str, source_phase: str = 'unknown') -> str:
def facets_for(text: str, role: str, question: str, phase: str | None = None) -> list[str]:
if reference_only(text):return []
n = normalized(text); facets = []
if role == 'assignment': facets.append('assignment')
if role == 'task':facets.append('plans')

View File

@ -47,6 +47,14 @@ def with_name_resolution(result: dict, retrieved: dict) -> dict:
def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
complete=None, db_file: Path = DB_FILE, include_diagnostics: bool = False) -> dict:
from tools.answers.reference_links import checked_repository_links
result=_deliver_answer(question,engine=engine,limit=limit,complete=complete,
db_file=db_file,include_diagnostics=include_diagnostics)
return checked_repository_links(result)
def _deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
complete=None, db_file: Path = DB_FILE, include_diagnostics: bool = False) -> dict:
if not _capacity.acquire(blocking=False):
return public_failure({'reason':'answer_capacity_exceeded','engine':engine})
try:

View File

@ -1,5 +1,6 @@
"""Public failure wording; transport/model exception text is never a UI message."""
MESSAGES={
'reference_unavailable':'Zdrojový repozitár sa nepodarilo verejne overiť. Odpoveď založenú iba na tomto odkaze neuvádzam.',
'content_index_unavailable':'Obsahové vyhľadávanie momentálne nie je pripravené alebo sa jeho výstup '
'nepodarilo overiť. Treba overiť extrakciu pre aktuálnu verziu dát; '
'nejde o potvrdenie, že hľadané údaje neexistujú.',

View File

@ -0,0 +1,92 @@
"""Verify public repository references without forwarding any credentials.
Only the school's Gitea and GitHub APIs are queried. Redirects, userinfo,
custom ports and unsupported git hosts never expand the network allowlist.
"""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from functools import lru_cache
from html import unescape, escape
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from tools.answer_evaluation.context_policy import text_urls
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
def repository_reference(url):
try:
host=(urllib.parse.urlsplit(url).hostname or '').lower()
return host in {'github.com','gitlab.com'} or host.startswith(('git.','gitlab.','gitea.'))
except ValueError:return False
@lru_cache(maxsize=512)
def _probe(url, minute):
try:
parsed=urllib.parse.urlsplit(url)
if (parsed.scheme!='https' or parsed.username or parsed.password or parsed.port not in {None,443}
or parsed.query or parsed.hostname not in {'git.kemt.fei.tuke.sk','github.com'}):
return 'unverified'
parts=parsed.path.strip('/').split('/')
if len(parts)<2 or any(not re.fullmatch(r'[A-Za-z0-9_.-]+',p) or p in {'.','..'} for p in parts[:2]):
return 'unverified'
owner,repo=parts[0],parts[1].removesuffix('.git')
base='https://api.github.com/repos/' if parsed.hostname=='github.com' else 'https://git.kemt.fei.tuke.sk/api/v1/repos/'
opener=urllib.request.build_opener(NoRedirect())
request=urllib.request.Request(base+owner+'/'+repo,headers={'Accept':'application/json','User-Agent':'ZP-Agent repository verification'})
with opener.open(request,timeout=4) as response:
raw=response.read(262145)
if len(raw)>262144:return 'unverified'
data=json.loads(raw)
if data.get('private') is not False or str(data.get('full_name','')).casefold()!=(owner+'/'+repo).casefold():
return 'unverified'
# A repository can exist while a linked file/branch has been removed.
if len(parts)>2:
with opener.open(urllib.request.Request(url,headers={'User-Agent':'ZP-Agent repository verification'}),timeout=4) as response:
if response.status!=200:return 'unverified'
return 'verified'
except urllib.error.HTTPError as error:
code=error.code;error.close()
return 'not_public' if code in {401,403,404,410} else 'unverified'
except Exception:
return 'unverified'
def repository_status(url):
return _probe(url,int(time.monotonic()//60))
def checked_repository_links(result, *, check=None):
if not result.get('accepted_for_delivery'):return result
text=result.get('answer','')
sources=result.get('source_urls',[])
urls=sorted(u for u in (text_urls(text)|set(sources)) if repository_reference(u))
if not urls:return result
checker=check or repository_status
with ThreadPoolExecutor(max_workers=4) as pool:
statuses=dict(zip(urls[:8],pool.map(checker,urls[:8])))
bad={url for url in urls if statuses.get(url)!='verified'}
if not bad:return result
if bad & set(sources):
from tools.answers.answer_errors import public_failure
return public_failure({**result,'reason':'reference_unavailable'})
variants={variant for url in bad for variant in (url,escape(url,quote=False))}
for url in sorted(variants,key=len,reverse=True):
text=re.sub(r'\[([^]\n]+)\]\(\s*<?'+re.escape(url)+r'>?(?:\s+"[^"]*")?\s*\)',r'\1',text)
text=re.sub(r'^\s*\[[^]\n]+\]:\s*<?'+re.escape(url)+r'>?.*$', '',text,flags=re.M)
text=re.sub(r'\s+(?:href|src)\s*=\s*(["\x27])([^"\x27]*)\1',
lambda m:'' if unescape(m[2])==url else m[0],text,flags=re.I)
text=text.replace(url,'[odkaz neoverený]')
text+='\n\nRepozitárový odkaz zo zdroja sa nepodarilo verejne overiť, preto ho neuvádzam. '
text+='Nedostupnosť ani HTTP 404 samy nerozlišujú zmazaný a súkromný repozitár.'
return {**result,'answer':text,'suppressed_repository_links':len(bad)}

View File

@ -11,7 +11,7 @@ import zipfile
ROOT = Path(__file__).resolve().parents[2]
BASE = ROOT / 'packages/claude'
ENGINES = ('rag', 'graphrag')
PACKAGE_VERSION = '1.7.0'
PACKAGE_VERSION = '1.7.1'
def json_bytes(value):

View File

@ -57,7 +57,16 @@ def _build(wiki_root, data_dir, complete, model_identity, *, graph, vectors, wor
units = [unit for d in documents for unit in profile_units(d)]
# Keep whole statements, including trailing negations/qualifications.
# Only the model's annotation view may be shortened; evidence never is.
bounded = [{**unit,'id':identifier(unit['id'],i,unit['text'])} for i,unit in enumerate(units)]
previous=data_dir/'knowledge/index.json'
prior=json.loads(previous.read_text()).get('units',[]) if previous.exists() else []
def source_identity(unit):
return (unit['source_path'],unit.get('work_label'),unit['line'],unit['text'])
prior_ids={source_identity(u):u['id'] for u in prior}
# Inserting a status note must not invalidate every subsequent profile's
# model cache just because its position in the corpus has shifted.
bounded = [{**unit,'id':prior_ids.get(source_identity(unit),identifier(unit['id'],i,unit['text']))}
for i,unit in enumerate(units)]
annotatable=[u for u in bounded if u['kind'] not in {'reference','lifecycle'}]
cache = data_dir / 'knowledge/cache'
cache.mkdir(parents=True, exist_ok=True)
# Reuse annotations independently of dates/line offsets. These are parsed
@ -97,7 +106,7 @@ def _build(wiki_root, data_dir, complete, model_identity, *, graph, vectors, wor
if not 1 <= workers <= 4:
raise ValueError('Extraction workers must be between 1 and 4.')
with ThreadPoolExecutor(max_workers=workers) as pool:
pending = iter(batches(bounded))
pending = iter(batches(annotatable))
while window := list(islice(pending, workers)):
# Bounded submission: an API failure cannot leave hundreds of
# queued requests running after a failed extraction.
@ -110,16 +119,19 @@ def _build(wiki_root, data_dir, complete, model_identity, *, graph, vectors, wor
# A batch can straddle two profiles. Give profiles with no accepted
# annotation their own bounded pass rather than silently losing them.
absent={u['person'] for u in extracted}-{u['person'] for u in extracted if u['concepts']}
mixed={u['person'] for batch in batches(bounded) if len({r['person'] for r in batch})>1 for u in batch}
mixed={u['person'] for batch in batches(annotatable) if len({r['person'] for r in batch})>1 for u in batch}
absent &= mixed
by_id={u['id']:u for u in extracted}
for person in sorted(absent):
own=[u for u in bounded if u['person']==person and u['kind']!='supervisor'][:32]
own=[u for u in annotatable if u['person']==person and u['kind']!='supervisor'][:32]
if own:
rows,cached=process(own)
by_id.update({u['id']:u for u in rows})
calls+=int(not cached);hits+=int(cached)
extracted=[by_id[u['id']] for u in extracted]
extracted.extend({**u,'concepts':[],'relations':[]} for u in bounded if u['kind'] in {'reference','lifecycle'})
order={u['id']:i for i,u in enumerate(bounded)}
extracted.sort(key=lambda u:order[u['id']])
index = {'schema': 1, 'index_version': version, 'model': model_identity,
'source_scope': 'published_student_profiles', 'profiles': len(documents),
'units': extracted, 'graph_ready': graph}

View File

@ -11,7 +11,7 @@ from datetime import date,timedelta
import re
from tools.answer_evaluation.context_policy import normalized
from tools.answer_evaluation.content_selection import administrative_only, facets_for
from tools.answer_evaluation.content_selection import administrative_only, facets_for, reference_only
from tools.retrieval.person_names import resolve_person_names
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
from tools.knowledge.store import load_index, read_graph
@ -44,7 +44,7 @@ def intent(question: str) -> str | None:
text = normalized(question)
if re.search(r'\b(?:podobn\w*|pribuzn\w*)\b', text) and re.search(r'\b(?:kto|student\w*|osob\w*|prac\w*)\b', text):
return 'similar'
if re.search(r'\b(?:vedie|viedol|veduc\w*|skolitel\w*)\b', text) and re.search(r'\b(?:ktore|ake|kto)\b', text):
if re.search(r'\b(?:vedie|viedol|veduc\w*|veden\w*|skolitel\w*)\b', text) and re.search(r'\b(?:ktore|ake|kto)\b', text):
return 'supervisor'
if re.search(r'\b(?:pokroc\w*|pokrok\w*|posun\w*|napred\w*)\b|\bna com\b', text):
return 'progress'
@ -248,7 +248,8 @@ def answer_from_index(question: str, index: dict, *, engine: str, limit: int = 5
return row.get('work_year') in range(int(period[0][:4]), int(period[1][:4])+1)
return True
filtered = [r for r in rows if scoped(r)]
filtered = [r for r in rows if scoped(r) and r['kind'] not in {'reference','lifecycle'}
and not reference_only(r['text'])]
candidate_limited=False
semantic_similarity=False
if mode == 'supervisor':
@ -257,7 +258,10 @@ def answer_from_index(question: str, index: dict, *, engine: str, limit: int = 5
from tools.answer_evaluation.context_policy import text_contains_value
names=people & supervisor_names
if len(names)!=1:
return respond('Uveď celé meno vedúceho práce.', status='ambiguous', reason='missing_supervisor')
years=re.findall(r'\b[12]\d{3}\b',resolved)
suffix=f' Podmienku roku {years[0]} zachovám.' if len(set(years))==1 else ''
return respond('Koho máš na mysli? Uveď celé meno vedúceho práce.'+suffix,
status='ambiguous', reason='missing_supervisor')
name=next(iter(names));students=people-{name}
filtered = [r for r in candidates if text_contains_value(r['text'], name, person=True)
and (not students or r['person'] in students and
@ -354,8 +358,29 @@ def answer_from_index(question: str, index: dict, *, engine: str, limit: int = 5
if row['text'] not in {r['text'] for r in groups[key]}:
groups[key].append(row)
if not groups:
if mode=='supervisor':
years=re.findall(r'\b[12]\d{3}\b',resolved)
period_text=f' s rokom {years[0]}' if len(set(years))==1 else ' zodpovedajúce otázke'
return respond(f'V dostupných podkladoch nie je doložené, ktoré práce{period_text} vedie {name}. '
'Údaj „autor dokumentu“ neurčuje vedúceho práce a rok práce nepotvrdzuje jej aktuálnu aktivitu. '
'Preto nemôžem uviesť potvrdený zoznam aktuálne vedených prác. '
'Môžeš sa osobitne opýtať na dokumenty daného autora a práce uvedené v nich.',
status='no_answer',reason='supervision_not_documented')
return respond('V dostupnom výbere podkladov sa nenašiel spoľahlivý dôkaz zodpovedajúci tejto otázke.',
status='no_answer', reason='no_content_evidence',truncated=candidate_limited)
if mode=='supervisor' and re.search(r'\b(?:aktualne|teraz|sucasn\w*)\b',normalized(question)):
return respond(f'Aktuálny zoznam prác, ktoré vedie {name}, dostupné podklady nepotvrdzujú. '
'Samotný údaj o vedúcom pri historickom zázname nepotvrdzuje, ktoré práce sú aktuálne aktívne. '
'Autor dokumentu nie je automaticky vedúci práce. '
'Môžem vypísať historicky evidované práce s explicitne uvedeným vedúcim alebo dokumenty daného autora.',
status='no_answer',reason='active_supervision_unknown')
if mode=='similar':
def similar_priority(key):
group=groups[key]
actual=[r for r in group if r['phase'] in {'result','progress'}]
substantive=[r for r in actual if not re.search(r'\bpoznam\w*|\bprecitan\w*',normalized(r['text']))]
return (bool(substantive),bool(actual),max((r.get('date') or '' for r in group),default=''))
groups=dict(sorted(groups.items(),key=lambda item:similar_priority(item[0]),reverse=True))
selected_people = list(groups)[:limit]
evidence, lines, omitted = [], [], False
phase_names = {'assignment': 'zadanie', 'planned': 'plán', 'result': 'zaznamenaný výsledok',
@ -363,6 +388,10 @@ def answer_from_index(question: str, index: dict, *, engine: str, limit: int = 5
for person in selected_people:
lines.append(f'**{person}**')
group = groups[person]
if mode=='similar':
group.sort(key=lambda r:(r['phase'] in {'result','progress'},
not bool(re.search(r'\bpoznam\w*|\bprecitan\w*',normalized(r['text']))),
r.get('date') or '',r.get('work_year') or 0),reverse=True)
if mode in {'progress','timeline','period'}:
group.sort(key=lambda r: (r.get('date') or '', r['line']), reverse=True)
chosen = group[:8]
@ -409,7 +438,7 @@ def answer_from_index(question: str, index: dict, *, engine: str, limit: int = 5
if row.get('completion_basis')=='checked_task':
prefix=f'{work}; úloha označená ako splnená'
if timestamp:
prefix += f'; {timestamp}' + (' (rok neuvedený)' if row['date_precision'] == 'day_month' else '')
prefix += f'; záznam {timestamp}' + (' (rok neuvedený)' if row['date_precision'] == 'day_month' else '')
quote = re.sub(r'^\s*(?:[-*+]|\d+[.)])\s+', '', row['text'])
if row['phase'] in {'assignment','planned'}:
from tools.answer_evaluation.presentation import goal_text
@ -425,6 +454,11 @@ def answer_from_index(question: str, index: dict, *, engine: str, limit: int = 5
lines.append('Zadanie je doložené; dosiahnuté výsledky ani pokrok tento profil nedokladá.')
if missing:
lines.append('V tomto profile chýbajú doložené údaje pre: '+', '.join(missing)+'.')
work_keys={(r['source_path'],r.get('work_label')) for r in chosen}
for note in rows:
if note['kind']=='lifecycle' and (note['source_path'],note.get('work_label')) in work_keys:
lines.append('Poznámka o stave tejto práce zo zdroja: '+note['text'])
evidence.append({**note,'evidence_role':'lifecycle'})
if len(group) > len(chosen):
omitted=True
lines.append('Zobrazený je výber podkladov; otázku môžeš zúžiť na prácu, dátum alebo konkrétny výsledok.')
@ -441,6 +475,8 @@ def answer_from_index(question: str, index: dict, *, engine: str, limit: int = 5
lines.append('Ide o obsahovú podobnosť podloženú uvedenými úryvkami, nie dôkaz spolupráce ani rovnakej fázy dokončenia.')
if mode in {'progress','period','timeline'} and re.search(r'\b(?:aktual\w*|teraz|dnes)\b',normalized(question)):
lines.append('Ide o stav zaznamenaný v profiloch; tieto podklady samy nepotvrdzujú aktuálny stav k dnešnému dňu.')
if mode=='timeline':
lines.append('Dátumy označujú záznamy o výsledkoch; presný deň vykonania práce z nich automaticky nevyplýva.')
truncated = len(groups) > len(selected_people) or omitted or candidate_limited
if candidate_limited:
lines.append('Zobrazené sú príklady z obmedzeného výberu textových dôkazov; zoznam nemusí byť úplný. '
@ -505,6 +541,12 @@ def deliver_content(question: str, *, engine: str, limit: int, db_file, complete
return public_failure({'reason':'content_index_unavailable','engine':engine})
return None
units = read_graph(index) if engine == 'graphrag' else index['units']
from tools.knowledge.sources import source_lifecycle_notes
from tools.rag.common import ZPWIKI_ROOT
notes=source_lifecycle_notes(ZPWIKI_ROOT,index['index_version'].get('source_sha256'))
existing={(row['source_path'],row.get('work_label'),row['text']) for row in units}
units=[*units,*(note for note in notes if
(note['source_path'],note.get('work_label'),note['text']) not in existing)]
# Slow model calls must not hold the publication lock. Everything below
# uses this captured source/graph generation, never a mixture of versions.
if not mode:
@ -514,7 +556,7 @@ def deliver_content(question: str, *, engine: str, limit: int, db_file, complete
'answer':'Upresni, či sa pýtaš na pokrok, výsledky s dátumami, podobné témy alebo konkrétnu činnosť študentov.',
'engine':engine,'claims':[],'source_urls':[],'index_version':index['index_version'],
'complete':False,'truncated':False,'truncation_reasons':[]}
if mode in {'discovery','similar'}:
if mode in {'discovery','similar','supervisor'}:
from tools.knowledge.sources import with_work_context
units=with_work_context(units)
relevance = None

View File

@ -13,10 +13,11 @@ from tools.rag.common import clean_heading_text
from tools.rag.search_core import make_source_url
from tools.answer_evaluation.context_policy import normalized
from tools.answer_evaluation.rag_evidence import academic_scope
from tools.answer_evaluation.content_selection import effective_role, evidence_phase
from tools.answer_evaluation.content_selection import effective_role, evidence_phase, reference_only
PROFILE = re.compile(r'(?:pages/)?students/\d{4}/[^/]+/README\.md$')
DATE = re.compile(r'(?<!\d)(\d{1,2})\.(\d{1,2})[.:](?:\s*([12]\d{3}))?(?!\d)')
LIFECYCLE = re.compile(r'\b(?:ukoncil\w* predcasne|prerusil\w* studium|odstupil\w*|praca (?:bola )?zrusena)\b')
def identifier(*values) -> str:
@ -24,9 +25,12 @@ def identifier(*values) -> str:
def supervisor_name(unit: dict) -> str | None:
if unit['kind']!='supervisor' or ':' not in unit['text']:
if unit['kind']!='supervisor':
return None
value=unit['text'].split(':',1)[1]
plain=re.sub(r'[*_]','',unit['text']).strip().lstrip('-+ ')
match=re.match(r'^(?:meno ved[uú]ceho(?: pr[aá]ce)?|ved[uú]c[iaí](?: pr[aá]ce)?|[šs]kolite[lľ])\s*[:\-]?\s*(.+)$',plain,re.I)
if not match:return None
value=match[1]
value=re.sub(r'\[([^]]+)\]\([^)]*\)',r'\1',value)
value=re.sub(r'\b(?:ing|mgr|bc|rndr|phdr|phd|doc|prof)\.?\b',' ',value,flags=re.I)
words=re.findall(r'[^\W\d_]+(?:-[^\W\d_]+)*',value,re.UNICODE)
@ -46,7 +50,8 @@ def with_work_context(units: list[dict]) -> list[dict]:
def meeting_date(line: str) -> dict | None:
text = normalized(line)
if not re.search(r'\b(?:stretnutie|revizia|konzultacia)\b', text) and not DATE.fullmatch(line.strip().rstrip(':')):
dated_status=bool(re.match(r'^(?:stav|stav pr[aá]ce)\s+\d{1,2}\.\d{1,2}',line.strip(' *#_'),re.I))
if not dated_status and not re.search(r'\b(?:stretnutie|revizia|konzultacia)\b', text) and not DATE.fullmatch(line.strip().rstrip(':')):
return None
match = DATE.search(line)
if not match:
@ -79,7 +84,8 @@ def profile_units(document: dict) -> list[dict]:
text = '\n'.join(buffer).strip()
buffer.clear()
plain = re.sub(r'^\s*(?:[-*+]|\d+[.)])\s+', '', text)
if len(normalized(text).split()) < 3 or re.fullmatch(r'https?://\S+', plain):
lifecycle=bool(LIFECYCLE.search(normalized(text)))
if (len(normalized(text).split()) < 3 and not lifecycle) or re.fullmatch(r'https?://\S+', plain):
if role=='topic':role='other'
return
current_role = effective_role(text, role)
@ -105,6 +111,8 @@ def profile_units(document: dict) -> list[dict]:
kind='topic'
if phase == 'result':
kind = 'result'
if reference_only(text):kind,phase='reference','unknown'
if lifecycle:kind,phase='lifecycle','unknown'
if re.search(r'\b(?:meno veduceho|veduci(?: prace)?|veduca(?: prace)?|skolitel)\b', normalized(text)):
kind, phase = 'supervisor', 'unknown'
units.append({'id': identifier(path, work_label, start, text), 'person': person,
@ -158,6 +166,11 @@ def profile_units(document: dict) -> list[dict]:
scope,work_label,work_level=None,None,None
role,source_phase='other','unknown'
timestamp={'date':None,'date_label':None,'date_precision':'unknown'}
inline_label=normalized(line.split(':',1)[0]) if ':' in line else ''
if (LIFECYCLE.search(normalized(line)) or inline_label in {
'nazov','nazov prace','predbezny nazov','tema','veduci','veduci prace','meno veduceho',
'meno veduceho prace','skolitel','repozitar'}):
flush()
if not line.strip() or re.match(r'^\s*(?:[-*+]\s+|\d+[.)]\s+)', line):
flush()
if line.strip():
@ -166,3 +179,31 @@ def profile_units(document: dict) -> list[dict]:
buffer.append(line)
flush()
return units
def source_lifecycle_notes(wiki_root, source_digest):
"""Read explicit lifecycle qualifications only from the indexed snapshot."""
return _source_lifecycle_notes(str(wiki_root),source_digest)
from functools import lru_cache
@lru_cache(maxsize=2)
def _source_lifecycle_notes(wiki_root, source_digest):
import json
from pathlib import Path
from tools.sync.sync_indexes import source_files
from tools.rag.common import load_zpwiki_page
root=Path(wiki_root)
if not (root/'pages').is_dir():return []
files=source_files(root)
if hashlib.sha256(json.dumps(files,sort_keys=True).encode()).hexdigest()!=source_digest:
return [] # Never attach qualifications from a different source version.
notes=[]
for path in files:
if not PROFILE.fullmatch(path):continue
if not LIFECYCLE.search(normalized((root/path).read_text())):continue
document=load_zpwiki_page(root/path,wiki_root=root)
if document.get('published') is False:continue
notes.extend({**row,'evidence_role':'lifecycle'} for row in profile_units(document) if row['kind']=='lifecycle')
return notes

View File

@ -80,7 +80,7 @@ def _score(words: list[str], canonical: list[str]) -> int | None:
_STOP = set('a aj ale alebo ak ako ake aky aku ktore ktori ktora ktory kto kolko ci co obaja obidvaja '
'ma maju mal mali je su bol bola bolo boli budu bude sa s so pri pre z zo na v od do roku rokoch '
'praca prace pracu projekt projekty dokument dokumenty nazov nazvy '
'praca prace pracu projekt projekty dokument dokumenty nazov nazvy nazv '
'tag tagy klucove slova slovo temu temy kategoriu kategorie '
'uved uvedte vypis vyber najdi zisti porovnaj popis opis čomu komu vymenuj pomenuj '
'vsetci vsetky vsetkych plan stav vysledky problemy metody experimenty '
@ -95,7 +95,7 @@ def _generic(word: str) -> bool:
'zameran', 'ciel', 'vysled', 'priebeh', 'pouz', 'experiment',
'problem', 'metod', 'riesen', 'system', 'vyhladav', 'inform',
'studij', 'databaz', 'zdroj', 'prehlad', 'presn', 'postup',
'hodnot', 'technol', 'zadani', 'uloh', 'spaja', 'evidov'))
'hodnot', 'technol', 'zadani', 'uloh', 'spaja', 'evidov', 'veden'))
def _identity(row: dict) -> tuple: