122 lines
8.2 KiB
Python
122 lines
8.2 KiB
Python
"""Author a source-bound, question-held-out TEST before any backend execution.
|
|
|
|
Evaluation-only templates, never imported by production. Gold comes from raw
|
|
Markdown, not a runtime extractor or its knowledge graph. Repeated profiles and
|
|
AI-authored gold do not constitute unseen-document or human-reviewed accuracy.
|
|
"""
|
|
from datetime import date
|
|
import hashlib
|
|
import json
|
|
import random
|
|
import re
|
|
from pathlib import Path
|
|
import frontmatter
|
|
|
|
ROOT=Path(__file__).resolve().parents[2]
|
|
WIKI=ROOT.parent/'zpwiki'
|
|
OUT=Path(__file__).parent
|
|
|
|
|
|
def profile_options():
|
|
used=set()
|
|
for p in (ROOT/'experiments/user_questions_v1').glob('*.json'):
|
|
cases=json.loads(p.read_text()).get('cases',[])
|
|
if isinstance(cases,list):used.update(e['path'] for c in cases for e in c.get('evidence',[]) if 'path' in e)
|
|
# Current conversational DEV subjects remain outside this new sample.
|
|
dev={'jan_holp','ondrej_megela','jakub_schwarc','jan_ptak','jan_malinovsky','maksym_zatirka','martin_sarissky','samuel_vasil','samuel_horani'}
|
|
result=[]
|
|
for p in sorted((WIKI/'pages/students').glob('*/*/README.md')):
|
|
relative=str(p.relative_to(WIKI))
|
|
if relative in used or p.parent.name in dev:continue
|
|
raw=p.read_text();page=frontmatter.loads(raw)
|
|
if page.get('published') is False:continue
|
|
lines=raw.splitlines()
|
|
headings=[(i,l.strip('# *')) for i,l in enumerate(lines) if re.match(r'^#+\s+(?:Bakalársk|Diplomov|Tímov|Výskumn|Vedeck)',l)]
|
|
for h,(start,label) in enumerate(headings):
|
|
if len(re.findall(r'\b20\d{2}\b',label))!=1 or re.search(r'\d/\d',label):continue
|
|
end=headings[h+1][0] if h+1<len(headings) else len(lines)
|
|
meetings=[]
|
|
for i in range(start+1,end):
|
|
if re.search(r'(?i)stretnut|konzult',lines[i]):
|
|
m=re.search(r'(\d{1,2})\.\s*(\d{1,2})\.\s*(20\d{2})',lines[i])
|
|
try:stamp=date(int(m[3]),int(m[2]),int(m[1])).isoformat() if m else None
|
|
except ValueError:stamp=None
|
|
meetings.append((i,stamp))
|
|
first=meetings[0][0] if meetings else end
|
|
goal_header=next((i for i in range(start+1,first) if re.search(r'(?i)zadanie|ciele',lines[i])),None)
|
|
if goal_header is None:continue
|
|
goals=[];in_goals=True
|
|
for i in range(goal_header+1,first):
|
|
text=lines[i].strip('# *: ')
|
|
if re.search(r'(?i)^(?:možné |predbežné )?(?:zadanie|ciele)\b',text):in_goals=True
|
|
elif lines[i].strip().endswith(':') or lines[i].startswith('#'):in_goals=False
|
|
if in_goals and re.match(r'^\s*(?:[-*]|\d+[.)])\s+',lines[i]):goals.append(i)
|
|
if not 1<=len(goals)<=8:continue
|
|
complete=[m for m in meetings if m[1]]
|
|
if not complete:continue
|
|
chosen=max(complete,key=lambda m:m[1]);mi,stamp=chosen
|
|
mend=next((i for i,_ in meetings if i>mi),end)
|
|
mode=None;state=[];tasks=[]
|
|
for i in range(mi+1,mend):
|
|
text=lines[i].strip('# *: ').casefold()
|
|
if text.startswith('stav'):mode='state';continue
|
|
if text.startswith(('úlohy','ulohy')):mode='tasks';continue
|
|
if text.startswith(('zásobník','zasobnik')):mode=None
|
|
if re.match(r'^\s*[-*]\s+',lines[i]):
|
|
if mode=='state':state.append(i)
|
|
if mode=='tasks':tasks.append(i)
|
|
if not 1<=len(state)<=4 or not 1<=len(tasks)<=4:continue
|
|
result.append({'name':str(page['title']).strip(),'path':relative,'label':label,'lines':lines,
|
|
'sha256':hashlib.sha256(raw.encode()).hexdigest(),'headings':headings,'goals':goals,
|
|
'state':state,'tasks':tasks,'meeting':mi,'date':stamp})
|
|
random.Random(20261110).shuffle(result)
|
|
selected=[];seen=set()
|
|
for p in result:
|
|
if p['path'] not in seen:selected.append(p);seen.add(p['path'])
|
|
if len(selected)==10:return selected
|
|
raise ValueError('Not enough independently source-bindable profiles; do not manufacture gold')
|
|
|
|
|
|
def build():
|
|
target=OUT/'test.json'
|
|
if target.exists():raise ValueError('Never overwrite a prepared TEST')
|
|
cases=[]
|
|
def ref(p,n,requirement,end=None):
|
|
end=n if end is None else end
|
|
return {'path':p['path'],'sha256':p['sha256'],'url':'https://zp.kemt.fei.tuke.sk/'+p['path'].removeprefix('pages/').removesuffix('/README.md'),
|
|
'start':n+1,'end':end+1,'quote':'\n'.join(p['lines'][n:end+1]),'requirements':[requirement],
|
|
'retrieval_terms':[[re.sub(r'^\s*(?:[-*]|\d+[.)])\s+','',p['lines'][n])]]}
|
|
def add(category,q,evidence,status='ok',requirements=()):
|
|
assert len(q)<=500
|
|
cases.append({'id':f'C{len(cases)+1:03d}','category':category,'question':q,
|
|
'expected_status':status,'evidence':evidence,'response_requirements':list(requirements)})
|
|
for p in profile_options():
|
|
name,label=p['name'],p['label'];year=re.search(r'\b20\d{2}\b',label)[0]
|
|
y,m,d=p['date'].split('-');day=f'{int(d)}.{int(m)}.{y}'
|
|
heading=next(i for i,l in p['headings'] if l==label)
|
|
work=[ref(p,heading,f'{name}: explicitly identify record {label}. Work year {year} is not study start or completion.')]
|
|
goals=[ref(p,n,f'{name}, {label}: include this goal as assigned or proposed, never already achieved: {p["lines"][n]}') for n in p['goals']]
|
|
state=[ref(p,n,f'{name}, {label}, record {day}: explain this recorded state, preserving uncertainty/negation and without claiming graduation: {p["lines"][n]}') for n in p['state']]
|
|
tasks=[ref(p,n,f'{name}, {label}, record {day}: planned follow-up, not an achieved result: {p["lines"][n]}') for n in p['tasks']]
|
|
marker=ref(p,p['meeting'],f'Explicit record date is {day}, unrelated to work year {year}.')
|
|
add('record_identity',f'U osoby {name} uveď typ a evidovaný rok záznamu „{label}“.',[*work])
|
|
inventory=[ref(p,n,f'Include this distinct record of {name}: {l}. A heading is not evidence of completed study.') for n,l in p['headings']]
|
|
add('inventory',f'Vypíš všetky evidované práce a projekty osoby {name}. Pri každom rozlíš typ a rok podľa nadpisu.',inventory)
|
|
add('goals',f'Čo bolo zadaním práce „{label}“ osoby {name}? Vysvetli všetky doložené ciele, nie domnelé výsledky.',goals)
|
|
add('dated_state',f'Ako vyzeral zaznamenaný stav osoby {name} pri práci „{label}“ na stretnutí {day}?',[marker,*state])
|
|
add('dated_plans',f'Aké ďalšie úlohy mala osoba {name} pri práci „{label}“ podľa zápisu {day}? Rozlišuj ich od hotovej práce.',[marker,*tasks])
|
|
add('latest_complete_record',f'Kedy je posledný záznam s úplným dátumom pri práci „{label}“ osoby {name}?',[marker],requirements=[
|
|
'Do not complete a partial date with a guessed year. Qualify latest as latest fully dated record; undated entries may not be ordered.'])
|
|
add('assignment_vs_record',f'Porovnaj zadanie práce „{label}“ osoby {name} s tým, čo je pri nej zaznamenané ako stav na stretnutí {day}. Nezamieňaj plán za výsledok.',[*goals,marker,*state])
|
|
assert not any('1997' in l for _,l in p['headings'])
|
|
add('absent_work_year',f'Aké práce má osoba {name} evidované v roku 1997?',[*inventory],status='no_answer',requirements=[
|
|
'No record from other work years may substitute the explicit 1997 filter. Absence in indexed records does not prove no such work exists anywhere.'])
|
|
value={'schema':1,'split':'TEST','authorship':'AI-authored against raw Markdown; no human gold review claimed',
|
|
'sampling':{'seed':20261110,'profiles':10,'questions_per_profile':8,
|
|
'exclusions':'Previously used user_questions_v1 gold profiles and current conversational DEV profiles.',
|
|
'limits':'Question-held-out public corpus; repeated profiles, known task families, selected structured profiles. Not an unseen-document population estimate.'},'cases':cases}
|
|
OUT.mkdir(parents=True,exist_ok=True);target.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
|
print(json.dumps({'cases':len(cases),'profiles':[p['name'] for p in profile_options()]},ensure_ascii=False))
|
|
|
|
if __name__=='__main__':build()
|