30 lines
4.4 KiB
Python
30 lines
4.4 KiB
Python
"""Known audited questions and paraphrases: DEV only, never an independent TEST."""
|
|
from build_cases import profile,evidence,OUT
|
|
import json,re
|
|
D=[]
|
|
def ev(slug,needle,req,terms,end=None):
|
|
p=profile(slug);n=next(i for i,s in enumerate(p['lines'],1) if needle in s)
|
|
return evidence(p,n,end or n,req,terms)
|
|
def add(q,refs,status='ok',req=(),short=False):
|
|
row={'id':f'D{len(D)+1:02}','category':'audit_regression','question':q,'expected_status':status,'evidence':refs,'response_requirements':list(req)}
|
|
if short:row['max_words']=120
|
|
D.append(row)
|
|
agent=ev('jan_malinovsky','Vytvorený jednoduchý agent','Ján Malinovský made an agent retrieving from the knowledge graph and generating an answer.',['Vytvorený jednoduchý agent'])
|
|
topic=ev('jan_malinovsky','Grafové vyhľadávanie pre podporu','The topic is graph retrieval supporting education.',['Grafové vyhľadávanie'])
|
|
add('Vysvetli mi jednoducho, na čom pracoval Ján Malinovský.',[agent,topic],short=True)
|
|
add('Ako pokročil Ján Malinovský so svojou prácou?',[agent],req=['Distinguish actual work from future plans.'])
|
|
p=profile('jan_holp');start=next(i for i,l in enumerate(p['lines'],1) if 'Vypracujte prehľad metód ohodnotenia' in l)
|
|
assignment=evidence(p,start,start+3,'All FOUR thesis assignment goals: review document-ranking methods; implement Slovak retrieval using ranking; design/evaluate experiments; suggest retrieval-accuracy improvements. Assignment is not a completed result.',['ohodnotenia','experimenty','zlepšenia'])
|
|
add('Čomu sa venuje diplomová práca Jána Holpa? Stručne predstav jej zadanie a uveď zdroj.',[assignment],short=True)
|
|
add('Aké bolo zadanie diplomovej práce Jána Holpa?',[assignment])
|
|
add('Aké technické problémy riešil Ondrej Megela?',[ev('ondrej_megela','Natrénovaný model wiki103 na stroji Quadro','Report the GPU training problem and its recorded workaround under the correct bachelor work.',['wiki103','GPU']),ev('ondrej_megela','Vytvorený notebook pre vyhodnotenie','Diploma thesis 2023: the BLEU evaluation notebook exists but is not working; ordering generated/reference questions and multiple questions per paragraph complicate evaluation. Do not report absence of technical problems for this work.',['BLEU','nefunguje'])],req=['Do not ask the user to select a work for this broad overview; group by works and distinguish known from missing details.'])
|
|
add('Aké metódy používal Ján Holp pri diplomovom projekte v roku 2020?',[],status='ambiguous',req=['Two distinct projects in 2020 exist; ask which one, listing both.'])
|
|
add('Aké metódy používal Ján Holp pri oboch diplomových projektoch v roku 2020?',[ev('jan_holp','Vyriešené technické problémy s cassandrou','Under project 2: Cassandra to ES indexing.',['Cassandra','ES']),ev('jan_holp','funguje databáza Cassandra','Under unnumbered diploma project: Cassandra works and file indexing works.',['Cassandra'])],req=['Both projects separately; never mix their activities.'])
|
|
add('Kedy mal Ján Malinovský naposledy stretnutie a čo na ňom riešili?',[ev('jan_malinovsky','Stretnutie 14.5.2026','Most recent complete record date 2026-05-14; describe its content.',['2026']),agent],req=['RAGAS/DeepEval from a general undated backlog must NOT be attributed to this meeting. Record date is not execution date.'])
|
|
add('Akú prácu mal Ján Holp v roku 1999?',[],status='no_answer',req=['No evidence for the requested person/year; do not substitute 2019,2020,2021.'])
|
|
add('Aké spoločné kľúčové slová majú Ján Pták a Ján Malinovský?',[ev('jan_ptak','tag:','Common tags NLP and RAG.',['nlp','rag']),ev('jan_malinovsky','tag:','Common tags NLP and RAG.',['nlp','rag'])])
|
|
add('Kto pracoval na niečom podobnom ako Ján Pták?',[ev('jan_ptak','Oboznámte sa s tým čo je znalostný graf','Anchor is a planned graph task, not proof of completed graph implementation.',['znalostný graf'])],req=['Return at least one actually sourced peer on RAG or knowledge graphs, with evidence for both people. Do not call this proof of collaboration or equal completion.'])
|
|
add('Ktoré práce aktuálne vedie Daniel Hladek?',[],status='no_answer',req=['Explicitly say current active supervision is not established. document_author is not supervisor; do not infer from historical records.'])
|
|
(OUT/'dev_v2.json').write_text(json.dumps({'schema':1,'split':'DEV','scope':'Audited, previously seen regressions and paraphrases; no independent-accuracy claim.','cases':D},ensure_ascii=False,indent=2)+'\n')
|
|
print('DEV',len(D))
|