jp170na/dp-zp-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

evaluation

Browse Source
This commit is contained in:
Ján Pták 2026-08-14 21:36:36 +02:00
parent 9dc45a687b
commit e4a4865f9d
1 changed files with 379 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

379
evaluation/questions.json Normal file
View File

@ -0,0 +1,379 @@
[
{
"id": "q001",
"category": "fact_year",
"question": "V akom roku robil Ján Holp diplomovú prácu?",
"expected_documents": [
"pages/students/2016/jan_holp/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
],
"expected_answer_contains": [
"2021"
],
"should_answer": true
},
{
"id": "q002",
"category": "fact_year",
"question": "V ktorom roku začal študovať Ján Holp?",
"expected_documents": [
"pages/students/2016/jan_holp/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
],
"expected_answer_contains": [
"2016"
],
"should_answer": true
},
{
"id": "q003",
"category": "fact_title",
"question": "Aký je názov diplomovej práce Jána Holpa?",
"expected_documents": [
"pages/students/2016/jan_holp/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
],
"expected_answer_contains": [
"Systém získavania informácií v slovenskom jazyku"
],
"should_answer": true
},
{
"id": "q004",
"category": "fact_title",
"question": "Aký je názov diplomovej práce Maroša Harahusa?",
"expected_documents": [
"pages/students/2016/maros_harahus/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
],
"expected_answer_contains": [
"Neurónová morfologická anotácia slovenského jazyka"
],
"should_answer": true
},
{
"id": "q005",
"category": "fact_topic",
"question": "Aká je téma diplomového projektu Jána Ptáka?",
"expected_documents": [
"pages/students/2022/jan_ptak/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
],
"expected_answer_contains": [
"Agent pre manažment záverečných prác"
],
"should_answer": true
},
{
"id": "q006",
"category": "fact_year",
"question": "V ktorom roku začal študovať Ján Pták?",
"expected_documents": [
"pages/students/2022/jan_ptak/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
],
"expected_answer_contains": [
"2022"
],
"should_answer": true
},
{
"id": "q007",
"category": "fact_tool",
"question": "Aké používateľské rozhranie má Ján Pták použiť pre svojho agenta?",
"expected_documents": [
"pages/students/2022/jan_ptak/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
],
"expected_answer_contains": [
"OpenWebUI"
],
"should_answer": true
},
{
"id": "q008",
"category": "paraphrase",
"question": "Kto pracuje na agentovi na zlepšenie správy záverečných prác v ZP Wiki?",
"expected_documents": [
"pages/students/2022/jan_ptak/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
],
"expected_answer_contains": [
"Ján Pták"
],
"should_answer": true
},
{
"id": "q009",
"category": "fact_topic",
"question": "Aká je téma diplomového projektu Jána Malinovského?",
"expected_documents": [
"pages/students/2022/jan_malinovsky/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
],
"expected_answer_contains": [
"Grafové vyhľadávanie pre podporu vzdelávania"
],
"should_answer": true
},
{
"id": "q010",
"category": "paraphrase",
"question": "Ktorý študent vytvára znalostný graf z oblasti vzdelávania a agenta, ktorý v ňom vyhľadáva?",
"expected_documents": [
"pages/students/2022/jan_malinovsky/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
],
"expected_answer_contains": [
"Ján Malinovský"
],
"should_answer": true
},
{
"id": "q011",
"category": "fact_title",
"question": "Ako sa volá diplomová práca Mateja Novotného?",
"expected_documents": [
"pages/students/2021/matej_novotny/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny"
],
"expected_answer_contains": [
"Klasifikácia webových dát pre lepšie jazykové modelovanie"
],
"should_answer": true
},
{
"id": "q012",
"category": "fact_year",
"question": "V ktorom roku je uvedená diplomová práca Mateja Novotného?",
"expected_documents": [
"pages/students/2021/matej_novotny/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny"
],
"expected_answer_contains": [
"2026"
],
"should_answer": true
},
{
"id": "q013",
"category": "fact_title",
"question": "Aký je názov diplomovej práce Dominika Nagya?",
"expected_documents": [
"pages/students/2016/dominik_nagy/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy"
],
"expected_answer_contains": [
"Neurónový strojový preklad pomocou knižnice Fairseq"
],
"should_answer": true
},
{
"id": "q014",
"category": "topic",
"question": "Nájdi dokumenty súvisiace so strojovým prekladom.",
"expected_documents": [
"pages/topics/translation/README.md",
"pages/students/2016/dominik_nagy/README.md",
"pages/students/2016/patrik_pavlisin/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/topics/translation",
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy",
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
],
"expected_answer_contains": [
"strojový preklad"
],
"should_answer": true
},
{
"id": "q015",
"category": "fact_goal",
"question": "Na čom pracoval Oliver Pejic počas IAESTE stáže?",
"expected_documents": [
"pages/interns/oliver_pejic/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/interns/oliver_pejic"
],
"expected_answer_contains": [
"Hate Speech",
"MTEB"
],
"should_answer": true
},
{
"id": "q016",
"category": "paraphrase",
"question": "Ktorý stážista pripravoval MTEB evaluačné úlohy pre slovenský hate speech a question answering?",
"expected_documents": [
"pages/interns/oliver_pejic/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/interns/oliver_pejic"
],
"expected_answer_contains": [
"Oliver Pejic"
],
"should_answer": true
},
{
"id": "q017",
"category": "fact_topic",
"question": "Aká bola téma stáže Bogdana Paula Chișa?",
"expected_documents": [
"pages/interns/bogdan_paul_chis/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis"
],
"expected_answer_contains": [
"Triplet extraction from medical data"
],
"should_answer": true
},
{
"id": "q018",
"category": "fact_tool",
"question": "Akými nástrojmi mal Bogdan Paul Chiș vyhodnocovať svojho agenta?",
"expected_documents": [
"pages/interns/bogdan_paul_chis/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis"
],
"expected_answer_contains": [
"DeepEval",
"RAGAS"
],
"should_answer": true
},
{
"id": "q019",
"category": "fact_goal",
"question": "Aký bol cieľ stáže Youssefa Ressaissiho?",
"expected_documents": [
"pages/interns/yussef_ressaissi/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/interns/yussef_ressaissi"
],
"expected_answer_contains": [
"summarization",
"Slovak"
],
"should_answer": true
},
{
"id": "q020",
"category": "fact_goal",
"question": "Aké boli hlavné ciele stáže Cesara Abascala Gutierreza?",
"expected_documents": [
"pages/interns/cesar_gutierrez/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/interns/cesar_gutierrez"
],
"expected_answer_contains": [
"named entities",
"training set",
"annotation schema"
],
"should_answer": true
},
{
"id": "q021",
"category": "fact_goal",
"question": "Aký je cieľ projektu AI4Steel?",
"expected_documents": [
"pages/topics/steel/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/topics/steel"
],
"expected_answer_contains": [
"tavnej zmesi",
"hlbokých neurónových sietí"
],
"should_answer": true
},
{
"id": "q022",
"category": "fact_topic",
"question": "Čomu sa venuje dizertačná práca Maroša Harahusa?",
"expected_documents": [
"pages/students/2016/maros_harahus/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
],
"expected_answer_contains": [
"Automatické opravy textu",
"spracovanie prirodzeného jazyka"
],
"should_answer": true
},
{
"id": "q023",
"category": "multi_document",
"question": "Nájdi dokumenty, v ktorých sa rieši Retrieval Augmented Generation alebo GraphRAG.",
"expected_documents": [
"pages/students/2022/jan_ptak/README.md",
"pages/students/2022/jan_malinovsky/README.md",
"pages/interns/bogdan_paul_chis/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
"https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis"
],
"expected_answer_contains": [
"RAG"
],
"should_answer": true
},
{
"id": "q024",
"category": "negative",
"question": "Akú diplomovú prácu robil Ján Holp v roku 2016?",
"expected_documents": [
"pages/students/2016/jan_holp/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
],
"expected_answer_contains": [
"2021"
],
"should_answer": true,
"note": "Kontrolná otázka proti zámene roku začiatku štúdia 2016 s rokom diplomovej práce 2021."
}
]