From e4a4865f9dc425cabb4104faeff9603580287520 Mon Sep 17 00:00:00 2001 From: jp170na Date: Fri, 14 Aug 2026 21:36:36 +0200 Subject: [PATCH] evaluation --- evaluation/questions.json | 379 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 evaluation/questions.json diff --git a/evaluation/questions.json b/evaluation/questions.json new file mode 100644 index 0000000..39dfd24 --- /dev/null +++ b/evaluation/questions.json @@ -0,0 +1,379 @@ +[ + { + "id": "q001", + "category": "fact_year", + "question": "V akom roku robil Ján Holp diplomovú prácu?", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "2021" + ], + "should_answer": true + }, + { + "id": "q002", + "category": "fact_year", + "question": "V ktorom roku začal študovať Ján Holp?", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "2016" + ], + "should_answer": true + }, + { + "id": "q003", + "category": "fact_title", + "question": "Aký je názov diplomovej práce Jána Holpa?", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "Systém získavania informácií v slovenskom jazyku" + ], + "should_answer": true + }, + { + "id": "q004", + "category": "fact_title", + "question": "Aký je názov diplomovej práce Maroša Harahusa?", + "expected_documents": [ + "pages/students/2016/maros_harahus/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus" + ], + "expected_answer_contains": [ + "Neurónová morfologická anotácia slovenského jazyka" + ], + "should_answer": true + }, + { + "id": "q005", + "category": "fact_topic", + "question": "Aká je téma diplomového projektu Jána Ptáka?", + "expected_documents": [ + "pages/students/2022/jan_ptak/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak" + ], + "expected_answer_contains": [ + "Agent pre manažment záverečných prác" + ], + "should_answer": true + }, + { + "id": "q006", + "category": "fact_year", + "question": "V ktorom roku začal študovať Ján Pták?", + "expected_documents": [ + "pages/students/2022/jan_ptak/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak" + ], + "expected_answer_contains": [ + "2022" + ], + "should_answer": true + }, + { + "id": "q007", + "category": "fact_tool", + "question": "Aké používateľské rozhranie má Ján Pták použiť pre svojho agenta?", + "expected_documents": [ + "pages/students/2022/jan_ptak/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak" + ], + "expected_answer_contains": [ + "OpenWebUI" + ], + "should_answer": true + }, + { + "id": "q008", + "category": "paraphrase", + "question": "Kto pracuje na agentovi na zlepšenie správy záverečných prác v ZP Wiki?", + "expected_documents": [ + "pages/students/2022/jan_ptak/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak" + ], + "expected_answer_contains": [ + "Ján Pták" + ], + "should_answer": true + }, + { + "id": "q009", + "category": "fact_topic", + "question": "Aká je téma diplomového projektu Jána Malinovského?", + "expected_documents": [ + "pages/students/2022/jan_malinovsky/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky" + ], + "expected_answer_contains": [ + "Grafové vyhľadávanie pre podporu vzdelávania" + ], + "should_answer": true + }, + { + "id": "q010", + "category": "paraphrase", + "question": "Ktorý študent vytvára znalostný graf z oblasti vzdelávania a agenta, ktorý v ňom vyhľadáva?", + "expected_documents": [ + "pages/students/2022/jan_malinovsky/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky" + ], + "expected_answer_contains": [ + "Ján Malinovský" + ], + "should_answer": true + }, + { + "id": "q011", + "category": "fact_title", + "question": "Ako sa volá diplomová práca Mateja Novotného?", + "expected_documents": [ + "pages/students/2021/matej_novotny/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny" + ], + "expected_answer_contains": [ + "Klasifikácia webových dát pre lepšie jazykové modelovanie" + ], + "should_answer": true + }, + { + "id": "q012", + "category": "fact_year", + "question": "V ktorom roku je uvedená diplomová práca Mateja Novotného?", + "expected_documents": [ + "pages/students/2021/matej_novotny/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny" + ], + "expected_answer_contains": [ + "2026" + ], + "should_answer": true + }, + { + "id": "q013", + "category": "fact_title", + "question": "Aký je názov diplomovej práce Dominika Nagya?", + "expected_documents": [ + "pages/students/2016/dominik_nagy/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy" + ], + "expected_answer_contains": [ + "Neurónový strojový preklad pomocou knižnice Fairseq" + ], + "should_answer": true + }, + { + "id": "q014", + "category": "topic", + "question": "Nájdi dokumenty súvisiace so strojovým prekladom.", + "expected_documents": [ + "pages/topics/translation/README.md", + "pages/students/2016/dominik_nagy/README.md", + "pages/students/2016/patrik_pavlisin/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/topics/translation", + "https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy", + "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin" + ], + "expected_answer_contains": [ + "strojový preklad" + ], + "should_answer": true + }, + { + "id": "q015", + "category": "fact_goal", + "question": "Na čom pracoval Oliver Pejic počas IAESTE stáže?", + "expected_documents": [ + "pages/interns/oliver_pejic/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/oliver_pejic" + ], + "expected_answer_contains": [ + "Hate Speech", + "MTEB" + ], + "should_answer": true + }, + { + "id": "q016", + "category": "paraphrase", + "question": "Ktorý stážista pripravoval MTEB evaluačné úlohy pre slovenský hate speech a question answering?", + "expected_documents": [ + "pages/interns/oliver_pejic/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/oliver_pejic" + ], + "expected_answer_contains": [ + "Oliver Pejic" + ], + "should_answer": true + }, + { + "id": "q017", + "category": "fact_topic", + "question": "Aká bola téma stáže Bogdana Paula Chișa?", + "expected_documents": [ + "pages/interns/bogdan_paul_chis/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis" + ], + "expected_answer_contains": [ + "Triplet extraction from medical data" + ], + "should_answer": true + }, + { + "id": "q018", + "category": "fact_tool", + "question": "Akými nástrojmi mal Bogdan Paul Chiș vyhodnocovať svojho agenta?", + "expected_documents": [ + "pages/interns/bogdan_paul_chis/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis" + ], + "expected_answer_contains": [ + "DeepEval", + "RAGAS" + ], + "should_answer": true + }, + { + "id": "q019", + "category": "fact_goal", + "question": "Aký bol cieľ stáže Youssefa Ressaissiho?", + "expected_documents": [ + "pages/interns/yussef_ressaissi/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/yussef_ressaissi" + ], + "expected_answer_contains": [ + "summarization", + "Slovak" + ], + "should_answer": true + }, + { + "id": "q020", + "category": "fact_goal", + "question": "Aké boli hlavné ciele stáže Cesara Abascala Gutierreza?", + "expected_documents": [ + "pages/interns/cesar_gutierrez/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/cesar_gutierrez" + ], + "expected_answer_contains": [ + "named entities", + "training set", + "annotation schema" + ], + "should_answer": true + }, + { + "id": "q021", + "category": "fact_goal", + "question": "Aký je cieľ projektu AI4Steel?", + "expected_documents": [ + "pages/topics/steel/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/topics/steel" + ], + "expected_answer_contains": [ + "tavnej zmesi", + "hlbokých neurónových sietí" + ], + "should_answer": true + }, + { + "id": "q022", + "category": "fact_topic", + "question": "Čomu sa venuje dizertačná práca Maroša Harahusa?", + "expected_documents": [ + "pages/students/2016/maros_harahus/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus" + ], + "expected_answer_contains": [ + "Automatické opravy textu", + "spracovanie prirodzeného jazyka" + ], + "should_answer": true + }, + { + "id": "q023", + "category": "multi_document", + "question": "Nájdi dokumenty, v ktorých sa rieši Retrieval Augmented Generation alebo GraphRAG.", + "expected_documents": [ + "pages/students/2022/jan_ptak/README.md", + "pages/students/2022/jan_malinovsky/README.md", + "pages/interns/bogdan_paul_chis/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky", + "https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis" + ], + "expected_answer_contains": [ + "RAG" + ], + "should_answer": true + }, + { + "id": "q024", + "category": "negative", + "question": "Akú diplomovú prácu robil Ján Holp v roku 2016?", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "2021" + ], + "should_answer": true, + "note": "Kontrolná otázka proti zámene roku začiatku štúdia 2016 s rokom diplomovej práce 2021." + } +] +