6473 lines
195 KiB
JSON
6473 lines
195 KiB
JSON
[
|
|
{
|
|
"id": "p0001",
|
|
"query": "Ktorá stáž osoby Bogdan Paul Chiș súvisela s témou medical?",
|
|
"source_question_id": "q0583",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/interns/bogdan_paul_chis/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Bogdan Paul Chiș",
|
|
"medical"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0002",
|
|
"query": "Nájdi dokument o osobe Samuel Vasiľ.",
|
|
"source_question_id": "q0510",
|
|
"source_category": "entity_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/samuel_vasil/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/samuel_vasil"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Samuel Vasiľ"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0003",
|
|
"query": "Akým spôsobom má Ján Pták implementovať synchronizáciu?",
|
|
"source_question_id": "q0392",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/jan_ptak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"WebHOOK"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0004",
|
|
"query": "Ktorá stáž osoby Oliver Pejic súvisela s témou SentenceTransformer?",
|
|
"source_question_id": "q0555",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/interns/oliver_pejic/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/oliver_pejic"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Oliver Pejic",
|
|
"SentenceTransformer"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0005",
|
|
"query": "Nájdi tematický dokument súvisiaci s pojmom jazykové zdroje.",
|
|
"source_question_id": "q0669",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/topics/nlp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/nlp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Nástroje a zdroje pre spracovanie prirodzeného jazyka"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0006",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia“.",
|
|
"source_question_id": "q0156",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Filip Tomáš",
|
|
"Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0007",
|
|
"query": "Ktorá stáž osoby Sevval Bulburu súvisela s témou Hate Speech?",
|
|
"source_question_id": "q0562",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/interns/sevval_bulburu/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/sevval_bulburu"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Sevval Bulburu",
|
|
"Hate Speech"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0008",
|
|
"query": "Aký source_url patrí k stránke „Morfologická analýza slovenského jazyka“?",
|
|
"source_question_id": "q0627",
|
|
"source_category": "source_url",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/topics/morphology/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/morphology"
|
|
],
|
|
"expected_answer_contains": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/morphology"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0009",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou SlovakBERT a osobou Vladimír Ferko?",
|
|
"source_question_id": "q0257",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2021/vladimir_ferko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Vladimír Ferko",
|
|
"SlovakBERT"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0010",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou neurónové siete a osobou Ondrej Megela?",
|
|
"source_question_id": "q0130",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ondrej Megela",
|
|
"neurónové siete"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0011",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou anotácia a osobou Artem Yatsenko?",
|
|
"source_question_id": "q0212",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/artem_yatsenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/artem_yatsenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Artem Yatsenko",
|
|
"anotácia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0012",
|
|
"query": "Aký je názov bakalárska práca osoby Filip Tomáš?",
|
|
"source_question_id": "q0154",
|
|
"source_category": "fact_title",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0013",
|
|
"query": "Aký dataset mal Tomáš Kuchárik strojovo prekladať?",
|
|
"source_question_id": "q0037",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/tomas_kucharik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"SQUAD"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0014",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou pomenované entity a osobou Jakub Maruniak?",
|
|
"source_question_id": "q0048",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Jakub Maruniak",
|
|
"pomenované entity"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0015",
|
|
"query": "Nájdi dokument o stážistovi Bogdan Paul Chiș.",
|
|
"source_question_id": "q0577",
|
|
"source_category": "entity_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/interns/bogdan_paul_chis/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Bogdan Paul Chiș"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0016",
|
|
"query": "Nájdi tematický dokument súvisiaci s pojmom odpovede.",
|
|
"source_question_id": "q0679",
|
|
"source_category": "topic_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/topics/question/navod/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/question/navod"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Anotácia otázok a odpovedí"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0017",
|
|
"query": "Aký je názov bakalárska práca osoby Vladimír Ferko?",
|
|
"source_question_id": "q0246",
|
|
"source_category": "fact_title",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2021/vladimir_ferko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Anotácia a vyhodnotenie slovenskej databázy nenávistnej reči"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0018",
|
|
"query": "Nájdi tematický dokument súvisiaci s pojmom pomenované entity.",
|
|
"source_question_id": "q0640",
|
|
"source_category": "topic_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/topics/named-entity/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/named-entity"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Pomenované entity"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0019",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou interpunkcia a osobou Dárius Lindvai?",
|
|
"source_question_id": "q0077",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/darius_lindvai/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dárius Lindvai",
|
|
"interpunkcia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0020",
|
|
"query": "Aký je názov diplomová práca osoby Maroš Harahus?",
|
|
"source_question_id": "q0017",
|
|
"source_category": "fact_title",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/maros_harahus/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Neurónová morfologická anotácia slovenského jazyka"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0021",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou OpenWebUI a osobou Samuel Vasiľ?",
|
|
"source_question_id": "q0520",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/samuel_vasil/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/samuel_vasil"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Samuel Vasiľ",
|
|
"OpenWebUI"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0022",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou strojové učenie a osobou Márk Fehér?",
|
|
"source_question_id": "q0100",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/mark_feher/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/mark_feher"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Márk Fehér",
|
|
"strojové učenie"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0023",
|
|
"query": "Aký je názov bakalárska práca osoby Ondrej Megela?",
|
|
"source_question_id": "q0123",
|
|
"source_category": "fact_title",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Neurónové jazykové modelovanie typu BERT"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0024",
|
|
"query": "V ktorom roku začal študovať Oleh Poiasnik?",
|
|
"source_question_id": "q0322",
|
|
"source_category": "fact_start_year",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/oleh_poiasnik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2022"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0025",
|
|
"query": "Nájdi tematický dokument súvisiaci s pojmom validácia.",
|
|
"source_question_id": "q0683",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/topics/question/validacie/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/question/validacie"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Validácia otázok a odpovedí"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0026",
|
|
"query": "Aký je názov bakalárska práca osoby Michal Stromko?",
|
|
"source_question_id": "q0175",
|
|
"source_category": "fact_title",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Indexovanie slovenského textu pomocou Elasticsearch"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0027",
|
|
"query": "V ktorom roku začal študovať Ján Malinovský?",
|
|
"source_question_id": "q0399",
|
|
"source_category": "fact_start_year",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/jan_malinovsky/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2022"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0028",
|
|
"query": "Aký typ jazykového modelu mal Ondrej Megela natrénovať alebo vyhodnotiť?",
|
|
"source_question_id": "q0131",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"BERT"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0029",
|
|
"query": "Akým spôsobom sa má agent Samuela Džurinu vyhodnotiť?",
|
|
"source_question_id": "q0509",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/samuel_dzurina/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/samuel_dzurina"
|
|
],
|
|
"expected_answer_contains": [
|
|
"dotazníka"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0030",
|
|
"query": "V ktorom roku je pri osobe Ronald Zalacko uvedená bakalárska práca?",
|
|
"source_question_id": "q0459",
|
|
"source_category": "fact_work_year",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2023/ronald_zalacko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2026"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0031",
|
|
"query": "Nájdi dokument o osobe Michal Stromko.",
|
|
"source_question_id": "q0174",
|
|
"source_category": "entity_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Michal Stromko"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0032",
|
|
"query": "V ktorom roku je pri osobe Samuel Vasiľ uvedená bakalárska práca?",
|
|
"source_question_id": "q0517",
|
|
"source_category": "fact_work_year",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/samuel_vasil/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/samuel_vasil"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2026"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0033",
|
|
"query": "Aký webový framework použila Sevval Bulburu pre aplikáciu?",
|
|
"source_question_id": "q0566",
|
|
"source_category": "specific_fact",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/interns/sevval_bulburu/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/sevval_bulburu"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Django"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0034",
|
|
"query": "Nájdi tematický dokument súvisiaci s pojmom seq2seq.",
|
|
"source_question_id": "q0657",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/topics/seq2seq/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/seq2seq"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Prepis postupností"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0035",
|
|
"query": "Ktorý dokument v ZP Wiki sa priamo venuje téme „Strojový preklad“?",
|
|
"source_question_id": "q0633",
|
|
"source_category": "topic_page_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/topics/translation/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/translation"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Strojový preklad"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0036",
|
|
"query": "Ktorá stáž osoby Youssef Ressaissi súvisela s témou ROUGE?",
|
|
"source_question_id": "q0545",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/interns/yussef_ressaissi/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/yussef_ressaissi"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Youssef Ressaissi",
|
|
"ROUGE"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0037",
|
|
"query": "Nájdi dokument o osobe Oleh Poiasnik.",
|
|
"source_question_id": "q0317",
|
|
"source_category": "entity_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/oleh_poiasnik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Oleh Poiasnik"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0038",
|
|
"query": "Aký je názov bakalárska práca osoby Vladyslav Yanchenko?",
|
|
"source_question_id": "q0349",
|
|
"source_category": "fact_title",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/vladyslav_yanchenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/vladyslav_yanchenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Kontinuálne nasadenie a testovanie aplikácie v klaudovom prostredí"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0039",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou slovenský text a osobou Simona Bobrovčanová?",
|
|
"source_question_id": "q0491",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/simona_bobrovcanova/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Simona Bobrovčanová",
|
|
"slovenský text"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0040",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Systém získavania informácií v slovenskom jazyku“.",
|
|
"source_question_id": "q0004",
|
|
"source_category": "topic_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ján Holp",
|
|
"Systém získavania informácií v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0041",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy“.",
|
|
"source_question_id": "q0135",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2018/martin_wencel/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Martin Wencel",
|
|
"Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0042",
|
|
"query": "Ktorá stáž osoby Bogdan Paul Chiș súvisela s témou Triplet extraction?",
|
|
"source_question_id": "q0581",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/interns/bogdan_paul_chis/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/bogdan_paul_chis"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Bogdan Paul Chiș",
|
|
"Triplet extraction"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0043",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou BiLSTM a osobou Dárius Lindvai?",
|
|
"source_question_id": "q0078",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/darius_lindvai/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dárius Lindvai",
|
|
"BiLSTM"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0044",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou webový korpus a osobou Yevhenii Medushivskyi?",
|
|
"source_question_id": "q0448",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/yevhenii_medushivskyi/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Yevhenii Medushivskyi",
|
|
"webový korpus"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0045",
|
|
"query": "V ktorom roku je pri osobe Vladyslav Yanchenko uvedená bakalárska práca?",
|
|
"source_question_id": "q0355",
|
|
"source_category": "fact_work_year",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/vladyslav_yanchenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/vladyslav_yanchenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2025"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0046",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Agentové systémy pomocou veľkého jazykového modelu“.",
|
|
"source_question_id": "q0217",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2021/martin_sarissky/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Martin Šarišský",
|
|
"Agentové systémy pomocou veľkého jazykového modelu"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0047",
|
|
"query": "Aký problém mali malé modely pri práci Simony Bobrovčanovej?",
|
|
"source_question_id": "q0494",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/simona_bobrovcanova/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova"
|
|
],
|
|
"expected_answer_contains": [
|
|
"príliš maly kontext"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0048",
|
|
"query": "Nájdi dokument o osobe Ján Malinovský.",
|
|
"source_question_id": "q0394",
|
|
"source_category": "entity_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/jan_malinovsky/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ján Malinovský"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0049",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Obnovenie interpunkcie pomocou hlbokých neurónových sietí“.",
|
|
"source_question_id": "q0070",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/darius_lindvai/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dárius Lindvai",
|
|
"Obnovenie interpunkcie pomocou hlbokých neurónových sietí"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0050",
|
|
"query": "Nájdi projektovú alebo informačnú stránku „Prepis postupností“.",
|
|
"source_question_id": "q0654",
|
|
"source_category": "topic_page_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/topics/seq2seq/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/seq2seq"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Prepis postupností"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "baseline_single_doc",
|
|
"complexity": "low",
|
|
"concurrency_group": "baseline",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Bežný jedno-dokumentový RAG dopyt na meranie základnej latencie a throughputu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0051",
|
|
"query": "Dárius Lindvai začiatok štúdia rok",
|
|
"source_question_id": "x0562",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/darius_lindvai/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0052",
|
|
"query": "Ján Holp získavanie informácií",
|
|
"source_question_id": "x0526",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ján Holp",
|
|
"získavanie informácií"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0053",
|
|
"query": "Maroš Harahus morfologická anotácia",
|
|
"source_question_id": "x0533",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/maros_harahus/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Maroš Harahus",
|
|
"morfologická anotácia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0054",
|
|
"query": "Ondrej Megela bakalárska práca názov",
|
|
"source_question_id": "x0596",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Neurónové jazykové modelovanie typu BERT"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0055",
|
|
"query": "Michal Stromko bakalárska práca názov",
|
|
"source_question_id": "x0631",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Indexovanie slovenského textu pomocou Elasticsearch"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0056",
|
|
"query": "Neurónový strojový preklad pomocou knižnice Fairseq kto",
|
|
"source_question_id": "x0569",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/dominik_nagy/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dominik Nagy"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0057",
|
|
"query": "Michal Stromko",
|
|
"source_question_id": "x0630",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Michal Stromko"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0058",
|
|
"query": "Lukáš Pokrývka NCCL",
|
|
"source_question_id": "x0557",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/lukas_pokryvka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Lukáš Pokrývka",
|
|
"NCCL"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0059",
|
|
"query": "Filip Tomáš Docker Compose",
|
|
"source_question_id": "x0621",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Filip Tomáš",
|
|
"Docker Compose"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0060",
|
|
"query": "Neurónová morfologická anotácia slovenského jazyka kto",
|
|
"source_question_id": "x0531",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/maros_harahus/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Maroš Harahus"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0061",
|
|
"query": "Ondrej Megela",
|
|
"source_question_id": "x0595",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ondrej Megela"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0062",
|
|
"query": "Tomáš Kuchárik SQuAD",
|
|
"source_question_id": "x0541",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/tomas_kucharik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Tomáš Kuchárik",
|
|
"SQuAD"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0063",
|
|
"query": "Kristián Sopkovič bakalárska práca názov",
|
|
"source_question_id": "x0624",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/kristian_sopkovic/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/kristian_sopkovic"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Model Spacy pre spracovanie prirodzeného jazyka v slovenčine"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0064",
|
|
"query": "Lukáš Pokrývka diplomová práca rok",
|
|
"source_question_id": "x0555",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/lukas_pokryvka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0065",
|
|
"query": "Tomáš Kuchárik diplomová práca rok",
|
|
"source_question_id": "x0539",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/tomas_kucharik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2022"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0066",
|
|
"query": "Dominik Nagy diplomová práca názov",
|
|
"source_question_id": "x0568",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/dominik_nagy/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Neurónový strojový preklad pomocou knižnice Fairseq"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0067",
|
|
"query": "Tomáš Kuchárik diplomová práca názov",
|
|
"source_question_id": "x0537",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/tomas_kucharik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Tvorba korpusu otázok a odpovedí v slovenskom jazyku pomocou strojového prekladu"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0068",
|
|
"query": "Obnovenie interpunkcie pomocou hlbokých neurónových sietí kto",
|
|
"source_question_id": "x0561",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/darius_lindvai/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dárius Lindvai"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0069",
|
|
"query": "Patrik Pavlišin",
|
|
"source_question_id": "x0581",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/patrik_pavlisin/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Patrik Pavlišin"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0070",
|
|
"query": "Lukáš Pokrývka GPU",
|
|
"source_question_id": "x0558",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/lukas_pokryvka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Lukáš Pokrývka",
|
|
"GPU"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0071",
|
|
"query": "Márk Fehér diplomová práca názov",
|
|
"source_question_id": "x0575",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/mark_feher/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/mark_feher"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Klasifikácia textu metódami strojového učenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0072",
|
|
"query": "Ondrej Megela jazykové modelovanie",
|
|
"source_question_id": "x0600",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ondrej Megela",
|
|
"jazykové modelovanie"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0073",
|
|
"query": "Dmytro Ushatenko dialógový systém",
|
|
"source_question_id": "x0594",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2017/dmytro_ushatenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/dmytro_ushatenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dmytro Ushatenko",
|
|
"dialógový systém"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0074",
|
|
"query": "Ondrej Megela BERT",
|
|
"source_question_id": "x0599",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ondrej Megela",
|
|
"BERT"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0075",
|
|
"query": "Ondrej Megela bakalárska práca rok",
|
|
"source_question_id": "x0598",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2020"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "short_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "short",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Krátky používateľský dopyt s minimom slov; testuje režijnú latenciu retrievalu a API."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0076",
|
|
"query": "Aky je nazov diplomova praca osoby David Omasta?",
|
|
"source_question_id": "x0143",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/david_omasta/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/david_omasta"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Generatívne modely pre automatické odpovede na otázky v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0077",
|
|
"query": "Kto sa venuje praci s nazvom „Neuronovy strojovy preklad pomocou kniznice Fairseq“?",
|
|
"source_question_id": "x0084",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/dominik_nagy/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dominik Nagy"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0078",
|
|
"query": "Skús podľa ZP Wiki odpovedať na otázku: Rozlíš rok začiatku štúdia a rok práce osoby Jakub Maruniak?",
|
|
"source_question_id": "x0386",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016",
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0079",
|
|
"query": "Skús podľa ZP Wiki odpovedať na otázku: O čom je práca osoby Patrik Pavlišin?",
|
|
"source_question_id": "x0446",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/patrik_pavlisin/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Metódy automatického prekladu"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0080",
|
|
"query": "Podľa údajov v ZP Wiki: Kto sa venuje práci s názvom „Paralelné trénovanie neurónových sietí“?",
|
|
"source_question_id": "x0394",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/lukas_pokryvka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Lukáš Pokrývka"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0081",
|
|
"query": "Kto sa venuje práci s názvom „Anotácia a rozpozávanie pomenovaných entít v slovenskom jazyku“?",
|
|
"source_question_id": "x0218",
|
|
"source_category": "typo_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Jakub Maruniak"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0082",
|
|
"query": "Prosím, zisti zo ZP Wiki: Nájdi študenta alebo dokument, ktorý rieši tému „Jednoduchý chatbot v jazyku Python“?",
|
|
"source_question_id": "x0455",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2017/dmytro_ushatenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/dmytro_ushatenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dmytro Ushatenko",
|
|
"Jednoduchý chatbot v jazyku Python"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0083",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou MBART a osobou Domnik Nagy?",
|
|
"source_question_id": "x0268",
|
|
"source_category": "typo_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/dominik_nagy/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dominik Nagy",
|
|
"MBART"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0084",
|
|
"query": "Je rok 2016 rokom zaverecnej prace osoby Jakub Maruniak?",
|
|
"source_question_id": "x0047",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016",
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0085",
|
|
"query": "V ktorom roku je pri osobe Jakub Maruiak uvedená diplomová práca?",
|
|
"source_question_id": "x0223",
|
|
"source_category": "typo_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0086",
|
|
"query": "Aky je nazov diplomova praca osoby Jan Holp?",
|
|
"source_question_id": "x0002",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Systém získavania informácií v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0087",
|
|
"query": "O čom je práca osoby Fiip Tomáš?",
|
|
"source_question_id": "x0335",
|
|
"source_category": "typo_robustness",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0088",
|
|
"query": "Rozlis rok zaciatku studia a rok prace osoby Darius Lindvai.",
|
|
"source_question_id": "x0075",
|
|
"source_category": "accent_robustness",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/darius_lindvai/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016",
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0089",
|
|
"query": "Zo ZP Wiki mi prosím zisti: Aký je názov diplomová práca osoby Dávid Omasta?",
|
|
"source_question_id": "x0483",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/david_omasta/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/david_omasta"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Generatívne modely pre automatické odpovede na otázky v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0090",
|
|
"query": "O com je praca osoby Patrik Pavlisin?",
|
|
"source_question_id": "x0106",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/patrik_pavlisin/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Metódy automatického prekladu"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0091",
|
|
"query": "O com je praca osoby Mark Feher?",
|
|
"source_question_id": "x0097",
|
|
"source_category": "accent_robustness",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/mark_feher/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/mark_feher"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Klasifikácia textu metódami strojového učenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0092",
|
|
"query": "Aky je nazov diplomova praca osoby Maros Harahus?",
|
|
"source_question_id": "x0017",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/maros_harahus/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Neurónová morfologická anotácia slovenského jazyka"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0093",
|
|
"query": "Najdi dokument o osobe Patrik Pavlisin.",
|
|
"source_question_id": "x0102",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/patrik_pavlisin/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Patrik Pavlišin"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0094",
|
|
"query": "Prosím, zisti zo ZP Wiki: V ktorom roku je pri osobe Ondrej Megela uvedená bakalárska práca?",
|
|
"source_question_id": "x0467",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2020"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0095",
|
|
"query": "Podľa údajov v ZP Wiki: Nájdi dokument o osobe Michal Stromko?",
|
|
"source_question_id": "x0514",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Michal Stromko"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0096",
|
|
"query": "O čom je práca osoby Dávid Omata?",
|
|
"source_question_id": "x0324",
|
|
"source_category": "typo_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/david_omasta/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/david_omasta"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Generatívne modely pre automatické odpovede na otázky v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0097",
|
|
"query": "Prosím, zisti zo ZP Wiki: O čom je práca osoby Filip Tomáš?",
|
|
"source_question_id": "x0497",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0098",
|
|
"query": "Vieš v ZP Wiki overiť toto: Ktorý dokument alebo študent súvisí s témou Spacy a osobou Kristián Sopkovič?",
|
|
"source_question_id": "x0510",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/kristian_sopkovic/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/kristian_sopkovic"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Kristián Sopkovič",
|
|
"Spacy"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0099",
|
|
"query": "Akymi metrikami mal Darius Lindvai vyhodnocovat model pre jednotlive triedy?",
|
|
"source_question_id": "x0081",
|
|
"source_category": "accent_robustness",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/darius_lindvai/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Precision",
|
|
"Recall"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0100",
|
|
"query": "Je pri osobe Jan Holp uvedeny rok zaciatku studia 2016?",
|
|
"source_question_id": "x0007",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "noisy_query",
|
|
"complexity": "medium",
|
|
"concurrency_group": "noise",
|
|
"expected_context_size": "small",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt s preklepom, chýbajúcou diakritikou alebo konverzačným šumom; sleduje dopad normalizácie na latenciu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0101",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Rostyslav Rodzhuk a bakalárskej práce osoby Samuel Džurina? Uveď oba názvy.",
|
|
"source_question_id": "x1241",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/rostyslav_rodzhuk/README.md",
|
|
"pages/students/2023/samuel_dzurina/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/rostyslav_rodzhuk",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/samuel_dzurina"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Inteligentný agent pre podporu práce špeciálneho pedagóga",
|
|
"Dialógový systém pre podporu firemných procesov"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0102",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Márk Fehér a Alina Vitko.",
|
|
"source_question_id": "x1117",
|
|
"source_category": "multi_document",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/mark_feher/README.md",
|
|
"pages/students/2019/alina_vitko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/mark_feher",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Márk Fehér",
|
|
"Alina Vitko"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0103",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Michal Stromko a bakalárskej práce osoby Vladimír Ferko? Uveď oba názvy.",
|
|
"source_question_id": "x1226",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md",
|
|
"pages/students/2021/vladimir_ferko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Indexovanie slovenského textu pomocou Elasticsearch",
|
|
"Anotácia a vyhodnotenie slovenskej databázy nenávistnej reči"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0104",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Michal Stromko a diplomového projektu osoby Ján Pták? Uveď oba názvy.",
|
|
"source_question_id": "x1243",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md",
|
|
"pages/students/2022/jan_ptak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Indexovanie slovenského textu pomocou Elasticsearch",
|
|
"Agent pre manažment záverečných prác"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0105",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Kristián Sopkovič a diplomového projektu osoby Ján Pták? Uveď oba názvy.",
|
|
"source_question_id": "x1209",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/kristian_sopkovic/README.md",
|
|
"pages/students/2022/jan_ptak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/kristian_sopkovic",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Model Spacy pre spracovanie prirodzeného jazyka v slovenčine",
|
|
"Agent pre manažment záverečných prác"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0106",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Patrik Pavlišin a Vladimír Ferko.",
|
|
"source_question_id": "x1179",
|
|
"source_category": "multi_document",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/patrik_pavlisin/README.md",
|
|
"pages/students/2021/vladimir_ferko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Patrik Pavlišin",
|
|
"Vladimír Ferko"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0107",
|
|
"query": "Ktoré z dokumentov o osobách Tetiana Mohorian a Rostyslav Rodzhuk súvisia s témou LLM? Uveď obe osoby.",
|
|
"source_question_id": "x1301",
|
|
"source_category": "multi_document_topic",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/tetiana_mohorian/README.md",
|
|
"pages/students/2023/rostyslav_rodzhuk/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/tetiana_mohorian",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/rostyslav_rodzhuk"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Tetiana Mohorian",
|
|
"Rostyslav Rodzhuk",
|
|
"LLM"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0108",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Daniil Huzenko a diplomového projektu osoby Ján Pták? Uveď oba názvy.",
|
|
"source_question_id": "x1247",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/daniil_huzenko/README.md",
|
|
"pages/students/2022/jan_ptak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/daniil_huzenko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Testovanie hybridného klaudu s využiťím kombinácie verejného a privátneho riešenia",
|
|
"Agent pre manažment záverečných prác"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0109",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Vladimír Ferko a diplomového projektu osoby Ján Pták? Uveď oba názvy.",
|
|
"source_question_id": "x1270",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2021/vladimir_ferko/README.md",
|
|
"pages/students/2022/jan_ptak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Anotácia a vyhodnotenie slovenskej databázy nenávistnej reči",
|
|
"Agent pre manažment záverečných prác"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0110",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Dominik Nagy a Ján Pták.",
|
|
"source_question_id": "x1156",
|
|
"source_category": "multi_document",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/dominik_nagy/README.md",
|
|
"pages/students/2022/jan_ptak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dominik Nagy",
|
|
"Ján Pták"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0111",
|
|
"query": "Aké sú názvy diplomového projektu osoby Ján Malinovský a bakalárskej práce osoby Ronald Zalacko? Uveď oba názvy.",
|
|
"source_question_id": "x1204",
|
|
"source_category": "comparison",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/jan_malinovsky/README.md",
|
|
"pages/students/2023/ronald_zalacko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Grafové vyhľadávanie pre podporu vzdelávania",
|
|
"Skórovací online systém pre robotické súťaže"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0112",
|
|
"query": "Ktoré z dokumentov o osobách Oleh Poiasnik a Maksym Zatirka súvisia s témou znalostný graf? Uveď obe osoby.",
|
|
"source_question_id": "x1283",
|
|
"source_category": "multi_document_topic",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/oleh_poiasnik/README.md",
|
|
"pages/students/2023/maksym_zatirka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/maksym_zatirka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Oleh Poiasnik",
|
|
"Maksym Zatirka",
|
|
"znalostný graf"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0113",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Tetiana Mohorian a Cesar Abascal Gutierrez.",
|
|
"source_question_id": "x1149",
|
|
"source_category": "multi_document",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/tetiana_mohorian/README.md",
|
|
"pages/interns/cesar_gutierrez/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/tetiana_mohorian",
|
|
"https://zp.kemt.fei.tuke.sk/interns/cesar_gutierrez"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Tetiana Mohorian",
|
|
"Cesar Abascal Gutierrez"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0114",
|
|
"query": "Aké sú názvy diplomovej práce osoby Jakub Maruniak a bakalárskej práce osoby Ondrej Megela? Uveď oba názvy.",
|
|
"source_question_id": "x1217",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md",
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak",
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Anotácia a rozpoznávanie pomenovaných entít v slovenskom jazyku",
|
|
"Neurónové jazykové modelovanie typu BERT"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0115",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Vladyslav Yanchenko a Ján Malinovský.",
|
|
"source_question_id": "x1138",
|
|
"source_category": "multi_document",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/vladyslav_yanchenko/README.md",
|
|
"pages/students/2022/jan_malinovsky/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/vladyslav_yanchenko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Vladyslav Yanchenko",
|
|
"Ján Malinovský"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0116",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Valerii Kutsenko a Oliver Pejic.",
|
|
"source_question_id": "x1101",
|
|
"source_category": "multi_document",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/valerii_kutsenko/README.md",
|
|
"pages/interns/oliver_pejic/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko",
|
|
"https://zp.kemt.fei.tuke.sk/interns/oliver_pejic"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Valerii Kutsenko",
|
|
"Oliver Pejic"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0117",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Ondrej Megela a bakalárskej práce osoby Michal Stromko? Uveď oba názvy.",
|
|
"source_question_id": "x1219",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md",
|
|
"pages/students/2019/michal_stromko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Neurónové jazykové modelovanie typu BERT",
|
|
"Indexovanie slovenského textu pomocou Elasticsearch"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0118",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Michal Stromko a diplomovej práce osoby Valerii Kutsenko? Uveď oba názvy.",
|
|
"source_question_id": "x1210",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md",
|
|
"pages/students/2022/valerii_kutsenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Indexovanie slovenského textu pomocou Elasticsearch",
|
|
"Zlepšenie vyhľadávania pomocou znalostných grafov"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0119",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Filip Tomáš a Oleksandr Dorybohov.",
|
|
"source_question_id": "x1130",
|
|
"source_category": "multi_document",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md",
|
|
"pages/students/2023/oleksandr_dorybohov/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Filip Tomáš",
|
|
"Oleksandr Dorybohov"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0120",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Dárius Lindvai a Oleksandr Dorybohov.",
|
|
"source_question_id": "x1187",
|
|
"source_category": "multi_document",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/darius_lindvai/README.md",
|
|
"pages/students/2023/oleksandr_dorybohov/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dárius Lindvai",
|
|
"Oleksandr Dorybohov"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0121",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Márk Fehér a Martin Wencel.",
|
|
"source_question_id": "x1188",
|
|
"source_category": "multi_document",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/mark_feher/README.md",
|
|
"pages/students/2018/martin_wencel/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/mark_feher",
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Márk Fehér",
|
|
"Martin Wencel"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0122",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Alina Vitko a Martin Šarišský.",
|
|
"source_question_id": "x1196",
|
|
"source_category": "multi_document",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/alina_vitko/README.md",
|
|
"pages/students/2021/martin_sarissky/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Alina Vitko",
|
|
"Martin Šarišský"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0123",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Filip Tomáš a bakalárskej práce osoby Vladyslav Yanchenko? Uveď oba názvy.",
|
|
"source_question_id": "x1227",
|
|
"source_category": "comparison",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md",
|
|
"pages/students/2022/vladyslav_yanchenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/vladyslav_yanchenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia",
|
|
"Kontinuálne nasadenie a testovanie aplikácie v klaudovom prostredí"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0124",
|
|
"query": "Aké sú názvy bakalárskej práce osoby Ondrej Megela a diplomového projektu osoby Ján Pták? Uveď oba názvy.",
|
|
"source_question_id": "x1207",
|
|
"source_category": "comparison",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md",
|
|
"pages/students/2022/jan_ptak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Neurónové jazykové modelovanie typu BERT",
|
|
"Agent pre manažment záverečných prác"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0125",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Márk Fehér a Youssef Ressaissi.",
|
|
"source_question_id": "x1151",
|
|
"source_category": "multi_document",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/mark_feher/README.md",
|
|
"pages/interns/yussef_ressaissi/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/mark_feher",
|
|
"https://zp.kemt.fei.tuke.sk/interns/yussef_ressaissi"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Márk Fehér",
|
|
"Youssef Ressaissi"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0126",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Ján Pták a Simona Bobrovčanová.",
|
|
"source_question_id": "x1092",
|
|
"source_category": "multi_document",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/jan_ptak/README.md",
|
|
"pages/students/2023/simona_bobrovcanova/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ján Pták",
|
|
"Simona Bobrovčanová"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0127",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Ondrej Megela a Tetiana Mohorian.",
|
|
"source_question_id": "x1191",
|
|
"source_category": "multi_document",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md",
|
|
"pages/students/2022/tetiana_mohorian/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/tetiana_mohorian"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ondrej Megela",
|
|
"Tetiana Mohorian"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0128",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Ján Holp a Simona Bobrovčanová.",
|
|
"source_question_id": "x1162",
|
|
"source_category": "multi_document",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md",
|
|
"pages/students/2023/simona_bobrovcanova/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/simona_bobrovcanova"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ján Holp",
|
|
"Simona Bobrovčanová"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0129",
|
|
"query": "Nájdi v ZP Wiki dokumenty o osobách Artem Yatsenko a Matej Ščišľak.",
|
|
"source_question_id": "x1089",
|
|
"source_category": "multi_document",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/artem_yatsenko/README.md",
|
|
"pages/students/2021/matej_scislak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/artem_yatsenko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Artem Yatsenko",
|
|
"Matej Ščišľak"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0130",
|
|
"query": "Ktoré z dokumentov o osobách Tomáš Kuchárik a Dominik Nagy súvisia s témou strojový preklad? Uveď obe osoby.",
|
|
"source_question_id": "x1286",
|
|
"source_category": "multi_document_topic",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/tomas_kucharik/README.md",
|
|
"pages/students/2016/dominik_nagy/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Tomáš Kuchárik",
|
|
"Dominik Nagy",
|
|
"strojový preklad"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "multi_document",
|
|
"complexity": "high",
|
|
"concurrency_group": "multi_doc",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt vyžadujúci viac dokumentov; vhodný na meranie diverzifikácie, fusion a väčšieho RAG kontextu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0131",
|
|
"query": "Akú výslednú známku dostala osoba Simona Bobrovčanová za svoju záverečnú prácu?",
|
|
"source_question_id": "x0973",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/simona_bobrovcanova/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0132",
|
|
"query": "Akú výslednú známku dostala osoba Dmytro Ushatenko za svoju záverečnú prácu?",
|
|
"source_question_id": "x0857",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2017/dmytro_ushatenko/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0133",
|
|
"query": "Aké bolo číslo protokolu o obhajobe záverečnej práce osoby Yevhenii Medushivskyi?",
|
|
"source_question_id": "x0980",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/yevhenii_medushivskyi/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0134",
|
|
"query": "Aké bolo číslo protokolu o obhajobe záverečnej práce osoby Ondrej Megela?",
|
|
"source_question_id": "x0872",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0135",
|
|
"query": "Aké bolo číslo protokolu o obhajobe záverečnej práce osoby Ronald Zalacko?",
|
|
"source_question_id": "x0964",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2023/ronald_zalacko/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0136",
|
|
"query": "Akú výslednú známku dostala osoba Alina Vitko za svoju záverečnú prácu?",
|
|
"source_question_id": "x0873",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/alina_vitko/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0137",
|
|
"query": "Aké bolo číslo protokolu o obhajobe záverečnej práce osoby Dmytro Ushatenko?",
|
|
"source_question_id": "x0860",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2017/dmytro_ushatenko/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0138",
|
|
"query": "Koľko bodov získala osoba Samuel Džurina pri obhajobe svojej záverečnej práce?",
|
|
"source_question_id": "x0970",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/samuel_dzurina/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0139",
|
|
"query": "V akej miestnosti prebiehala obhajoba záverečnej práce osoby Daniil Huzenko?",
|
|
"source_question_id": "x0919",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/daniil_huzenko/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0140",
|
|
"query": "Aké bolo číslo protokolu o obhajobe záverečnej práce osoby Márk Fehér?",
|
|
"source_question_id": "x0844",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/mark_feher/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0141",
|
|
"query": "V akej miestnosti prebiehala obhajoba záverečnej práce osoby Samuel Vasiľ?",
|
|
"source_question_id": "x0935",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/samuel_vasil/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0142",
|
|
"query": "Akú výslednú známku dostala osoba Oleksandr Dorybohov za svoju záverečnú prácu?",
|
|
"source_question_id": "x0957",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2023/oleksandr_dorybohov/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0143",
|
|
"query": "Akú výslednú známku dostala osoba Rostyslav Rodzhuk za svoju záverečnú prácu?",
|
|
"source_question_id": "x0965",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/rostyslav_rodzhuk/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0144",
|
|
"query": "Koľko bodov získala osoba Vladyslav Yanchenko pri obhajobe svojej záverečnej práce?",
|
|
"source_question_id": "x0946",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/vladyslav_yanchenko/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0145",
|
|
"query": "Aké bolo číslo protokolu o obhajobe záverečnej práce osoby Patrik Pavlišin?",
|
|
"source_question_id": "x0852",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/patrik_pavlisin/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0146",
|
|
"query": "Akú výslednú známku dostala osoba Kristián Sopkovič za svoju záverečnú prácu?",
|
|
"source_question_id": "x0885",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/kristian_sopkovic/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0147",
|
|
"query": "V akej miestnosti prebiehala obhajoba záverečnej práce osoby Ján Pták?",
|
|
"source_question_id": "x0927",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/jan_ptak/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0148",
|
|
"query": "Akú výslednú známku dostala osoba Ján Holp za svoju záverečnú prácu?",
|
|
"source_question_id": "x0833",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0149",
|
|
"query": "V akej miestnosti prebiehala obhajoba záverečnej práce osoby Yevhenii Medushivskyi?",
|
|
"source_question_id": "x0979",
|
|
"source_category": "no_answer",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2023/yevhenii_medushivskyi/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0150",
|
|
"query": "Akú výslednú známku dostala osoba Maksym Zatirka za svoju záverečnú prácu?",
|
|
"source_question_id": "x0953",
|
|
"source_category": "no_answer",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2023/maksym_zatirka/README.md"
|
|
],
|
|
"expected_source_urls": [],
|
|
"expected_answer_contains": [],
|
|
"should_answer": false,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "no_answer",
|
|
"complexity": "medium",
|
|
"concurrency_group": "negative",
|
|
"expected_context_size": "medium",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Negatívny dopyt bez podporenej odpovede; meria náklady groundingu a abstention správania."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0151",
|
|
"query": "Kto sa venuje práci s názvom „Neurónový strojový preklad pomocou knižnice Fairseq“? Odpoveď podlož presným source_url zo ZP Wiki.",
|
|
"source_question_id": "x0724",
|
|
"source_category": "citation_required",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/dominik_nagy/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dominik Nagy"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0152",
|
|
"query": "Tomáš Kuchárik Question Answering",
|
|
"source_question_id": "x0540",
|
|
"source_category": "concise_query",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/tomas_kucharik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Tomáš Kuchárik",
|
|
"Question Answering"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0153",
|
|
"query": "Kto je autorom dokumentu ZP Wiki o osobe Eduard Matovka?",
|
|
"source_question_id": "x0784",
|
|
"source_category": "fact_author",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2021/eduard_matovka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/eduard_matovka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Daniel Hladek"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0154",
|
|
"query": "Ktory dokument alebo student suvisi s temou neuronove siete a osobou Ondrej Megela?",
|
|
"source_question_id": "x0130",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/ondrej_megela/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ondrej Megela",
|
|
"neurónové siete"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0155",
|
|
"query": "O čom je práca osoby Filip Tomáš?",
|
|
"source_question_id": "q0157",
|
|
"source_category": "paraphrase",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0156",
|
|
"query": "Začala osoba Vladimír Ferko študovať v roku 2064? Ak nie, uveď správny rok začiatku štúdia.",
|
|
"source_question_id": "x1038",
|
|
"source_category": "negative_verification",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2021/vladimir_ferko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0157",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou morfologická anotácia a osobou Maroš Harahus?",
|
|
"source_question_id": "q0022",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/maros_harahus/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Maroš Harahus",
|
|
"morfologická anotácia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0158",
|
|
"query": "Prosím, zisti zo ZP Wiki: V ktorom roku začal študovať Jakub Maruniak?",
|
|
"source_question_id": "x0383",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0159",
|
|
"query": "Najdi dokument o osobe Maros Harahus.",
|
|
"source_question_id": "x0016",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/maros_harahus/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Maroš Harahus"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0160",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou dialógový systém a osobou Dmytro Ushatenko? Odpoveď podlož presným source_url zo ZP Wiki.",
|
|
"source_question_id": "x0760",
|
|
"source_category": "citation_required",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2017/dmytro_ushatenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/dmytro_ushatenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dmytro Ushatenko",
|
|
"dialógový systém"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0161",
|
|
"query": "O čom je práca osoby Michal Stromko?",
|
|
"source_question_id": "q0178",
|
|
"source_category": "paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Indexovanie slovenského textu pomocou Elasticsearch"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0162",
|
|
"query": "Aký source_url patrí k stránke „Anotácia textových dát“?",
|
|
"source_question_id": "q0653",
|
|
"source_category": "source_url",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/topics/prodigy/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/prodigy"
|
|
],
|
|
"expected_answer_contains": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/prodigy"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0163",
|
|
"query": "Je pri osobe Oleh Poiasnik uvedená diplomová práca v roku 2064? Ak nie, uveď správny rok.",
|
|
"source_question_id": "x1059",
|
|
"source_category": "negative_verification",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/oleh_poiasnik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2027"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0164",
|
|
"query": "Skús podľa ZP Wiki odpovedať na otázku: Aký je názov bakalárska práca osoby Filip Tomáš?",
|
|
"source_question_id": "x0494",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0165",
|
|
"query": "Kto je autorom dokumentu ZP Wiki o osobe Patrik Pavlišin?",
|
|
"source_question_id": "x0773",
|
|
"source_category": "fact_author",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/patrik_pavlisin/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Daniel Hladek"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0166",
|
|
"query": "Ktorá stáž osoby Sevval Bulburu súvisela s témou SMOTE?",
|
|
"source_question_id": "q0564",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/interns/sevval_bulburu/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/sevval_bulburu"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Sevval Bulburu",
|
|
"SMOTE"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0167",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Kontinuálne nasadenie a testovanie aplikácie v klaudovom prostredí“.",
|
|
"source_question_id": "q0351",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/vladyslav_yanchenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/vladyslav_yanchenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Vladyslav Yanchenko",
|
|
"Kontinuálne nasadenie a testovanie aplikácie v klaudovom prostredí"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0168",
|
|
"query": "Je pri osobe Matej Ščišľak uvedený rok začiatku štúdia 2021?",
|
|
"source_question_id": "q0235",
|
|
"source_category": "verification",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2021/matej_scislak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0169",
|
|
"query": "Prosím, zisti zo ZP Wiki: Je rok 2016 rokom záverečnej práce osoby Lukáš Pokrývka?",
|
|
"source_question_id": "x0401",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/lukas_pokryvka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016",
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0170",
|
|
"query": "V ktorom roku začal študovať Lukáš Pokrývka?",
|
|
"source_question_id": "q0057",
|
|
"source_category": "fact_start_year",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/lukas_pokryvka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "larger_result_limit",
|
|
"complexity": "medium",
|
|
"concurrency_group": "limit10",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Rovnaký typ RAG dopytu, ale s limit=10; testuje vplyv väčšieho retrieval result setu na latenciu a kontext."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0171",
|
|
"query": "Akým nástrojom mal Daniil Huzenko konfigurovať alebo resetovať klaster?",
|
|
"source_question_id": "q0377",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/daniil_huzenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/daniil_huzenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ansible"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0172",
|
|
"query": "Aké tri hlavné ciele mal Cesar Abascal Gutierrez?",
|
|
"source_question_id": "q0575",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/interns/cesar_gutierrez/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/interns/cesar_gutierrez"
|
|
],
|
|
"expected_answer_contains": [
|
|
"recognize unknown named entities",
|
|
"manually annotated training set",
|
|
"annotation schema"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0173",
|
|
"query": "Aké slovné vektory boli zahrnuté do modelu Spacy?",
|
|
"source_question_id": "q0613",
|
|
"source_category": "specific_fact",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/topics/spacy/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/spacy"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Fasttext Common Crawl"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0174",
|
|
"query": "Nájdi tematický dokument súvisiaci s pojmom strojový preklad.",
|
|
"source_question_id": "q0635",
|
|
"source_category": "topic_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/topics/translation/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/translation"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Strojový preklad"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0175",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou RAG a osobou Valerii Kutsenko?",
|
|
"source_question_id": "q0314",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/valerii_kutsenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Valerii Kutsenko",
|
|
"RAG"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0176",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Anotácia a rozpoznávanie pomenovaných entít v slovenskom jazyku“.",
|
|
"source_question_id": "q0041",
|
|
"source_category": "topic_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Jakub Maruniak",
|
|
"Anotácia a rozpoznávanie pomenovaných entít v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0177",
|
|
"query": "Nájdi tematický dokument súvisiaci s pojmom Wikipedia.",
|
|
"source_question_id": "q0620",
|
|
"source_category": "topic_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/topics/question/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/question"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Question Answering"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0178",
|
|
"query": "Z akej oblasti má Ján Malinovský zostaviť malý znalostný graf?",
|
|
"source_question_id": "q0407",
|
|
"source_category": "specific_fact",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2022/jan_malinovsky/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
|
|
],
|
|
"expected_answer_contains": [
|
|
"vzdelávania"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0179",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Systémy pre generovanie odpovede na otázku v prirodzenom jazyku“.",
|
|
"source_question_id": "q0188",
|
|
"source_category": "topic_lookup",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/alina_vitko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Alina Vitko",
|
|
"Systémy pre generovanie odpovede na otázku v prirodzenom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0180",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou právo a osobou Oleksandr Dorybohov?",
|
|
"source_question_id": "q0476",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2023/oleksandr_dorybohov/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Oleksandr Dorybohov",
|
|
"právo"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0181",
|
|
"query": "Koľko jednotiek Raspberry Pi 4 mal Filip Tomáš k dispozícii pri návrhu klastra?",
|
|
"source_question_id": "q0163",
|
|
"source_category": "specific_fact",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"5"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0182",
|
|
"query": "Aké dve adaptačné techniky mala Tetiana Mohorian použiť podľa návrhu témy?",
|
|
"source_question_id": "q0346",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2022/tetiana_mohorian/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/tetiana_mohorian"
|
|
],
|
|
"expected_answer_contains": [
|
|
"prompting",
|
|
"LORA"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0183",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou scikit a osobou Márk Fehér?",
|
|
"source_question_id": "q0101",
|
|
"source_category": "keyword_grounding",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/mark_feher/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/mark_feher"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Márk Fehér",
|
|
"scikit"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0184",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou vyhľadávanie a osobou Michal Stromko?",
|
|
"source_question_id": "q0181",
|
|
"source_category": "keyword_grounding",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/michal_stromko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Michal Stromko",
|
|
"vyhľadávanie"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0185",
|
|
"query": "Koľko článkov mal Artem Yatsenko minimálne anotovať cez webové rozhranie?",
|
|
"source_question_id": "q0213",
|
|
"source_category": "specific_fact",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2019/artem_yatsenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/artem_yatsenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"20"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "section_context_heavy",
|
|
"complexity": "high",
|
|
"concurrency_group": "context_heavy",
|
|
"expected_context_size": "large",
|
|
"repetitions": 1,
|
|
"warmup_eligible": true,
|
|
"description": "Dopyt zameraný na detail v sekcii alebo tematický kontext; vhodný na meranie section-lead expansion overheadu."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0186",
|
|
"query": "Vieš v ZP Wiki overiť toto: Nájdi dokument o osobe Jakub Maruniak?",
|
|
"source_question_id": "x0378",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jakub_maruniak/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Jakub Maruniak"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0187",
|
|
"query": "Podľa údajov v ZP Wiki: Ktorý dokument alebo študent súvisí s témou grafové neurónové siete a osobou Maroš Harahus?",
|
|
"source_question_id": "x0364",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/maros_harahus/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Maroš Harahus",
|
|
"grafové neurónové siete"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0188",
|
|
"query": "Aky je nazov bakalarska praca osoby Patrik Pavlisin?",
|
|
"source_question_id": "x0103",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/patrik_pavlisin/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Metódy automatického prekladu"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0189",
|
|
"query": "O čom je práca osoby Martin Wencel?",
|
|
"source_question_id": "q0136",
|
|
"source_category": "paraphrase",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2018/martin_wencel/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0190",
|
|
"query": "Nájdi študenta alebo dokument, ktorý rieši tému „Klasifikácia webových dát pre lepšie jazykové modelovanie“.",
|
|
"source_question_id": "q0278",
|
|
"source_category": "topic_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2021/matej_novotny/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Matej Novotný",
|
|
"Klasifikácia webových dát pre lepšie jazykové modelovanie"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0191",
|
|
"query": "Systémy pre generovanie odpovede na otázku v prirodzenom jazyku kto",
|
|
"source_question_id": "x0639",
|
|
"source_category": "concise_query",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/alina_vitko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Alina Vitko"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0192",
|
|
"query": "O čom je práca osoby Dmytro Ushatenko? Odpoveď podlož presným source_url zo ZP Wiki.",
|
|
"source_question_id": "x0756",
|
|
"source_category": "citation_required",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2017/dmytro_ushatenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/dmytro_ushatenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Jednoduchý chatbot v jazyku Python"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0193",
|
|
"query": "Kto je autorom stránky ZP Wiki „Anotácia pomenovaných entít“?",
|
|
"source_question_id": "x0810",
|
|
"source_category": "fact_author",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/topics/named-entity/navod/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/named-entity/navod"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Daniel Hladek"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0194",
|
|
"query": "Aký je názov diplmová práca osoby Ján Holp?",
|
|
"source_question_id": "x0181",
|
|
"source_category": "typo_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Systém získavania informácií v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0195",
|
|
"query": "Kto sa venuje práci s názvom „Tvorba korpusu otázok a odpovedí v slovenskom jazyku pomocou strojvého prekladu“?",
|
|
"source_question_id": "x0207",
|
|
"source_category": "typo_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2016/tomas_kucharik/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Tomáš Kuchárik"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0196",
|
|
"query": "Aky je nazov bakalarska praca osoby Filip Tomas?",
|
|
"source_question_id": "x0154",
|
|
"source_category": "accent_robustness",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2019/filip_tomas/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Aplikačné testy v systémoch kontinuálnej integrácie a nasadenia"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0197",
|
|
"query": "Nájdi tematický dokument súvisiaci s pojmom chatbot.",
|
|
"source_question_id": "q0601",
|
|
"source_category": "topic_lookup",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/topics/chatbot/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/chatbot"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dialógový systém"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0198",
|
|
"query": "Ktory dokument alebo student suvisi s temou slovencina a osobou David Omasta?",
|
|
"source_question_id": "x0150",
|
|
"source_category": "accent_robustness",
|
|
"split": "dev",
|
|
"expected_documents": [
|
|
"pages/students/2018/david_omasta/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/david_omasta"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dávid Omasta",
|
|
"slovenčina"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0199",
|
|
"query": "Prosím, zisti zo ZP Wiki: Nájdi študenta alebo dokument, ktorý rieši tému „Paralelné trénovanie neurónových sietí“?",
|
|
"source_question_id": "x0395",
|
|
"source_category": "conversational_paraphrase",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2016/lukas_pokryvka/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Lukáš Pokrývka",
|
|
"Paralelné trénovanie neurónových sietí"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
},
|
|
{
|
|
"id": "p0200",
|
|
"query": "Ktorý dokument alebo študent súvisí s témou Python a osobou Dmytro Ushaenko?",
|
|
"source_question_id": "x0297",
|
|
"source_category": "typo_robustness",
|
|
"split": "test",
|
|
"expected_documents": [
|
|
"pages/students/2017/dmytro_ushatenko/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/dmytro_ushatenko"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Dmytro Ushatenko",
|
|
"Python"
|
|
],
|
|
"should_answer": true,
|
|
"request": {
|
|
"endpoint": "/rag",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"performance": {
|
|
"profile": "repeated_hot_query",
|
|
"complexity": "low",
|
|
"concurrency_group": "hot",
|
|
"expected_context_size": "small",
|
|
"repetitions": 5,
|
|
"warmup_eligible": true,
|
|
"description": "Query určené na opakované spúšťanie po warmupe; umožňuje porovnať cold/warm správanie a stabilitu latencie."
|
|
}
|
|
}
|
|
]
|