382 lines
14 KiB
JSON
382 lines
14 KiB
JSON
{
|
|
"schema": 1,
|
|
"split": "DEV",
|
|
"scope": "Audited, previously seen regressions and paraphrases; no independent-accuracy claim.",
|
|
"cases": [
|
|
{
|
|
"id": "D01",
|
|
"category": "audit_regression",
|
|
"question": "Vysvetli mi jednoducho, na čom pracoval Ján Malinovský.",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2022/jan_malinovsky/README.md",
|
|
"sha256": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"start": 44,
|
|
"end": 44,
|
|
"quote": "- Vytvorený jednoduchý agent, ktorý vyhľadáva v znalostnom grafe a na základe týchto informácií generuje odpoveď.",
|
|
"requirements": [
|
|
"Ján Malinovský made an agent retrieving from the knowledge graph and generating an answer."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"Vytvorený jednoduchý agent"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"path": "pages/students/2022/jan_malinovsky/README.md",
|
|
"sha256": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"start": 18,
|
|
"end": 18,
|
|
"quote": "Grafové vyhľadávanie pre podporu vzdelávania",
|
|
"requirements": [
|
|
"The topic is graph retrieval supporting education."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"Grafové vyhľadávanie"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": [],
|
|
"max_words": 120
|
|
},
|
|
{
|
|
"id": "D02",
|
|
"category": "audit_regression",
|
|
"question": "Ako pokročil Ján Malinovský so svojou prácou?",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2022/jan_malinovsky/README.md",
|
|
"sha256": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"start": 44,
|
|
"end": 44,
|
|
"quote": "- Vytvorený jednoduchý agent, ktorý vyhľadáva v znalostnom grafe a na základe týchto informácií generuje odpoveď.",
|
|
"requirements": [
|
|
"Ján Malinovský made an agent retrieving from the knowledge graph and generating an answer."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"Vytvorený jednoduchý agent"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": [
|
|
"Distinguish actual work from future plans."
|
|
]
|
|
},
|
|
{
|
|
"id": "D03",
|
|
"category": "audit_regression",
|
|
"question": "Čomu sa venuje diplomová práca Jána Holpa? Stručne predstav jej zadanie a uveď zdroj.",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2016/jan_holp/README.md",
|
|
"sha256": "80d6a8492766e5bfa61673a372e5f059d30c8e335de39842866c51485960e4d3",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"start": 21,
|
|
"end": 24,
|
|
"quote": "1. Vypracujte prehľad metód ohodnotenia dokumentov v systémoch získavania informácií.\n2. Implementujte jednoduchý systém pre získavanie informácií v slovenskom jazyku, v ktorom využijete ohodnotenie dokumentov.\n3. Navrhnite a vypracujte experimenty, v ktorých vyhodnotíte vybrané metódy odhodnotenia dokumentov.\n4. Navrhnite možné zlepšenia presnosti vyhľadávania.",
|
|
"requirements": [
|
|
"All FOUR thesis assignment goals: review document-ranking methods; implement Slovak retrieval using ranking; design/evaluate experiments; suggest retrieval-accuracy improvements. Assignment is not a completed result."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"ohodnotenia"
|
|
],
|
|
[
|
|
"experimenty"
|
|
],
|
|
[
|
|
"zlepšenia"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": [],
|
|
"max_words": 120
|
|
},
|
|
{
|
|
"id": "D04",
|
|
"category": "audit_regression",
|
|
"question": "Aké bolo zadanie diplomovej práce Jána Holpa?",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2016/jan_holp/README.md",
|
|
"sha256": "80d6a8492766e5bfa61673a372e5f059d30c8e335de39842866c51485960e4d3",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"start": 21,
|
|
"end": 24,
|
|
"quote": "1. Vypracujte prehľad metód ohodnotenia dokumentov v systémoch získavania informácií.\n2. Implementujte jednoduchý systém pre získavanie informácií v slovenskom jazyku, v ktorom využijete ohodnotenie dokumentov.\n3. Navrhnite a vypracujte experimenty, v ktorých vyhodnotíte vybrané metódy odhodnotenia dokumentov.\n4. Navrhnite možné zlepšenia presnosti vyhľadávania.",
|
|
"requirements": [
|
|
"All FOUR thesis assignment goals: review document-ranking methods; implement Slovak retrieval using ranking; design/evaluate experiments; suggest retrieval-accuracy improvements. Assignment is not a completed result."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"ohodnotenia"
|
|
],
|
|
[
|
|
"experimenty"
|
|
],
|
|
[
|
|
"zlepšenia"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": []
|
|
},
|
|
{
|
|
"id": "D05",
|
|
"category": "audit_regression",
|
|
"question": "Aké technické problémy riešil Ondrej Megela?",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2018/ondrej_megela/README.md",
|
|
"sha256": "969ff2e1a436b8f957e7168cf5eb678196b99aeda17701c502b016441f644065",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
|
|
"start": 311,
|
|
"end": 311,
|
|
"quote": "- Natrénovaný model wiki103 na stroji Quadro. Problém sa vyriešil vypnutím GPU pri trénovaní,",
|
|
"requirements": [
|
|
"Report the GPU training problem and its recorded workaround under the correct bachelor work."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"wiki103"
|
|
],
|
|
[
|
|
"GPU"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"path": "pages/students/2018/ondrej_megela/README.md",
|
|
"sha256": "969ff2e1a436b8f957e7168cf5eb678196b99aeda17701c502b016441f644065",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
|
|
"start": 51,
|
|
"end": 51,
|
|
"quote": "- Vytvorený notebook pre vyhodnotenie generovanie otázok pomocou BLEU. Zatiaľ nefunguje, pretože asi je potrebné spoávne zoradiť vygenerovanú otázku a referenciu, Ku paragrafu môže existovať viacero rôznych otázok, preto je tažké ich porovnať pomocou BLEU ngramov.",
|
|
"requirements": [
|
|
"Diploma thesis 2023: the BLEU evaluation notebook exists but is not working; ordering generated/reference questions and multiple questions per paragraph complicate evaluation. Do not report absence of technical problems for this work."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"BLEU"
|
|
],
|
|
[
|
|
"nefunguje"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": [
|
|
"Do not ask the user to select a work for this broad overview; group by works and distinguish known from missing details."
|
|
]
|
|
},
|
|
{
|
|
"id": "D06",
|
|
"category": "audit_regression",
|
|
"question": "Aké metódy používal Ján Holp pri diplomovom projekte v roku 2020?",
|
|
"expected_status": "ambiguous",
|
|
"evidence": [],
|
|
"response_requirements": [
|
|
"Two distinct projects in 2020 exist; ask which one, listing both."
|
|
]
|
|
},
|
|
{
|
|
"id": "D07",
|
|
"category": "audit_regression",
|
|
"question": "Aké metódy používal Ján Holp pri oboch diplomových projektoch v roku 2020?",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2016/jan_holp/README.md",
|
|
"sha256": "80d6a8492766e5bfa61673a372e5f059d30c8e335de39842866c51485960e4d3",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"start": 73,
|
|
"end": 73,
|
|
"quote": "- Vyriešené technické problémy s cassandrou. Vieme indexovať z Cassandra do ES.",
|
|
"requirements": [
|
|
"Under project 2: Cassandra to ES indexing."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"Cassandra"
|
|
],
|
|
[
|
|
"ES"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"path": "pages/students/2016/jan_holp/README.md",
|
|
"sha256": "80d6a8492766e5bfa61673a372e5f059d30c8e335de39842866c51485960e4d3",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"start": 146,
|
|
"end": 146,
|
|
"quote": "- funguje databáza Cassandra",
|
|
"requirements": [
|
|
"Under unnumbered diploma project: Cassandra works and file indexing works."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"Cassandra"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": [
|
|
"Both projects separately; never mix their activities."
|
|
]
|
|
},
|
|
{
|
|
"id": "D08",
|
|
"category": "audit_regression",
|
|
"question": "Kedy mal Ján Malinovský naposledy stretnutie a čo na ňom riešili?",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2022/jan_malinovsky/README.md",
|
|
"sha256": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"start": 34,
|
|
"end": 34,
|
|
"quote": "Stretnutie 14.5.2026",
|
|
"requirements": [
|
|
"Most recent complete record date 2026-05-14; describe its content."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"2026"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"path": "pages/students/2022/jan_malinovsky/README.md",
|
|
"sha256": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"start": 44,
|
|
"end": 44,
|
|
"quote": "- Vytvorený jednoduchý agent, ktorý vyhľadáva v znalostnom grafe a na základe týchto informácií generuje odpoveď.",
|
|
"requirements": [
|
|
"Ján Malinovský made an agent retrieving from the knowledge graph and generating an answer."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"Vytvorený jednoduchý agent"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": [
|
|
"RAGAS/DeepEval from a general undated backlog must NOT be attributed to this meeting. Record date is not execution date."
|
|
]
|
|
},
|
|
{
|
|
"id": "D09",
|
|
"category": "audit_regression",
|
|
"question": "Akú prácu mal Ján Holp v roku 1999?",
|
|
"expected_status": "no_answer",
|
|
"evidence": [],
|
|
"response_requirements": [
|
|
"No evidence for the requested person/year; do not substitute 2019,2020,2021."
|
|
]
|
|
},
|
|
{
|
|
"id": "D10",
|
|
"category": "audit_regression",
|
|
"question": "Aké spoločné kľúčové slová majú Ján Pták a Ján Malinovský?",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2022/jan_ptak/README.md",
|
|
"sha256": "b0ab69aff5be35a11344ae03c8fb532382657f77965a88d24ad83b251173c1ad",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",
|
|
"start": 6,
|
|
"end": 6,
|
|
"quote": " tag: [rag,nlp]",
|
|
"requirements": [
|
|
"Common tags NLP and RAG."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"nlp"
|
|
],
|
|
[
|
|
"rag"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"path": "pages/students/2022/jan_malinovsky/README.md",
|
|
"sha256": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"start": 6,
|
|
"end": 6,
|
|
"quote": " tag: [rag,nlp]",
|
|
"requirements": [
|
|
"Common tags NLP and RAG."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"nlp"
|
|
],
|
|
[
|
|
"rag"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": []
|
|
},
|
|
{
|
|
"id": "D11",
|
|
"category": "audit_regression",
|
|
"question": "Kto pracoval na niečom podobnom ako Ján Pták?",
|
|
"expected_status": "ok",
|
|
"evidence": [
|
|
{
|
|
"path": "pages/students/2022/jan_ptak/README.md",
|
|
"sha256": "b0ab69aff5be35a11344ae03c8fb532382657f77965a88d24ad83b251173c1ad",
|
|
"url": "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",
|
|
"start": 45,
|
|
"end": 45,
|
|
"quote": "- Oboznámte sa s tým čo je znalostný graf. Napíšte to vlastnými slovami a citujte články.",
|
|
"requirements": [
|
|
"Anchor is a planned graph task, not proof of completed graph implementation."
|
|
],
|
|
"retrieval_terms": [
|
|
[
|
|
"znalostný graf"
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"response_requirements": [
|
|
"Return at least one actually sourced peer on RAG or knowledge graphs, with evidence for both people. Do not call this proof of collaboration or equal completion."
|
|
]
|
|
},
|
|
{
|
|
"id": "D12",
|
|
"category": "audit_regression",
|
|
"question": "Ktoré práce aktuálne vedie Daniel Hladek?",
|
|
"expected_status": "no_answer",
|
|
"evidence": [],
|
|
"response_requirements": [
|
|
"Explicitly say current active supervision is not established. document_author is not supervisor; do not infer from historical records."
|
|
]
|
|
}
|
|
]
|
|
}
|