diff --git a/README.md b/README.md index 17baaf7..63a63c7 100644 --- a/README.md +++ b/README.md @@ -1377,7 +1377,72 @@ regresiu vrátane lokálneho Neo4j. Prepínač `--git-network` navyše iba načíta hlavičku nastavenej wiki vetvy cez existujúce SSH pripojenie. Počas pytestu sú externé spojenia zablokované a kľúč generujúceho modelu odstránený. HTTP transport testy povoľujú iba svoj dočasný loopback port. -Výsledky a log ostávajú lokálne v `data/`. +Výsledky ostávajú lokálne v `data/reports/verification/` a pytest logy +v `data/logs/verification/`. Každé spustenie má vlastný časový identifikátor; +existujúci report sa neprepisuje. Skript počká najviac 90 sekúnd na +pripravenosť API po štarte Dockera (`--wait-ready-seconds`). + +### Reporty, logy a uchovávanie + +Prevádzkové indexy `documents.json`, `chunks.json`, `index_version.json`, +SQLite databázy a zámky ostávajú priamo v `data/`. Reporty sa zapisujú +do `data/reports//`, logy do `data/logs/`. Výstupy inštalovateľných +nástrojov zostávajú v `data/tool_packages/`. + +Päť najnovších dokončených kontrol projektu zostáva v aktívnom priečinku; +staršie kontroly aj ich logy sa presunú do `archive/verification/`. +Neúspešná dokončená kontrola sa tiež uchová. Rozpracované reporty sa +automaticky nearchivujú. Staršie pôvodné výstupy sú v `archive/legacy/`. +Súbory sa nemažú a ich obsah sa nemení; `data/reports/migrations/` +obsahuje záznam pôvodnej a novej cesty spolu s SHA-256. Replay klienti +vedia cez tento záznam načítať aj presunutý vstup. + +```bash +.venv/bin/python scripts/manage_reports.py # náhľad presunov +.venv/bin/python scripts/manage_reports.py --apply # presun, bez mazania +``` + +Formálne evaluačné výsledky v `tools/*/evaluation/results/`, baseline, +GraphRAG verzie a podklady pre diplomovku sa týmto uchovávaním nemenia. +RAG merania, audity a modelové odpovede v `reports/` sa automaticky +nemažú ani nerotujú; uchovávajú sa kvôli reprodukcii. Celé `data/` zostáva +ignorované Gitom. Parametre `--output` kontrolných skriptov sú voliteľné; +staré ploché cesty `data/nazov.json` sa presmerujú do príslušnej oblasti. + +### Širšie overenie kontrolovaného RAG + +`tools/rag/evaluation/json_files/verified_rag_regression.json` obsahuje +vopred vybraných 80 známych otázok z deviatich kategórií, vrátane 10 +neodpovedateľných otázok a 52 otázok s typovanými očakávaniami. +Je to regresná vzorka; jej úspešnosť nie je nový nezávislý odhad presnosti. +Očakávania overuje oddelený oracle priamo z pôvodných Markdownov. +Zdrojové cesty, odtlačky a pôvod akademického roku sú uložené pri otázkach. +Ľudská anotácia sa tým nenahrádza. Dve otázky na Dária Lindvaia používajú +rok z kategórie `dp2021`; kontrolovaná cesta taký rok nepovažuje za +explicitný dôkaz roku práce. Táto neistota je uvedená v datasete. + +```bash +# Len lokálne vyhľadávanie a kontrola podkladov, bez generujúceho modelu: +.venv/bin/python -m tools.answer_evaluation.evaluate_verified_rag --mode retrieve + +# Úmyselné chybné odpovede na uložených skutočných kontextoch, bez LLM: +.venv/bin/python -m tools.answer_evaluation.check_verified_rag_guards --input + +# Skutočné generovanie cez /answer: vyžaduje nakonfigurovaný model a povolené +# odoslanie otázok aj wiki podkladov tomuto poskytovateľovi: +.venv/bin/python -m tools.answer_evaluation.evaluate_verified_rag --mode deliver + +# Opätovné vyhodnotenie uložených odpovedí bez nového generovania: +.venv/bin/python -m tools.answer_evaluation.evaluate_verified_rag --mode replay --input +``` + +Runner posiela na backend iba otázku a parametre; očakávané odpovede model +nedostane. Vyhodnocuje finálny stav, prijatie odpovede, citácie, úplné +väzby osoba–práca–typ–rok a pokrytie očakávaných údajov vo výňatkoch. +Nesprávne „nenájdené“, blokované odpovede a technické zlyhania sú vedené +samostatne a zostávajú v menovateli. Doslovná citácia sama osebe nedokazuje +relevantnosť. Automatická kontrola pokrytia známych údajov nenahrádza +ľudské posúdenie významu všeobecných odpovedí. Kontrola syntaxe najdôležitejších upravovaných modulov: diff --git a/app/routes.py b/app/routes.py index a99be3b..2ad9df6 100644 --- a/app/routes.py +++ b/app/routes.py @@ -184,7 +184,8 @@ class AnswerRequest(StrictRequestModel): @router.post('/answer', dependencies=[Depends(require_search_api_key)], include_in_schema=False) def verified_answer(request: AnswerRequest) -> dict: try: - result = deliver_answer(request.query, engine=request.engine, limit=request.limit) + result = deliver_answer(request.query, engine=request.engine, limit=request.limit, + include_diagnostics=request.include_diagnostics) except Exception: logger.exception('Príprava overenej odpovede zlyhala.') return {'status': 'error', 'reason': 'answer_unavailable', 'accepted_for_delivery': False, @@ -196,6 +197,7 @@ def verified_answer(request: AnswerRequest) -> dict: 'claims': [], 'source_urls': []} if not request.include_diagnostics: result.pop('attempts', None) + result.pop('retrieval_context', None) return result diff --git a/scripts/manage_reports.py b/scripts/manage_reports.py new file mode 100644 index 0000000..d091349 --- /dev/null +++ b/scripts/manage_reports.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Organize local reports and logs. Dry run by default; never delete files.""" +import argparse +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from tools.runtime.report_files import migration_plan, retention_plan, apply_moves + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--apply', action='store_true') + parser.add_argument('--keep-verifications', type=int, default=5) + args = parser.parse_args() + moves = migration_plan() + retention_plan(keep=args.keep_verifications) + if args.apply and moves: + record = apply_moves(moves) + print(json.dumps({'moved': len(moves), 'deleted': 0, 'journal': str(record)})) + else: + print(json.dumps({'dry_run': True, 'moves': moves, 'deleted': 0}, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_project.py b/scripts/verify_project.py index 0e47e6e..cd2105e 100644 --- a/scripts/verify_project.py +++ b/scripts/verify_project.py @@ -103,13 +103,17 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--docker-context', default='default') parser.add_argument('--git-network', action='store_true', help='Read-only probe of wiki origin over its configured SSH connection') - parser.add_argument('--output', type=Path, default=ROOT / 'data/project_verification.json') + parser.add_argument('--wait-ready-seconds', type=int, default=90, + help='Bounded wait for API/model warm-up after Docker startup') + parser.add_argument('--output', type=Path) parser.add_argument('--test-worker', action='store_true', help=argparse.SUPPRESS) args = parser.parse_args() os.chdir(ROOT) sys.path.insert(0, str(ROOT)) if args.test_worker: return local_test_worker() + from tools.runtime.report_files import report_output, report_log, retention_plan, apply_moves + args.output = report_output(args.output, 'verification') report = {'checks': {}, 'external_llm_calls': False} args.output.parent.mkdir(parents=True, exist_ok=True) @@ -142,9 +146,17 @@ def main() -> int: check('compose_configuration', lambda: command(*compose, 'config', '--quiet')) def health(): - with urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=15) as response: - value = json.load(response) - assert value['ready'] and all(tool['ready'] for tool in value['tools'].values()) + deadline = time.monotonic() + max(0, min(args.wait_ready_seconds, 300)) + while True: + try: + with urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3) as response: + value = json.load(response) + assert value['ready'] and all(tool['ready'] for tool in value['tools'].values()) + break + except (OSError, AssertionError): + if time.monotonic() >= deadline: + raise + time.sleep(1) assert value['tools']['rag']['index_version'] == value['tools']['graphrag']['index_version'] return {'ready': True, 'index_version': value['index_version'], 'tools': {name: item['ready'] for name, item in value['tools'].items()}} @@ -208,7 +220,7 @@ def main() -> int: check('openwebui_tool_packages', tool_packages) def tests(): - log = args.output.with_suffix('.pytest.log') + log = report_log(args.output) environment = {**os.environ} environment.pop('OPENWEBUI_API_KEY', None) with log.open('w') as output: @@ -223,6 +235,9 @@ def main() -> int: check('full_local_regression', tests) report['passed'] = all(item['passed'] for item in report['checks'].values()) args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + moves = retention_plan() + if moves: + apply_moves(moves) print('Report: ' + str(args.output)) return 0 if report['passed'] else 1 diff --git a/test/test_report_files.py b/test/test_report_files.py new file mode 100644 index 0000000..4f297ad --- /dev/null +++ b/test/test_report_files.py @@ -0,0 +1,75 @@ +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from tools.runtime.report_files import (apply_moves, migration_plan, report_log, + report_output, resolve_report_input, retention_plan) + + +def test_migration_preserves_operational_data_and_report_bytes(tmp_path): + for name in ['documents.json', 'chunks.json', 'index_version.json', '.reindex.lock', 'zp_index.sqlite']: + (tmp_path/name).write_text('operational') + (tmp_path/'old.json').write_bytes(b'{"old":true}\n') + (tmp_path/'old.pytest.log').write_text('log') + plan = migration_plan(tmp_path) + assert {Path(m['source']).name for m in plan} == {'old.json', 'old.pytest.log'} + journal = apply_moves(plan, data_dir=tmp_path) + assert len(json.loads(journal.read_text())['moves']) == 2 + for move in plan: + assert hashlib.sha256(Path(move['destination']).read_bytes()).hexdigest() == move['sha256'] + assert resolve_report_input(Path(move['source']), data_dir=tmp_path) == Path(move['destination']) + assert (tmp_path/'documents.json').read_text() == 'operational' + assert not migration_plan(tmp_path) + + +def test_no_overwrites_and_changed_plan_is_rejected_before_any_move(tmp_path): + for name in ['a.json','b.json']: + (tmp_path/name).write_text('{}') + plan = migration_plan(tmp_path) + (tmp_path/'b.json').write_text('{"changed":1}') + with pytest.raises(RuntimeError): + apply_moves(plan, data_dir=tmp_path) + assert (tmp_path/'a.json').exists() + output = report_output(None, 'verification', data_dir=tmp_path) + output.write_text('{}') + with pytest.raises(FileExistsError): + report_output(output, 'verification', data_dir=tmp_path) + for name in ['documents.json', 'chunks.json', 'index_version.json', 'zp_index.sqlite']: + with pytest.raises(ValueError): + report_output(tmp_path/name, 'verification', data_dir=tmp_path) + + +def test_flat_legacy_output_is_redirected_and_unique_default_has_separate_log(tmp_path): + output = report_output(tmp_path/'old.json', 'verification', data_dir=tmp_path) + assert output == tmp_path/'reports/verification/old.json' + assert report_log(output, data_dir=tmp_path) == tmp_path/'logs/verification/old.pytest.log' + assert report_output(None, 'verification', data_dir=tmp_path) != report_output(None, 'verification', data_dir=tmp_path) + assert report_log(tmp_path/'custom/result.json', data_dir=tmp_path).parent == tmp_path/'logs/custom' + + +def test_retention_archives_only_completed_verifications_and_follows_move_chain(tmp_path): + outputs = [] + for i in range(8): + output = report_output(tmp_path/f'run{i}.json', 'verification', data_dir=tmp_path) + output.write_text(json.dumps({'passed': bool(i%2)})) + os.utime(output, (100+i,100+i)) + report_log(output, data_dir=tmp_path).write_text('log') + outputs.append(output) + incomplete = tmp_path/'reports/verification/incomplete.json' + incomplete.write_text('{}') + experiment = report_output(None, 'rag_verified', data_dir=tmp_path) + experiment.write_text('{"passed":true}') + plan = retention_plan(tmp_path, keep=5) + assert len(plan) == 6 + apply_moves(plan, data_dir=tmp_path) + assert incomplete.exists() and experiment.exists() + assert len(list((tmp_path/'reports/verification').glob('*.json'))) == 6 + archived = resolve_report_input(outputs[0], data_dir=tmp_path) + assert archived.exists() and archived.parent == tmp_path/'reports/archive/verification' + next_target = tmp_path/'reports/archive/older'/archived.name + apply_moves([{'source':str(archived),'destination':str(next_target), + 'sha256':hashlib.sha256(archived.read_bytes()).hexdigest()}], data_dir=tmp_path) + assert resolve_report_input(outputs[0], data_dir=tmp_path) == next_target diff --git a/test/test_verified_rag_extended.py b/test/test_verified_rag_extended.py new file mode 100644 index 0000000..99e6aa0 --- /dev/null +++ b/test/test_verified_rag_extended.py @@ -0,0 +1,132 @@ +import json + +import pytest + +from tools.answer_evaluation.evaluate_verified_rag import binding_matches, evaluate_case +from tools.answer_evaluation.rag_evidence import rag_evidence_context +from tools.answer_evaluation.verified_rag import rag_answer_issues, generate_verified_rag +from tools.rag.rag_utils import build_context_text + + +def payload(text, *, categories=None): + sources = [{'source_id':'S1', 'title':'Alena Nová', + 'document_path':'pages/students/2020/alena_nova/README.md', + 'source_url':'https://example.test/alena', 'categories':categories or [], + 'text':text}] + return {'status':'ok', 'complete':None, 'sources':sources, 'context':build_context_text(sources)} + + +def refusal(): + return {'status':'no_answer','claims':[],'evidence_quotes':[],'source_urls':[]} + + +@pytest.mark.parametrize('question', ['V ktorom roku začal študovať Alena Nová?', + 'Aký je rok začiatku štúdia osoby Alena Nová?']) +def test_explicit_study_start_cannot_be_falsely_refused(question): + p = payload('Dokument: Alena Nová\n\n*Rok začiatku štúdia:* 2020') + context = rag_evidence_context(question, p) + assert context['requirements']['start_year'] + assert 'rejected_supported_academic_answer' in rag_answer_issues(refusal(), context) + result = generate_verified_rag(question, p, lambda messages: {'choices':[{'message':{'content':json.dumps(refusal())}}]}) + assert not result['accepted_for_delivery'] + + +def test_study_start_never_comes_from_cohort_or_academic_year(): + c = rag_evidence_context('Kedy začal študovať Alena Nová?', + payload('Sekcia: Bakalárska práca 2023\nNázov: Overené modelovanie textu', categories=['bp2023'])) + assert not any('start_year' in row for row in c['facts']) + assert rag_answer_issues(refusal(), c) == [] + + +@pytest.mark.parametrize('year,unsupported', [('3000',False),('12345',True)]) +def test_explicit_year_is_honored_or_rejected_even_in_pre_rok_form(year,unsupported): + c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová pre rok '+year+'?', + payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu')) + assert c['requirements']['unsupported_year'] == unsupported + answer = {'status':'ok','claims':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025, + 'title':'Overené modelovanie textu'}], 'source_urls':['https://example.test/alena']} + assert ('unsupported_requested_year' if unsupported else 'wrong_requested_academic_year') in rag_answer_issues(answer,c) + + +def test_multiline_title_and_unlabelled_title_preceding_assignment(): + p = payload('Sekcia: Diplomová práca 2025\nNázov práce:\n\nOverené modelovanie textu\n\nZadanie:\n1. Skúmajte dáta.') + c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová?', p) + assert any(row.get('title')=='Overené modelovanie textu' and row['year']==2025 for row in c['facts']) + p = payload('Sekcia: Diplomová práca 2025\nOverené modelovanie textu\nCieľ:\nZlepšiť model.') + assert any(row.get('title')=='Overené modelovanie textu' for row in rag_evidence_context('Práca?',p)['facts']) + + +@pytest.mark.parametrize('text', [ + 'Sekcia: Diplomová práca 2025\nStretnutie po úspešných experimentoch\nStav:\nHotovo.', + 'Sekcia: Diplomová práca 2025\nTéma:\nOverené modelovanie tex\n[OBSAH ZDROJA OREZANÝ]', + 'Sekcia: Diplomová práca 2025\nNázov:\n\nĎALŠÍ DÔKAZ Z ROVNAKEJ ŠTUDENTSKEJ VETVY 2\nSekcia: Diplomová práca 2025\nStretnutie 12.3.' +]) +def test_notes_clipped_titles_and_frame_labels_never_become_titles(text): + assert not any(row.get('title') for row in rag_evidence_context('Práca?',payload(text))['facts']) + + +def test_sibling_scope_and_preparation_cannot_inherit_previous_work(): + p = payload('# Diplomová práca 2025\nNázov: Overené modelovanie textu\n' + '# Poznámky\nNázov: Úplne iný dokument\n' + '# Príprava na diplomový projekt 2026\nNázov: Ďalší prípravný dokument') + c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová?',p) + titles = {row.get('title') for row in c['facts'] if row.get('work_type')} + assert titles == {'Overené modelovanie textu'} + + +def test_typed_root_proposal_needs_one_matching_academic_scope(): + p = payload('Dokument: Alena Nová\nNávrh na názov DP:\nOverené modelovanie textu\n' + 'Sekcia: Diplomová práca 2025\nStretnutie 12.3.') + c = rag_evidence_context('Práca?',p) + assert any(r.get('title')=='Overené modelovanie textu' and r['year']==2025 for r in c['facts']) + p['sources'][0]['text'] += '\nSekcia: Diplomová práca 2024\nStretnutie 5.3.' + p['context'] = build_context_text(p['sources']) + assert not any(r.get('title') for r in rag_evidence_context('Práca?',p)['facts']) + + +def test_quote_from_another_student_or_only_metadata_is_not_an_answer(): + p = payload('Rok začiatku štúdia: 2020\nPoužíva Python.') + other = {**p['sources'][0], 'source_id':'S2','title':'Boris Nový', + 'document_path':'pages/students/2020/boris_novy/README.md', + 'source_url':'https://example.test/boris', 'text':'Používa Rust.'} + p['sources'].append(other); p['context']=build_context_text(p['sources']) + c = rag_evidence_context('Aký jazyk používa Alena Nová?',p) + a = {'status':'ok','claims':[],'source_urls':['https://example.test/boris'], + 'evidence_quotes':[{'source_id':'S2','quote':'Používa Rust.'}]} + assert 'unrequested_quote_person' in rag_answer_issues(a,c) + a.update(source_urls=['https://example.test/alena'], evidence_quotes=[{'source_id':'S1','quote':'Názov dokumentu: Alena Nová'}]) + assert 'metadata_quote_does_not_answer_fact' in rag_answer_issues(a,c) + + +def test_evaluator_rejects_cross_person_join_wrong_tuple_and_false_refusal(): + expected = {'person':'Alena Nová', 'title':'Overené modelovanie textu','year':2025,'work_type':'diploma_thesis'} + assert not binding_matches(expected,[{'person':'Alena Nová','title':expected['title']}, + {'person':'Boris Nový','year':2025,'work_type':'diploma_thesis'}]) + p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu') + case = {'id':'fixture','category':'fact_title','question':'Aký je názov diplomovej práce osoby Alena Nová?', + 'should_answer':True,'expected_bindings':[expected], 'expected_source_urls':['https://example.test/alena'], + 'expected_answer_contains':['Overené modelovanie textu']} + refused = {'status':'no_answer','accepted_for_delivery':True,'answer':'Nenájdené.','claims':[],'source_urls':[],'attempts':[]} + assert 'false_no_answer' in evaluate_case(case,p,refused)['issues'] + bad = {**refused,'status':'ok','answer':'Overené modelovanie textu','source_urls':case['expected_source_urls'], + 'claims':[{**expected,'year':2024}]} + assert 'unsupported_person_work_type_year_binding' in evaluate_case(case,p,bad)['issues'] + + +def test_evaluator_accepts_extra_supported_project_but_rejects_wrong_rendered_year(): + from tools.answer_evaluation.verified_rag import render_rag_answer + p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu\n' + 'Sekcia: Diplomový projekt 2024\nStretnutie 12.3.') + question = 'Aký je rok práce osoby Alena Nová?' + context = rag_evidence_context(question,p) + claims = [{'person':'Alena Nová','work_type':'diploma_thesis','year':2025}, + {'person':'Alena Nová','work_type':'diploma_project','year':2024}] + parsed = {'status':'ok','claims':claims,'source_urls':['https://example.test/alena'],'evidence_quotes':[]} + delivered = {**parsed,'accepted_for_delivery':True,'answer':render_rag_answer(parsed,context), + 'attempts':[{'parsed_answer':parsed,'issues':[]}]} + case = {'id':'fixture','category':'fact_work_year','question':question,'should_answer':True, + 'expected_source_urls':parsed['source_urls'], 'expected_answer_contains':['2025'], + 'expected_bindings':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025}]} + assert evaluate_case(case,p,delivered)['passed'] + delivered['answer'] += '\nPráca bola ukončená v roku 2099.' + assert 'delivered_text_differs_from_verified_renderer' in evaluate_case(case,p,delivered)['issues'] diff --git a/tools/answer_evaluation/check_context_scenarios.py b/tools/answer_evaluation/check_context_scenarios.py index 95c5e03..e7dd2c4 100644 --- a/tools/answer_evaluation/check_context_scenarios.py +++ b/tools/answer_evaluation/check_context_scenarios.py @@ -63,10 +63,10 @@ def scenarios(): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--output', type=Path) args = parser.parse_args() - if args.output.exists(): - parser.error('Refusing to overwrite results.') + from tools.runtime.report_files import report_output + args.output = report_output(args.output, 'context_scenarios') files = [Path(__file__), HERE/'graph_answer_prompt.txt', HERE/'context_policy.py', HERE/'checked_generation.py', HERE/'answer_format.py', HERE/'render_answer.py'] hashes = {str(p): sha256_file(p) for p in files} declared = scenarios() diff --git a/tools/answer_evaluation/check_deployment.py b/tools/answer_evaluation/check_deployment.py index 31ea0c4..b56075b 100644 --- a/tools/answer_evaluation/check_deployment.py +++ b/tools/answer_evaluation/check_deployment.py @@ -18,13 +18,14 @@ from tools.rag.common import DB_FILE def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--output', type=Path) parser.add_argument('--questions', type=Path, required=True) parser.add_argument('--audit', type=Path, required=True) parser.add_argument('--freeze-manifest', type=Path, required=True) parser.add_argument('--release-manifest', type=Path) args = parser.parse_args() - if args.output.exists(): parser.error('Refusing to overwrite deployment verification.') + from tools.runtime.report_files import report_output + args.output = report_output(args.output, 'deployment') base = Path('tools/graphrag/evaluation') release_path = args.release_manifest release_match = None diff --git a/tools/answer_evaluation/check_rag_runtime.py b/tools/answer_evaluation/check_rag_runtime.py index 4466a98..786d563 100644 --- a/tools/answer_evaluation/check_rag_runtime.py +++ b/tools/answer_evaluation/check_rag_runtime.py @@ -13,9 +13,10 @@ from .rag_delivery import finalize_rag_answer def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--output', type=Path) args = parser.parse_args() - if args.output.exists(): parser.error('Refusing to overwrite smoke results.') + from tools.runtime.report_files import report_output + args.output = report_output(args.output, 'rag_runtime') source = Path('tools/rag/evaluation/results/test_classic_rag_frozen/rag_answers_test.json') historic = json.loads(source.read_text())['results'] selected = ['q0053', 'q0060', 'q0608', 'q1147', 'q0888'] diff --git a/tools/answer_evaluation/check_verified_rag_guards.py b/tools/answer_evaluation/check_verified_rag_guards.py new file mode 100644 index 0000000..7ff926c --- /dev/null +++ b/tools/answer_evaluation/check_verified_rag_guards.py @@ -0,0 +1,96 @@ +"""Adversarial guard replay on real RAG contexts; no generative model calls. + +These are acceptance/rejection checks with deliberately constructed answers, +not measured LLM accuracy. Known benchmark gold never enters production code. +""" +import argparse +import copy +import json +from pathlib import Path + +from .evaluate_verified_rag import DATASET, FIELDS, binding_matches, digest +from .rag_evidence import rag_evidence_context +from .verified_rag import generate_verified_rag, rag_answer_issues +from tools.runtime.report_files import report_output, resolve_report_input + + +def check_cases(dataset: dict, saved: dict) -> list[dict]: + results = [] + for case in dataset['cases']: + payload = saved[case['id']]['retrieval_context'] + context = rag_evidence_context(case['question'], payload) + negative = {'status':'no_answer', 'answer':'', 'claims':[], 'evidence_quotes':[], 'source_urls':[]} + + def check(name, answer, should_accept): + issues = rag_answer_issues(answer, context) + # Exercise the production generator's two-attempt rejection and + # deterministic renderer, with a controlled local completion stub. + calls = [] + def complete(messages): + calls.append(messages) + return {'choices':[{'message':{'content':json.dumps(answer, ensure_ascii=False)}}]} + delivered = generate_verified_rag(case['question'], payload, complete) + actual = delivered['accepted_for_delivery'] + passed = actual == should_accept and bool(not issues) == should_accept + if not should_accept: + passed = passed and len(calls) == 2 and delivered['status'] == 'error' + results.append({'id':case['id'], 'check':name, 'passed':passed, + 'expected_acceptance':should_accept, 'issues':issues}) + + if not case['should_answer']: + check('accept_supported_missing_fact_refusal', negative, True) + continue + bindings = case['expected_bindings'] + if not bindings or not all(binding_matches(b, context['facts']) for b in bindings): + continue + claims = [{k:b[k] for k in FIELDS if k in b} for b in bindings] + for claim in claims: + matching = [row for row in context['facts'] if binding_matches(claim, [row])] + for field in ['title', 'year']: + values = {row[field] for row in matching if field in row} + if context['requirements'][field] and field not in claim and len(values) == 1: + claim[field] = next(iter(values)) + urls = sorted({row['source_url'] for row in context['facts'] if any( + row['person'] == b['person'] for b in bindings)}) + answer = {'status':'ok', 'answer':'', 'claims':claims, 'evidence_quotes':[], 'source_urls':urls} + check('accept_supported_tuple', answer, True) + check('reject_false_refusal', negative, False) + for field, value in [('year',9876), ('title','Nepodložený názov z cudzieho dokumentu'), + ('person','Neexistujúca Osoba'), ('work_type','doctoral_thesis')]: + bad = copy.deepcopy(answer) + if field == 'work_type' and bad['claims'][0].get(field) == value: + value = 'bachelor_thesis' + bad['claims'][0][field] = value + check('reject_wrong_'+field, bad, False) + bad = copy.deepcopy(answer) + bad['source_urls'] = ['https://example.invalid/not-provided'] + check('reject_wrong_citation', bad, False) + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--input', required=True, type=Path) + parser.add_argument('--dataset', type=Path, default=DATASET) + parser.add_argument('--output', type=Path) + args = parser.parse_args() + source = resolve_report_input(args.input) + saved_report = json.loads(source.read_text()) + dataset = json.loads(args.dataset.read_text()) + assert saved_report['configuration']['dataset_sha256'] == digest(args.dataset) + saved = {r['id']:r for r in saved_report['results']} + results = check_cases(dataset, saved) + output = report_output(args.output, 'rag_guards') + summary = {'checks':len(results), 'passed':sum(r['passed'] for r in results), + 'contexts':len(saved), 'llm_calls':0, + 'note':'Controlled fault injection, not an LLM accuracy measurement.'} + output.write_text(json.dumps({'method':__doc__, 'input':str(source), 'input_sha256':digest(source), + 'summary':summary, 'results':results}, ensure_ascii=False, indent=2)+'\n') + print(json.dumps(summary)) + print('Report: '+str(output)) + if any(not r['passed'] for r in results): + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/answer_evaluation/evaluate_verified_rag.py b/tools/answer_evaluation/evaluate_verified_rag.py new file mode 100644 index 0000000..e8b8563 --- /dev/null +++ b/tools/answer_evaluation/evaluate_verified_rag.py @@ -0,0 +1,218 @@ +"""Audit the real checked RAG path on a predeclared, known regression sample. + +Default mode only retrieves locally. `--mode deliver` explicitly invokes the +configured backend model. Expected answers are used here, never sent to /answer. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor, as_completed +import hashlib +import json +import os +from pathlib import Path +import time +import urllib.request + +from dotenv import load_dotenv + +from tools.answer_evaluation.context_policy import normalized, citation_check, allowed_urls +from tools.answer_evaluation.rag_evidence import rag_evidence_context +from tools.answer_evaluation.verified_rag import rag_answer_issues, render_rag_answer +from tools.graphrag.evaluation.source_oracle import DEFAULT_WIKI, load_source_oracle +from tools.rag.evaluation.rag_metrics import expected_phrase_matches +from tools.runtime.report_files import report_output, resolve_report_input + +ROOT = Path(__file__).resolve().parents[2] +DATASET = ROOT / 'tools/rag/evaluation/json_files/verified_rag_regression.json' +FIELDS = ['person', 'work_type', 'title', 'year', 'start_year'] + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def binding_matches(expected: dict, rows: list[dict]) -> bool: + # Study start can be a separate fact about the same person; the academic + # fields must all occur together in one actual academic row. + academic = {k: expected[k] for k in FIELDS if k in expected and k != 'start_year'} + academic_ok = any(all(normalized(row.get(k)) == normalized(v) for k, v in academic.items()) for row in rows) + start_ok = 'start_year' not in expected or any( + normalized(row.get('person')) == normalized(expected['person']) and + row.get('start_year') == expected['start_year'] for row in rows) + return academic_ok and start_ok + + +def source_audit(case: dict, oracle: dict) -> list[str]: + issues = [] + for item in case['source_checks']: + if digest(DEFAULT_WIKI / item['document_path']) != item['sha256']: + issues.append('changed_source_document') + for binding in case['expected_bindings']: + person = oracle['people'].get(binding['person'], {}) + rows = [{'person': binding['person'], 'work_type': a['academic_type'], **a} + for a in person.get('academics', []) if a['document_path'] == binding['document_path']] + rows.append({'person': binding['person'], 'start_year': person.get('start_year')}) + if not binding_matches(binding, rows): + issues.append('gold_binding_not_in_original_markdown') + if 'start_year' in binding and person.get('start_year_basis') != 'explicit_text': + issues.append('study_start_not_explicit') + return issues + + +def evaluate_case(case: dict, payload: dict, delivered: dict | None = None) -> dict: + context = rag_evidence_context(case['question'], payload) + source_urls = allowed_urls(payload) + missing = [b for b in case['expected_bindings'] if not binding_matches(b, context['facts'])] + retrieval = {'expected_sources_found': bool(source_urls & set(case['expected_source_urls'])), + 'missing_bindings': missing, + 'expected_values_in_context': [v for v in case['expected_answer_contains'] + if expected_phrase_matches(v, payload.get('context', ''))]} + row = {'id': case['id'], 'category': case['category'], 'question': case['question'], + 'should_answer': case['should_answer'], 'retrieval': retrieval, + 'typed_expected': bool(case['expected_bindings']), + 'index_version': payload.get('index_version')} + if delivered is None: + return row + issues = [] + accepted = delivered.get('accepted_for_delivery') is True + status = delivered.get('status') + attempts = delivered.get('attempts', []) + parsed = attempts[-1].get('parsed_answer') if attempts else None + if accepted and parsed: + issues.extend(rag_answer_issues(parsed, context)) + if delivered.get('answer') != render_rag_answer(parsed, context): + issues.append('delivered_text_differs_from_verified_renderer') + if delivered.get('claims', []) != parsed.get('claims', []): + issues.append('delivered_claims_differ_from_verified_selection') + if accepted and not citation_check(delivered, source_urls)['citations_ok']: + issues.append('delivered_unsupported_citation') + claims = delivered.get('claims', []) + if accepted: + # Validate every delivered claim against the actual retrieved tuple. + # Additional supported projects of the same person are not a wrong + # thesis year. Required benchmark tuples are checked separately below. + issues.extend(rag_answer_issues({'status':status, 'claims':claims, + 'source_urls':delivered.get('source_urls', []), + 'evidence_quotes':(parsed or {}).get('evidence_quotes', [])}, context)) + if case['should_answer']: + if not accepted: + issues.append('delivery_blocked') + elif status != 'ok': + issues.append('false_no_answer' if status == 'no_answer' else 'nonpositive_answer') + if not all(binding_matches(b, claims) for b in case['expected_bindings']): + issues.append('missing_expected_binding') + # Exclude sources from factual scoring: a name/year in a URL cannot + # supply the requested fact. Quotes must cover the expected subject. + text = delivered.get('answer', '').split('\nZdroje:', 1)[0] + missing_values = [v for v in case['expected_answer_contains'] if not expected_phrase_matches(v, text)] + if missing_values: + issues.append('missing_expected_answer_value') + quotes = (parsed or {}).get('evidence_quotes', []) + if quotes: + content = '\n'.join(q['quote'] for q in quotes) + names = {s.get('title') for s in context['source_blocks']} + # Names in source headers are attribution, not answers to a factual + # question. Value coverage checks relevance beyond literal copying. + relevant = [v for v in case['expected_answer_contains'] if v not in names] + row['quote_expected_value_coverage'] = {v: expected_phrase_matches(v, content) for v in relevant} + elif not accepted or status != 'no_answer': + issues.append('incorrect_negative_answer') + row.update(passed=not issues, issues=sorted(set(issues)), delivered=delivered, + first_attempt_accepted=bool(attempts and not attempts[0].get('issues'))) + return row + + +def summarize(rows: list[dict]) -> dict: + counted = [r for r in rows if 'passed' in r] + total = len(counted) + passed = sum(r['passed'] for r in counted) + return {'total': len(rows), 'answers_evaluated': total, 'passed': passed, + 'success_percent': round(100 * passed / total, 2) if total else None, + 'false_no_answers': sum('false_no_answer' in r['issues'] for r in counted), + 'blocked_deliveries': sum('delivery_blocked' in r['issues'] for r in counted), + 'first_attempt_accepted': sum(r.get('first_attempt_accepted', False) for r in counted), + 'failure_reasons': dict(Counter(i for r in counted for i in r['issues'])), + 'source_coverage': sum(r['retrieval']['expected_sources_found'] for r in rows), + 'answerable_source_coverage': {'passed': sum(r['retrieval']['expected_sources_found'] for r in rows if r['should_answer']), + 'total': sum(r['should_answer'] for r in rows)}, + 'typed_binding_coverage': {'passed': sum(not r['retrieval']['missing_bindings'] for r in rows if r.get('typed_expected')), + 'total': sum(r.get('typed_expected', False) for r in rows)}, + 'categories': {c: {'total': sum(r['category'] == c for r in rows), + 'passed': sum(r.get('passed', False) for r in counted if r['category'] == c)} for c in sorted({r['category'] for r in rows})}} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--mode', choices=['retrieve', 'deliver', 'replay'], default='retrieve') + parser.add_argument('--input', type=Path, help='Saved report to regrade without model calls') + parser.add_argument('--dataset', type=Path, default=DATASET) + parser.add_argument('--backend', default='http://127.0.0.1:8000') + parser.add_argument('--output', type=Path) + args = parser.parse_args() + load_dotenv(ROOT / '.env', override=False) + dataset = json.loads(args.dataset.read_text()) + assert digest(ROOT / dataset['source_file']) == dataset['source_sha256'], 'Historical baseline changed' + oracle = load_source_oracle() + audit = {c['id']: source_audit(c, oracle) for c in dataset['cases']} + assert not any(audit.values()), 'Expected answer/source audit failed' + output = report_output(args.output, 'rag_verified') + files = [Path(__file__), ROOT/'tools/answers/answer_delivery.py', ROOT/'app/routes.py', + *[(ROOT/'tools/answer_evaluation'/name) for name in ['verified_rag.py','rag_evidence.py','context_policy.py']], + ROOT/'tools/rag/rag_utils.py', ROOT/'tools/rag/llm_context.py'] + metadata = {'method': dataset['method'], 'mode': args.mode, 'dataset_sha256': digest(args.dataset), + 'limitations': dataset.get('limitations', []), + 'source_audit': audit, 'source_corpus': oracle['source_corpus'], + 'code_hashes': {str(p.relative_to(ROOT)): digest(p) for p in files}, + 'model_calls': args.mode == 'deliver', 'backend': args.backend, + 'model': os.getenv('ANSWER_MODEL', 'model120-fast') if args.mode == 'deliver' else None} + output.with_suffix('.declaration.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2)+'\n') + key = os.getenv('SEARCH_API_KEY', '') + saved = None + if args.mode == 'replay': + if not args.input: + parser.error('--input is required for replay') + saved = {r['id']: r for r in json.loads(resolve_report_input(args.input).read_text())['results']} + + def run(case): + started = time.monotonic() + if saved is not None: + old = saved[case['id']] + return evaluate_case(case, old['retrieval_context'], old.get('delivered')) | {'retrieval_context': old['retrieval_context']} + endpoint = '/answer' if args.mode == 'deliver' else '/rag' + body = {'query': case['question'], 'limit': 5} + if args.mode == 'deliver': + body.update(engine='rag', include_diagnostics=True) + req = urllib.request.Request(args.backend.rstrip('/')+endpoint, data=json.dumps(body).encode(), + headers={'Content-Type': 'application/json', 'X-API-Key': key}) + try: + with urllib.request.urlopen(req, timeout=300) as response: + data = json.load(response) + payload = data.pop('retrieval_context') if args.mode == 'deliver' else data + result = evaluate_case(case, payload, data if args.mode == 'deliver' else None) + result.update(retrieval_context=payload, seconds=round(time.monotonic()-started, 2)) + return result + except Exception as error: + # Count operational failures in the denominator; never retry only + # selected failed questions or expose raw HTTP/token-bearing errors. + return {'id': case['id'], 'category': case['category'], 'question': case['question'], + 'should_answer': case['should_answer'], 'passed': False, 'issues': ['request_or_evaluation_error'], + 'typed_expected': bool(case['expected_bindings']), + 'error_type': type(error).__name__, 'retrieval': {'expected_sources_found': False, 'missing_bindings': case['expected_bindings']}} + + rows = [] + with ThreadPoolExecutor(max_workers=2) as pool: + futures = {pool.submit(run, case): case for case in dataset['cases']} + for future in as_completed(futures): + rows.append(future.result()) + result = {'configuration': metadata, 'summary': summarize(rows), 'results': sorted(rows, key=lambda r: r['id'])} + output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') + print(f'{len(rows)}/{len(dataset["cases"])} {rows[-1]["id"]}: {rows[-1].get("issues", "retrieved")}', flush=True) + assert metadata['code_hashes'] == {name: digest(ROOT/name) for name in metadata['code_hashes']}, 'Code changed during measurement' + print(json.dumps(result['summary'], ensure_ascii=False)) + print('Report: '+str(output)) + + +if __name__ == '__main__': + main() diff --git a/tools/answer_evaluation/rag_delivery.py b/tools/answer_evaluation/rag_delivery.py index 900e6b4..e3cba47 100644 --- a/tools/answer_evaluation/rag_delivery.py +++ b/tools/answer_evaluation/rag_delivery.py @@ -34,9 +34,11 @@ def main(): parser = argparse.ArgumentParser(description='Replay saved RAG outputs through the delivery layer; no model calls.') parser.add_argument('--input', type=Path, required=True) - parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--output', type=Path) args = parser.parse_args() - if args.output.exists(): parser.error('Refusing to overwrite a replay.') + from tools.runtime.report_files import report_output, resolve_report_input + args.output = report_output(args.output, 'rag_replay') + args.input = resolve_report_input(args.input) original = json.loads(args.input.read_text()) source = Path(original['configuration']['source_file']) assert sha256_file(source) == original['configuration']['source_sha256'] diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index 10618fc..a02c4cc 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -44,7 +44,7 @@ def work_type(text: str) -> str | None: def academic_scope(path: str) -> tuple[str, int | None] | None: for heading in reversed(path.split('>')): if normalized(heading).startswith(('priprava na ', 'priprava k ')): - continue + return None kind = work_type(heading) if kind: years = [int(year) for year in re.findall(r'(? dict: 'document_path': source['document_path']} windows = [] scope = None + heading_stack = [] lines = [] def flush(): if lines: @@ -141,23 +142,42 @@ def rag_evidence_context(question: str, payload: dict) -> dict: lines.clear() for line in source['body'].splitlines(): heading = re.match(r'^(?:Sekcia(?: dôkazu)?:|#{1,6}\s)(.*)', line) - if line.startswith('Dokument:'): - flush(); scope = None + if line.startswith(('RELEVANTNÁ SEKCIA ', 'DÔKAZ Z ', 'ĎALŠÍ DÔKAZ ', + 'ĎALŠÍ RELEVANTNÝ ', 'NAJRELEVANTNEJŠÍ ', 'ZAČIATOK RELEVANTNEJ ')): + flush(); scope = None; heading_stack.clear() + elif line.startswith('Dokument:'): + flush(); scope = None; heading_stack.clear() elif heading: flush() - detected = academic_scope(heading[1]) - if line.startswith('Sekcia') or detected: - scope = detected + if line.startswith('Sekcia'): + scope = academic_scope(heading[1]) + heading_stack = [(0, scope)] + else: + level = len(line) - len(line.lstrip('#')) + while heading_stack and heading_stack[-1][0] >= level: + heading_stack.pop() + detected = academic_scope(heading[1]) + scope = (None if normalized(heading[1]).startswith(('priprava na ', 'priprava k ')) else + detected or (heading_stack[-1][1] if heading_stack else None)) + heading_stack.append((level, scope)) else: lines.append(line) flush() unscoped = [] for section, text in windows: titles = [] - for line in text.splitlines(): + text_lines = text.splitlines() + for index, line in enumerate(text_lines): if ':' in line: label, value = line.split(':', 1) if normalized(label) in TITLE_LABELS: + value_index = index + if not value.strip(): + value_index = next((j for j in range(index+1, len(text_lines)) if text_lines[j].strip()), index) + value = text_lines[value_index] if value_index != index else '' + following = next((v.strip() for v in text_lines[value_index+1:] if v.strip()), '') + if following == '[OBSAH ZDROJA OREZANÝ]' or value.strip() == '[OBSAH ZDROJA OREZANÝ]': + continue title = clean_title(value) if title: titles.append(title) @@ -166,6 +186,21 @@ def rag_evidence_context(question: str, payload: dict) -> dict: facts.append({**base, 'start_year': int(start[1])}) if section: kind, year = section + if not titles: + # Wiki also puts an unlabelled title immediately after its + # academic heading (sometimes preceded by explicit study + # start). Never promote meeting notes, tasks or list items. + lead_lines = [v.strip() for v in text_lines if v.strip() and + not normalized(v).startswith('rok zaciatku studia')] + lead = lead_lines[0] if lead_lines else '' + next_line = lead_lines[1] if len(lead_lines) > 1 else '' + if (lead and re.match(r'^(?:Cie[lľe]+\s*:|Zadanie\s*:|Myšlienky\s*:|1[.)]\s)', next_line, re.I) and + not re.match(r'^(?:[-*+]\s|\d+[.)]\s|#{1,6}\s)', lead) and + not normalized(lead).startswith(('stretnutie','virtualne stretnutie','stav','ulohy', + 'ciel','zadanie','zasobnik','rok ','zaciatok studia','projekt ','praca ','tema na '))): + title = clean_title(lead) + if title and len(normalized(title).split()) >= 3: + titles.append(title) record = {**base, 'work_type': kind} if year is not None: record['year'] = year @@ -180,6 +215,18 @@ def rag_evidence_context(question: str, payload: dict) -> dict: if section and normalized(title) in normalized(text): facts.append({**base, 'title': title, 'work_type': section[0], **({'year': section[1]} if section[1] is not None else {})}) + # A root label explicitly naming BP/DP licenses only that type; + # attach it only if the delivered evidence has exactly one such + # academic scope. Taxonomy categories still cannot supply a year. + for label, kind in [('navrh na nazov dp', 'diploma_thesis'), + ('navrh na nazov bp', 'bachelor_thesis')]: + explicit = any(re.search(r'\b'+re.escape(label)+r'\s+'+re.escape(normalized(title))+r'\b', + normalized(text)) for section, text in windows if section is None) + scopes = {s for s, _ in windows if s and s[0] == kind} + if explicit and len(scopes) == 1: + only = next(iter(scopes)) + facts.append({**base, 'title': title, 'work_type': kind, + **({'year': only[1]} if only[1] is not None else {})}) for field, values in [('keyword', source.get('tags', [])), ('category', source.get('categories', [])), ('document_author', [source['author']] if source.get('author') else [])]: facts.extend({**base, field: value} for value in values) @@ -187,17 +234,19 @@ def rag_evidence_context(question: str, payload: dict) -> dict: requested = [person for person in people if text_contains_value(question, person, person=True)] expected_types = requested_types(question, requested) query = normalized(question) - academic = bool(TYPE_PATTERN.search(query) or re.search(r'\brok\w* prac\w*\b', query)) + academic = bool(TYPE_PATTERN.search(query) or re.search(r'\b(?:rok\w* prac\w*|nazov\w* prac\w*)\b', query)) + start_year = bool(re.search(r'\b(?:zaciat\w* stud\w*|zacal\w* stud\w*)\b', query)) requirements = {'people': requested, 'work_types': expected_types, 'academic': academic, 'title': 'nazov' in query or 'tema' in query, 'year': academic and ('rok' in query or 'porovnaj' in query), - 'start_year': 'zaciat' in query and 'studi' in query} - requirements['years'] = ([int(value) for value in re.findall(r'\b(?:v roku|z roku|roku)\s+([12]\d{3})\b', query)] - if academic and not requirements['start_year'] else []) - requirements['unsupported_year'] = academic and bool(re.search(r'\b(?:v roku|z roku|roku)\s+\d{5,}\b', query)) + 'start_year': start_year} + stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else [] + requirements['years'] = ([int(value) for value in stated_years if len(value) == 4] + if not requirements['start_year'] else []) + requirements['unsupported_year'] = any(len(value) != 4 for value in stated_years) if requested: facts = [row for row in facts if row['person'] in requested] - return {**{key: payload.get(key) for key in ['status', 'reason', 'complete', 'truncated', + return {'question': question, **{key: payload.get(key) for key in ['status', 'reason', 'complete', 'truncated', 'truncation_reasons', 'index_version']}, 'sources': payload.get('sources', []), 'facts': facts, 'source_blocks': blocks, 'requirements': requirements, 'aggregation': None, 'answer_candidates': []} diff --git a/tools/answer_evaluation/regrade_citations.py b/tools/answer_evaluation/regrade_citations.py index 568e0f9..10ba64f 100644 --- a/tools/answer_evaluation/regrade_citations.py +++ b/tools/answer_evaluation/regrade_citations.py @@ -14,9 +14,11 @@ from .context_policy import citation_check, allowed_urls def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--input', type=Path, action='append', required=True) - parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--output', type=Path) args = parser.parse_args() - if args.output.exists(): parser.error('Refusing to overwrite regrading.') + from tools.runtime.report_files import report_output, resolve_report_input + args.output = report_output(args.output, 'citations') + args.input = [resolve_report_input(path) for path in args.input] reports = [] for path in args.input: data = json.loads(path.read_text()); rows = [] diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index d844aea..3164cf7 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -39,6 +39,8 @@ Ak sa otázka pýta na prácu osoby, vyplň claims: person, work_type a požadov Použi presný názov, typ a rok TOHO ISTÉHO záznamu danej osoby. Rok kategórie nie je rok práce. Pri porovnávaní odpovedz za každú požadovanú osobu; rešpektuj requirements.work_types. start_year je len explicitný začiatok štúdia, nikdy rok z adresára. +Pri otázke na začiatok štúdia vyplň claims so start_year; pri rozlíšení rokov +uveď start_year aj year a work_type. Citácia ani nesúvisiaci výňatok údaj nenahrádzajú. Pri všeobecnej otázke môžeš namiesto claims vybrať evidence_quotes: source_id a presný doslovný výňatok, ktorý odpovedá na otázku. Výňatok nesmieš parafrázovať ani meniť čísla. Cituj iba source_urls zo sources, ktoré dokladajú vybrané údaje. Názov profilu nie je názov práce. @@ -92,7 +94,7 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: issues.append('wrong_or_missing_requested_work_type') if requirements['years'] and claim.get('year') is not None and claim['year'] not in requirements['years']: issues.append('wrong_requested_academic_year') - if requirements['academic'] and answer['status'] == 'ok': + if (requirements['academic'] or requirements['start_year']) and answer['status'] == 'ok': if requirements['unsupported_year']: issues.append('unsupported_requested_year') if not requested: @@ -102,7 +104,7 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: for key in ['title', 'year', 'start_year']: if requirements[key] and not any(claim.get(key) is not None for claim in matching): issues.append('missing_requested_' + key) - if answer['status'] == 'no_answer' and requirements['academic'] and requested: + if answer['status'] == 'no_answer' and (requirements['academic'] or requirements['start_year']) and requested: # Reject an unsupported refusal only if every requested academic tuple # is explicitly present. Never invent a value to fill an absent field. supported = all(any(row['person'] == person and @@ -110,7 +112,10 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: (not requirements['years'] or row.get('year') in requirements['years']) and all(not requirements[key] or row.get(key) is not None for key in ['title', 'year']) for row in facts) for person in requested) - if supported and (requirements['title'] or requirements['year']) and not requirements['start_year'] and not requirements['unsupported_year']: + start_supported = all(any(row['person'] == person and row.get('start_year') is not None + for row in facts) for person in requested) + if (supported and (requirements['title'] or requirements['year'] or requirements['start_year']) + and (not requirements['start_year'] or start_supported) and not requirements['unsupported_year']): issues.append('rejected_supported_academic_answer') cited = set(answer.get('source_urls', [])) blocks = {row['source_id']: row for row in context['source_blocks']} @@ -120,6 +125,15 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: issues.append('unsupported_evidence_quote') elif source.get('source_url') not in cited: issues.append('missing_quote_source') + if source: + if (requested and source.get('document_path', '').startswith(('pages/students/', 'pages/interns/')) and + not any(source.get('document_path', '').startswith(row['document_path'].removesuffix('README.md')) + for row in context['source_blocks'] if row.get('title') in requested)): + issues.append('unrequested_quote_person') + identity_question = bool(re.search(r'\b(?:url|odkaz\w*|dokument\w*)\b', normalized(context.get('question', '')))) + if (not identity_question and + ' '.join(quote['quote'].split()) not in ' '.join(source['body'].split())): + issues.append('metadata_quote_does_not_answer_fact') # A quoted embedded URL is not automatically an allowed citation. if not citation_check({'answer': quote['quote'], 'source_urls': []}, allowed_urls(context))['citations_ok']: issues.append('unsupported_quote_citation') @@ -139,7 +153,9 @@ def render_rag_answer(answer: dict, context: dict) -> str: values = [WORK_LABELS[claim['work_type']]] if claim.get('work_type') else [] values += [f'{label}: {claim[key]}' for key, label in labels.items() if claim.get(key) is not None] lines.append(claim['person'] + ' — ' + '; '.join(values) + '.') - lines += ['Zo zdroja: „' + quote['quote'] + '“' for quote in answer.get('evidence_quotes', [])] + sources = {row['source_id']: row for row in context['source_blocks']} + lines += ['Zo zdroja ' + str(sources[quote['source_id']].get('title') or quote['source_id']) + + ': „' + quote['quote'] + '“' for quote in answer.get('evidence_quotes', [])] if answer.get('count') is not None: lines.append(f'Počet: {answer["count"]}.') elif answer['status'] == 'no_answer': diff --git a/tools/answers/answer_delivery.py b/tools/answers/answer_delivery.py index 5df6572..642d334 100644 --- a/tools/answers/answer_delivery.py +++ b/tools/answers/answer_delivery.py @@ -35,7 +35,7 @@ def complete_with_configured_model(messages: list[dict]) -> dict: def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, - complete=None, db_file: Path = DB_FILE) -> dict: + complete=None, db_file: Path = DB_FILE, include_diagnostics: bool = False) -> dict: if not _capacity.acquire(blocking=False): return {'status': 'error', 'reason': 'answer_capacity_exceeded', 'accepted_for_delivery': False, 'answer': 'Agent práve spracúva iné odpovede. Skúste to o chvíľu.'} @@ -44,6 +44,7 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, if engine == 'rag': retrieved = compact_rag_payload(build_rag_context(db_file, question, limit=limit)) result = generate_verified_rag(question, retrieved, model) + context = retrieved elif engine == 'graphrag': context = graph_context(search_graph(question)) prompt = (Path(__file__).resolve().parents[1] / 'answer_evaluation/graph_answer_prompt.txt').read_text() @@ -59,6 +60,7 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, 'index_version': context.get('index_version'), 'attempts': generated['attempts']} else: raise ValueError('Unsupported answer engine') - return {**result, 'engine': engine} + return {**result, 'engine': engine, + **({'retrieval_context': context} if include_diagnostics else {})} finally: _capacity.release() diff --git a/tools/rag/evaluation/json_files/verified_rag_regression.json b/tools/rag/evaluation/json_files/verified_rag_regression.json new file mode 100644 index 0000000..06fd70a --- /dev/null +++ b/tools/rag/evaluation/json_files/verified_rag_regression.json @@ -0,0 +1,2315 @@ +{ + "method": "Predeclared stratified regression sample of known classic RAG questions, not an independent TEST. Gold/source oracle used only by evaluator, never sent to model.", + "seed": 20261027, + "source_file": "tools/rag/evaluation/results/test_classic_rag_frozen/rag_answers_test.json", + "source_sha256": "eba03639f8a3c32ee47a4e294d8592c1071fc6fd43e38967e39131b840fefa7e", + "source_corpus": { + "sha256": "440da12a7a991824e923f70cada7c616c3fb3b0f12767ec7d75039a6a3231b10", + "document_count": 114, + "documents": { + "pages/interns/bogdan_paul_chis/README.md": "3b4b946eb3c10e9f9a4092e6cffa01c5080e7579aded2f6ed3120c41ed62b647", + "pages/interns/cesar_gutierrez/README.md": "b0be60576eed28d4b1920e9acf198684519373dfc13171567247c853b3ed699e", + "pages/interns/oliver_pejic/README.md": "0470f3c8465be8f02204b08e4c75ef47f68c7e4ad1038faa5641cdf1811f33d5", + "pages/interns/sevval_bulburu/README.md": "ff2a9d66253802f1af0fdfd74832e40fae41b6df23c5e61dc266f6cd0cc16c4d", + "pages/interns/yussef_ressaissi/README.md": "f997a65699da0db3222ef7a3c37652d05dfc3419ad4ec74683bafc2c897bb987", + "pages/students/2016/darius_lindvai/README.md": "fcaa47bb9708a6439eec5c4b7f7b4ed6c7359e423e1d940e1aed9f44957e6d62", + "pages/students/2016/darius_lindvai/dp2021/README.md": "be3d0fea2a6d592b225eecabd9b19445c2fce62b8289c07ab6af93835db12970", + "pages/students/2016/darius_lindvai/timovy_projekt/resers/README.md": "352162de1b1e192228fa5d26506c028ac47e567e90df53b1bbaa6ad736ec4766", + "pages/students/2016/darius_lindvai/timovy_projekt/tutorial/README.md": "b4a81ff457624b089d618e4479afb1e3d748f6724f72104ef5518e535ae02fe5", + "pages/students/2016/dominik_nagy/README.md": "1c4037a158d84c45fa705a0c38e47142a9d46e226f0c138bc2d56d7c38f6d74d", + "pages/students/2016/dominik_nagy/timovy_projekt/README.md": "b81a4dab7316f7b846dcf36515f6aa9d44035c86f46f0e07b7a43308bcd31458", + "pages/students/2016/ediz_morochovic/README.md": "045e5eb9a3df2b84140808fbca0c3163266ad0efa144bc42d9d5c597881c6ac6", + "pages/students/2016/jakub_maruniak/README.md": "6734c41a95b9d0c1e3d226b393ad109ee4a7b82ebfffa73df2d7e0a743fefa14", + "pages/students/2016/jakub_maruniak/dp2021/README.md": "9cd884aec70705cdb2aaa0f514b513f16de263c37b749ced7f92916cd2f91c39", + "pages/students/2016/jakub_maruniak/dp2021/annotation/README.md": "debae0c50aea1accef0840912b9b1f37b2138f856002bb50e91d7ea2dbbe7963", + "pages/students/2016/jakub_maruniak/dp2021/train_experiments/README.md": "01dff38b37043414fa1db204394ddbe7aade00e14d02d495572b65c85d09b194", + "pages/students/2016/jakub_maruniak/timovy_projekt/README.md": "04114b1f3d84664aad5bdba5a0b614c5a6f23392be3c31fd9123d63db1be0a35", + "pages/students/2016/jan_holp/README.md": "80d6a8492766e5bfa61673a372e5f059d30c8e335de39842866c51485960e4d3", + "pages/students/2016/jan_holp/dp2021/README.md": "a62acb74009376766c7e2f9856ee01d21f84593eb9b3ce85cf2e6032aa412476", + "pages/students/2016/jan_holp/dp2021/zdrojove_subory/README.md": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "pages/students/2016/jan_holp/timovy_projekt/README.md": "79eebdfd3af7e8d5c279077e285a580731b1cf8cc9b4454f6596796e9915f70c", + "pages/students/2016/lukas_pokryvka/README.md": "f279456c5c277ff1ee0dd89a3464ed9f9f67b44bc185a5da9811ef5cfb675002", + "pages/students/2016/lukas_pokryvka/dp2021/README.md": "39b61376ce108e4a930f926ab2d6a997207457121bccfcf617129a3b210a15cf", + "pages/students/2016/lukas_pokryvka/timovy_projekt/README.md": "4de4b6c8cfed17b6c60090f82df1d55a66be70a79d3013850689b2baebdd801b", + "pages/students/2016/mark_feher/README.md": "bec37c0319fe0869a0377d771db23d090dbda93c27328f5697cdaeb3bbccfe9b", + "pages/students/2016/maros_harahus/README.md": "0da150e91918632c9e8cd1443f04da403595a372517e254928ce9f76c9d5033d", + "pages/students/2016/maros_harahus/timovy_projekt/README.md": "75af19f7c68f4b646fa1dbae75f3f3dfc52633a9f9da1400d21808bb20be0d83", + "pages/students/2016/patrik_pavlisin/README.md": "dc1c648d7cd9b0f0e021c57f9b98189e37c100871439e7e304390715913f2763", + "pages/students/2016/patrik_pavlisin/dp21/README.md": "ac1eb0fd0360ad34aa21b7c8da5bb010a6edf10be5721f28dedc3986cd1d8c54", + "pages/students/2016/patrik_pavlisin/dp22/README.md": "f2a8c619a812b99abab618ae388eb672407950e1f33fb8d7da7a528c4de81d50", + "pages/students/2016/patrik_pavlisin/tp20/README.md": "ae9c771f27e7e093bc29c6888c8d0b27ed61fb41519fd0d7e7f4f6cbb4626edc", + "pages/students/2016/tomas_kucharik/README.md": "6a1a215433eb1857660177b4cb2ecec03c6a86743d3a04cd6d5c9d9682c9e63b", + "pages/students/2017/dmytro_ushatenko/README.md": "65f4ac177d492162b08ca5da3d9eb048bd088a3c117a28a662113b7e4bc3a4ec", + "pages/students/2017/martin_jancura/README.md": "6a0a18d5095dcfb12abfdd72c9cfd8025d714fb5e29a7858cd2b285e4cb986b9", + "pages/students/2017/stanislav_matsunych/README.md": "90ffa11eff4a4360b0252dbc53a116c3a5777b3d5dd86ce5c89c02bdd2051c73", + "pages/students/2017/than_trung_thanh/README.md": "2150519b8580e7337520db4de64d2d1c82d2634380537d019f41c506dbf2979d", + "pages/students/2018/NaN1name_buday/README.md": "23c8c58090bdc954347e36f141bea91f43c8d5b1d777e346864a2ebe84c0bcce", + "pages/students/2018/adrian_remias/README.md": "e5bfe1d5d4e626beaa636a02d34ec58cf2cff9b0e57833905af35b80bf1465a2", + "pages/students/2018/david_ilas/README.md": "96c45ff9c5a821ddcb1a2fc86d487bd8f4729d1b76887914b0f5fbf99af45af0", + "pages/students/2018/david_omasta/README.md": "4164051870e1e1cf8949ed18a74196e9ea789dfbe4fee24bdeb736ca42848327", + "pages/students/2018/martin_wencel/README.md": "0e6a7a981689f22304f8baa5de97d4315c0695c94c9277de7851943eb012253b", + "pages/students/2018/oleh_bilykh/README.md": "4072c62dd8ee753a0c8fa43fdfbbdf566e394c2df13e6a79d08a302e332710b1", + "pages/students/2018/ondrej_megela/README.md": "969ff2e1a436b8f957e7168cf5eb678196b99aeda17701c502b016441f644065", + "pages/students/2018/samuel_sirotnik/README.md": "cc230ede67b6e8632511c92a5ed6d1e9deb42e70819280222bdfd369ee77fc70", + "pages/students/2019/alina_vitko/README.md": "9b071fe2aff470c275ae027be236fe836f741f774cabb9cbbb55304a8d5102cd", + "pages/students/2019/artem_yatsenko/README.md": "328b101e46c51c959e792cf7ca226942f81d2ab8fe9648ad6d0ecd07016429e2", + "pages/students/2019/david_stromp/README.md": "e46ca8691d5ec894128f6f6f2ed33e2c7bd141f80719bb0449b20f10f30cca09", + "pages/students/2019/dmytro_mural/README.md": "e881a47d847c3ef8d680844113fdbfbb87553b28982bf9eef4936f6d30923e93", + "pages/students/2019/filip_tomas/README.md": "75a1fe7b921e4bff0fdf9e2a957b4adad4b4a57aa7ebdffbb73dd82c02123f94", + "pages/students/2019/kristian_sopkovic/README.md": "3286960fb30f584df35455f9b6dfae81ca4827a1d711d2bcd6e48c3f150c9279", + "pages/students/2019/michal_stromko/README.md": "3ae1dd9b6638a6a249a94be3f3846286555666aefa225bb3a2c7288b35900ce7", + "pages/students/2019/michal_stromko/vp2021/README.md": "feb2e093fdd6461239e79b1e1f96e65a1c8cd96e8a2b70535ae556a9c198046a", + "pages/students/2019/michal_stromko/vp2021/tutorials/README.md": "40e3fa448c7329ef863e705435e23fe6b4788de440710f066fe0eed92d132bad", + "pages/students/2019/michal_stromko/vp2021/tutorials/img/README.md": "e8d3ac160b17dfdee2c3e996bf6a03af7a6f16f9140bf5cf7ee343a82dd1865f", + "pages/students/2019/patrik_pokrivcak/README.md": "3e3957ad0f7aca59d6eb64c38a3f5e21d8fd953200825d0d0e5cacabfbc35a7d", + "pages/students/2019/samuel_horani/README.md": "86e6959ccc612b069687b57996fa3f61660075d6a0d65bfec64f423d39a0a999", + "pages/students/2020/david_kostilnik/README.md": "2927524b75da59a64120fc364a0fc374986fa4d364d54e52c36800762c203264", + "pages/students/2020/jakub_kristian_lukas/README.md": "f139d9b1ebf5a3f45612f3df5703ecdd935b90701763c27c7c529044c0b0455a", + "pages/students/2020/kamil_tomcufcik/README.md": "9d865adf538bc6d7992ee4f4882713447da5c936fca17c43ca0f576072537048", + "pages/students/2020/matej_kobylan/README.md": "3d7b289e17b8f25678a07e9ee0e6741688485a0ccb1e50940a6a6acc6312a547", + "pages/students/2020/matus_suchanic/README.md": "770ca49538cca22df35f8e7ba736ee97483e4fe5788c221496553592b4e78dc6", + "pages/students/2020/pavol_hudak/README.md": "33d06fccc6451e5f19b27b0cad001a34e81635b4ea294a01dec7771c823abc40", + "pages/students/2020/vladyslav_krupko/README.md": "4c28e6d38651bb2cafcbf4a18e35210cd369aef2caff6c5b0aa52fac38f0fe7d", + "pages/students/2021/artur_hyrenko/README.md": "0568261f4ade4de5472edcfeb1ffed9abae18dc67e4050f4ce2ad088839a2bb3", + "pages/students/2021/eduard_matovka/README.md": "de2a7ae38ec87110e347956b37d0605a298ed2847f972961cc9b977da8221d43", + "pages/students/2021/manohar_gowdru_shridharu/README.md": "c00ceb620ed7087ead5d9448d476bbb98d273d1d9e7bf1bc4355c367b26c7a6b", + "pages/students/2021/martin_sarissky/README.md": "8e97b45a253745faa55f63c844e0b5dfe680aeaf6eb01fb8f9d06ac6e8ee8bcf", + "pages/students/2021/matej_novotny/README.md": "b083a7cadde532e3428dc9a57bc61456fcc5905bda91e011b96033d42bf5d9cb", + "pages/students/2021/matej_scislak/README.md": "b9a32cce8605e044a4f2828ff2d91a2995a62ef3a841e62f0a225c97221f0cc8", + "pages/students/2021/nikita_bodnar/README.md": "2016c0dd923c0f8e561d5cea42ac81da3610fdcc54a4527f56d078e34e53ce90", + "pages/students/2021/oleksandr_hryshchenko/README.md": "4b073992b715b6c75b79fd2c8f850a69f940966e3a61a82c625c32c945a93036", + "pages/students/2021/vladimir_ferko/README.md": "68e7d311896eda2171e5eec1ec3a8da90d326ed91acaace8058f68664a50d271", + "pages/students/2022/andrii_pervashov/README.md": "319dcaa1cc2e29dd7e3a710bab26cb3ac6944ab71b0b97e1de076b1199122334", + "pages/students/2022/daniil_huzenko/README.md": "390782fbff6220a9462060ce1a62959f2a2b88f42d06e2c51d1f406b75d73f2c", + "pages/students/2022/jakub_schwarc/README.md": "f541a8852c2b0f6f9d7182e01a34c4e3beee9bfa55427a06b331638b11eb8be6", + "pages/students/2022/jan_malinovsky/README.md": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab", + "pages/students/2022/jan_ptak/README.md": "131cf053764fd5423541ebe672c3b4f5da875cfb8e4b821f1335ddb967282282", + "pages/students/2022/oleh_poiasnik/README.md": "c2f7dc3f61f8b4a5b43ded1e3ae4c7d3e7f874439ee6b17dd9c8d45dabc25bc8", + "pages/students/2022/samuel_vasil/README.md": "67c807049147aacd04f3050f4ff0233a8b3f0a239ab173b430e13dfe113ad0a5", + "pages/students/2022/serhii_yemets/README.md": "ffbeac893947acafe855d576322a8826f9d4f87ac344cb31f8ceeefa244b9133", + "pages/students/2022/tetiana_mohorian/README.md": "c9efdc632bf1ca89a0bfde0dc00be7aaac8a85da3d92e2ce8506236c73e93cfb", + "pages/students/2022/valerii_kutsenko/README.md": "dc764c609ad3c1df59cfea997d4db82a591b1b1e0764453e319d841115507e4f", + "pages/students/2022/vladyslav_yanchenko/README.md": "4d1ce216e3b526ee437ddbc658e51d7bb83530087e43628c096a569a3854a47f", + "pages/students/2022/yevhenii_leonov/README.md": "b90e41db0cec2451027020a9102d32a24d777e8a38b604ccca87b36cd6b3a292", + "pages/students/2023/denis_le_thanh/README.md": "2cb83f942d86a02712b0c58778e46dc2211346d1a6eba606471dfa832245db61", + "pages/students/2023/maksym_zatirka/README.md": "7a7d2cbe276c352dacfbc6ba6de5f15303d30ecf34845ff8da8d085f3d67c3dd", + "pages/students/2023/oleksandr_dorybohov/README.md": "c24d6f941de0dafb4459cea8259cf147821cafc86ba65be5b528f419f734bc18", + "pages/students/2023/ronald_zalacko/README.md": "724b0f8e6e641236dba9bddbd77f4546e09514076c7dfb151807aaf4b882c96f", + "pages/students/2023/rostyslav_rodzhuk/README.md": "de356dd7427c8a0716c165a0f88ae47b691a86ee7a388f60e2d2e9306298415c", + "pages/students/2023/samuel_dzurina/README.md": "4818191d044b91219c796a35010c95da5cf2093e050d494f30b219b91e13fa0e", + "pages/students/2023/simona_bobrovcanova/README.md": "e473a1911dae10ab40e2cf6696a8be616a58c22aaf09af5953fbac475791a35e", + "pages/students/2023/yevhenii_medushivskyi/README.md": "5710ebf91c893a12fe6d19f5372311938d1e7c7bbdf203f2dc8ed2fe7c4f5a24", + "pages/topics/akopisat/README.md": "c7245e1fbfa4160618502c0baf8cc6bc47cd9a82b8a85bc1c645314829a7b76a", + "pages/topics/bert/README.md": "35e507f7ec2ad1d3526e2a15f94ea085ee25f680a02daf6df62fd429bd9c25c9", + "pages/topics/chatbot/README.md": "6f2f83ffb80109be47f9750cca9d3036ec3645abef9f2b590bd6825780c7c48a", + "pages/topics/faq/README.md": "15cfbd9c2a1649c80a7a58922e2b66dadab5c86a5ef33ef16c82bebd4d2dbf06", + "pages/topics/hatespeech/README.md": "364df94e70dd740c63544e2586ba7b957780f714fc0c941d1b04b02233e6cbc5", + "pages/topics/legal/README.md": "003bb9bfe1e0ca21652091c2d11496312eb04e0689fe459e14c2d5a738e8eeea", + "pages/topics/morphology/README.md": "e242cc5ff1a4cc98e227566c0ead999bbc95937b7c20be62cb2afb773d0d35c0", + "pages/topics/named-entity/README.md": "66b2e25f70211d432d596e377d552b3b8f0c525b14c31e3bc9c956c3aeadca84", + "pages/topics/named-entity/navod/README.md": "1ff87079a98411de0414754c60241e08d2214ddcd63ddf974e4340cce2ce5b02", + "pages/topics/nlp/README.md": "6f90abf9dc3f0ff33475283c13b4a08561abf631425ec2daacb076155a9e02c2", + "pages/topics/otvorene/README.md": "cd5544925553fe3840c0ea6f8ef60ab756e26ab6757bf810025c948d8d4f98fc", + "pages/topics/prodigy/README.md": "5e4f691f2754c2ec4afbaab2f41e6db77feb5b322dd9344465eb40c35f116499", + "pages/topics/python/README.md": "2ef073adba672142c39e9a59fa2e00a98ab52bda54233d46fbe6940ee23f9fcc", + "pages/topics/question/README.md": "408078a2a67cea2699e12359e85ab641136231e6104c2afdf874a705879525df", + "pages/topics/question/navod/README.md": "061e737c69184b2bd244316ca7ba841f7186fe01ee80ee993742a733dcad3ecd", + "pages/topics/question/nezodpovedatelne/README.md": "b6703d9690f4599b973efb27595ce157773ae0fc0d55cf2c0a09d9f50e8e4c92", + "pages/topics/question/validacie/README.md": "3216a7127abdfc6da50ed116f44ff2d6785e7da4af70d23aceda774e745ad562", + "pages/topics/resources/README.md": "09405397387f00cbcc5680f7d655f2d861e4d92e5df792a62d9dfd04f3534f48", + "pages/topics/seq2seq/README.md": "969d15c03a63a3f8d5edc9017fe5a0fb7311fdf4de4a3473a08266241f92c1e2", + "pages/topics/spacy/README.md": "f504e5db77cb8694f08a57fb8636e522d59a0e669216f4bc8273bc3c5f375f87", + "pages/topics/steel/README.md": "b4068cef2ed197711467ac47630d5b2064fdc8828f898f7af99664e94d833330", + "pages/topics/translation/README.md": "a7d4289e72199b307dc5ad8fc69cda39c1a33c27a6c7447af70aa9c3cc6b8635" + } + }, + "category_counts": { + "fact_title": 10, + "fact_work_year": 10, + "fact_start_year": 8, + "disambiguation_year": 8, + "multi_document": 8, + "single_document_synthesis": 8, + "specific_fact": 10, + "keyword_grounding": 8, + "unanswerable_missing_fact": 10 + }, + "cases": [ + { + "id": "q0053", + "category": "fact_title", + "question": "Aký je názov diplomová práca osoby Lukáš Pokrývka?", + "expected_answer_contains": [ + "Paralelné trénovanie neurónových sietí" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Lukáš Pokrývka", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/lukas_pokryvka/README.md", + "heading_line": 18, + "title_line": 14, + "title": "Paralelné trénovanie neurónových sietí" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/lukas_pokryvka/README.md", + "sha256": "f279456c5c277ff1ee0dd89a3464ed9f9f67b44bc185a5da9811ef5cfb675002" + } + ] + }, + { + "id": "q0318", + "category": "fact_title", + "question": "Aký je názov diplomová práca osoby Oleh Poiasnik?", + "expected_answer_contains": [ + "Expertný agentový systém na podporu rozhodovania v lekárni" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Oleh Poiasnik", + "work_type": "diploma_thesis", + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "heading_line": 14, + "title_line": 16, + "title": "Expertný agentový systém na podporu rozhodovania v lekárni" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "sha256": "c2f7dc3f61f8b4a5b43ded1e3ae4c7d3e7f874439ee6b17dd9c8d45dabc25bc8" + } + ] + }, + { + "id": "q0133", + "category": "fact_title", + "question": "Aký je názov bakalárska práca osoby Martin Wencel?", + "expected_answer_contains": [ + "Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Martin Wencel", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2018/martin_wencel/README.md", + "heading_line": 17, + "title_line": 19, + "title": "Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2018/martin_wencel/README.md", + "sha256": "0e6a7a981689f22304f8baa5de97d4315c0695c94c9277de7851943eb012253b" + } + ] + }, + { + "id": "q0289", + "category": "fact_title", + "question": "Aký je názov diplomová práca osoby Andrii Pervashov?", + "expected_answer_contains": [ + "Prepis reči pre tvorbu štruktúrovaného zdravotného záznamu" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/andrii_pervashov" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Andrii Pervashov", + "work_type": "diploma_thesis", + "document_path": "pages/students/2022/andrii_pervashov/README.md", + "heading_line": 13, + "title_line": 17, + "title": "Prepis reči pre tvorbu štruktúrovaného zdravotného záznamu" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/andrii_pervashov/README.md", + "sha256": "319dcaa1cc2e29dd7e3a710bab26cb3ac6944ab71b0b97e1de076b1199122334" + } + ] + }, + { + "id": "q0410", + "category": "fact_title", + "question": "Aký je názov bakalárska práca osoby Maksym Zatirka?", + "expected_answer_contains": [ + "Grafová databáza pre podporu generovania slovenského jazyka" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/maksym_zatirka" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Maksym Zatirka", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/maksym_zatirka/README.md", + "heading_line": 16, + "title_line": 21, + "title": "Grafová databáza pre podporu generovania slovenského jazyka" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/maksym_zatirka/README.md", + "sha256": "7a7d2cbe276c352dacfbc6ba6de5f15303d30ecf34845ff8da8d085f3d67c3dd" + } + ] + }, + { + "id": "q0757", + "category": "fact_title", + "question": "Aký názov práce je pri osobe Matej Ščišľak uvedený pre rok 2026?", + "expected_answer_contains": [ + "Znalostné grafy pre zlepšenie vyhľadávania právnych informácií" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Matej Ščišľak", + "work_type": "diploma_thesis", + "document_path": "pages/students/2021/matej_scislak/README.md", + "heading_line": 13, + "title_line": 17, + "title": "Znalostné grafy pre zlepšenie vyhľadávania právnych informácií." + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/matej_scislak/README.md", + "sha256": "b9a32cce8605e044a4f2828ff2d91a2995a62ef3a841e62f0a225c97221f0cc8" + } + ] + }, + { + "id": "q0767", + "category": "fact_title", + "question": "Aký názov práce je pri osobe Vladyslav Yanchenko uvedený pre rok 2025?", + "expected_answer_contains": [ + "Kontinuálne nasadenie a testovanie aplikácie v klaudovom prostredí" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/vladyslav_yanchenko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Vladyslav Yanchenko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2022/vladyslav_yanchenko/README.md", + "heading_line": 13, + "title_line": 17, + "title": "Kontinuálne nasadenie a testovanie aplikácie v klaudovom prostredí." + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/vladyslav_yanchenko/README.md", + "sha256": "4d1ce216e3b526ee437ddbc658e51d7bb83530087e43628c096a569a3854a47f" + } + ] + }, + { + "id": "q0745", + "category": "fact_title", + "question": "Aký názov práce je pri osobe Dmytro Ushatenko uvedený pre rok 2020?", + "expected_answer_contains": [ + "Jednoduchý chatbot v jazyku Python" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2017/dmytro_ushatenko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Dmytro Ushatenko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2017/dmytro_ushatenko/README.md", + "heading_line": 11, + "title_line": 14, + "title": "Jednoduchý chatbot v jazyku Python," + } + ], + "source_checks": [ + { + "document_path": "pages/students/2017/dmytro_ushatenko/README.md", + "sha256": "65f4ac177d492162b08ca5da3d9eb048bd088a3c117a28a662113b7e4bc3a4ec" + } + ] + }, + { + "id": "q0215", + "category": "fact_title", + "question": "Aký je názov diplomová práca osoby Martin Šarišský?", + "expected_answer_contains": [ + "Agentové systémy pomocou veľkého jazykového modelu" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Martin Šarišský", + "work_type": "diploma_thesis", + "document_path": "pages/students/2021/martin_sarissky/README.md", + "heading_line": 12, + "title_line": 18, + "title": "Agentové systémy pomocou veľkého jazykového modelu" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/martin_sarissky/README.md", + "sha256": "8e97b45a253745faa55f63c844e0b5dfe680aeaf6eb01fb8f9d06ac6e8ee8bcf" + } + ] + }, + { + "id": "q0467", + "category": "fact_title", + "question": "Aký je názov bakalárska práca osoby Oleksandr Dorybohov?", + "expected_answer_contains": [ + "Automatické odpovede na otázky v právnej oblasti" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Oleksandr Dorybohov", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "heading_line": 14, + "title_line": 20, + "title": "Automatické odpovede na otázky v právnej oblasti" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "sha256": "c24d6f941de0dafb4459cea8259cf147821cafc86ba65be5b528f419f734bc18" + } + ] + }, + { + "id": "q0059", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Lukáš Pokrývka uvedená diplomová práca?", + "expected_answer_contains": [ + "2021" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Lukáš Pokrývka", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/lukas_pokryvka/README.md", + "heading_line": 18, + "title_line": 14, + "year": 2021, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/lukas_pokryvka/README.md", + "sha256": "f279456c5c277ff1ee0dd89a3464ed9f9f67b44bc185a5da9811ef5cfb675002" + } + ] + }, + { + "id": "q0532", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Denis Le Thanh uvedená bakalárska práca?", + "expected_answer_contains": [ + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/denis_le_thanh" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Denis Le Thanh", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/denis_le_thanh/README.md", + "heading_line": 19, + "title_line": 24, + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/denis_le_thanh/README.md", + "sha256": "2cb83f942d86a02712b0c58778e46dc2211346d1a6eba606471dfa832245db61" + } + ] + }, + { + "id": "q0473", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Oleksandr Dorybohov uvedená bakalárska práca?", + "expected_answer_contains": [ + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Oleksandr Dorybohov", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "heading_line": 14, + "title_line": 20, + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "sha256": "c24d6f941de0dafb4459cea8259cf147821cafc86ba65be5b528f419f734bc18" + } + ] + }, + { + "id": "q0192", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Alina Vitko uvedená bakalárska práca?", + "expected_answer_contains": [ + "2022" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Alina Vitko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2019/alina_vitko/README.md", + "heading_line": 12, + "title_line": 14, + "year": 2022, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2019/alina_vitko/README.md", + "sha256": "9b071fe2aff470c275ae027be236fe836f741f774cabb9cbbb55304a8d5102cd" + } + ] + }, + { + "id": "q0459", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Ronald Zalacko uvedená bakalárska práca?", + "expected_answer_contains": [ + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Ronald Zalacko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/ronald_zalacko/README.md", + "heading_line": 17, + "title_line": 22, + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/ronald_zalacko/README.md", + "sha256": "724b0f8e6e641236dba9bddbd77f4546e09514076c7dfb151807aaf4b882c96f" + } + ] + }, + { + "id": "q0267", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Eduard Matovka uvedená bakalárska práca?", + "expected_answer_contains": [ + "2024" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/eduard_matovka" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Eduard Matovka", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2021/eduard_matovka/README.md", + "heading_line": 12, + "title_line": 18, + "year": 2024, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/eduard_matovka/README.md", + "sha256": "de2a7ae38ec87110e347956b37d0605a298ed2847f972961cc9b977da8221d43" + } + ] + }, + { + "id": "q0517", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Samuel Vasiľ uvedená bakalárska práca?", + "expected_answer_contains": [ + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/samuel_vasil" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Samuel Vasiľ", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2022/samuel_vasil/README.md", + "heading_line": 14, + "title_line": 20, + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/samuel_vasil/README.md", + "sha256": "67c807049147aacd04f3050f4ff0233a8b3f0a239ab173b430e13dfe113ad0a5" + } + ] + }, + { + "id": "q0074", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Dárius Lindvai uvedená diplomová práca?", + "expected_answer_contains": [ + "2021" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Dárius Lindvai", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/darius_lindvai/README.md", + "heading_line": null, + "title_line": 14, + "year": 2021, + "year_basis": "frontmatter_category" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/darius_lindvai/README.md", + "sha256": "fcaa47bb9708a6439eec5c4b7f7b4ed6c7359e423e1d940e1aed9f44957e6d62" + } + ] + }, + { + "id": "q0416", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Maksym Zatirka uvedená bakalárska práca?", + "expected_answer_contains": [ + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/maksym_zatirka" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Maksym Zatirka", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/maksym_zatirka/README.md", + "heading_line": 16, + "title_line": 21, + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/maksym_zatirka/README.md", + "sha256": "7a7d2cbe276c352dacfbc6ba6de5f15303d30ecf34845ff8da8d085f3d67c3dd" + } + ] + }, + { + "id": "q0117", + "category": "fact_work_year", + "question": "V ktorom roku je pri osobe Dmytro Ushatenko uvedená bakalárska práca?", + "expected_answer_contains": [ + "2020" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2017/dmytro_ushatenko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Dmytro Ushatenko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2017/dmytro_ushatenko/README.md", + "heading_line": 11, + "title_line": 14, + "year": 2020, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2017/dmytro_ushatenko/README.md", + "sha256": "65f4ac177d492162b08ca5da3d9eb048bd088a3c117a28a662113b7e4bc3a4ec" + } + ] + }, + { + "id": "q0515", + "category": "fact_start_year", + "question": "V ktorom roku začal študovať Samuel Vasiľ?", + "expected_answer_contains": [ + "2022" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/samuel_vasil" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Samuel Vasiľ", + "start_year": 2022, + "document_path": "pages/students/2022/samuel_vasil/README.md", + "basis": "explicit_text" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/samuel_vasil/README.md", + "sha256": "67c807049147aacd04f3050f4ff0233a8b3f0a239ab173b430e13dfe113ad0a5" + } + ] + }, + { + "id": "q0353", + "category": "fact_start_year", + "question": "V ktorom roku začal študovať Vladyslav Yanchenko?", + "expected_answer_contains": [ + "2022" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/vladyslav_yanchenko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Vladyslav Yanchenko", + "start_year": 2022, + "document_path": "pages/students/2022/vladyslav_yanchenko/README.md", + "basis": "explicit_text" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/vladyslav_yanchenko/README.md", + "sha256": "4d1ce216e3b526ee437ddbc658e51d7bb83530087e43628c096a569a3854a47f" + } + ] + }, + { + "id": "q0471", + "category": "fact_start_year", + "question": "V ktorom roku začal študovať Oleksandr Dorybohov?", + "expected_answer_contains": [ + "2023" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Oleksandr Dorybohov", + "start_year": 2023, + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "basis": "explicit_text" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "sha256": "c24d6f941de0dafb4459cea8259cf147821cafc86ba65be5b528f419f734bc18" + } + ] + }, + { + "id": "q0219", + "category": "fact_start_year", + "question": "V ktorom roku začal študovať Martin Šarišský?", + "expected_answer_contains": [ + "2021" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Martin Šarišský", + "start_year": 2021, + "document_path": "pages/students/2021/martin_sarissky/README.md", + "basis": "explicit_text" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/martin_sarissky/README.md", + "sha256": "8e97b45a253745faa55f63c844e0b5dfe680aeaf6eb01fb8f9d06ac6e8ee8bcf" + } + ] + }, + { + "id": "q0265", + "category": "fact_start_year", + "question": "V ktorom roku začal študovať Eduard Matovka?", + "expected_answer_contains": [ + "2021" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/eduard_matovka" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Eduard Matovka", + "start_year": 2021, + "document_path": "pages/students/2021/eduard_matovka/README.md", + "basis": "explicit_text" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/eduard_matovka/README.md", + "sha256": "de2a7ae38ec87110e347956b37d0605a298ed2847f972961cc9b977da8221d43" + } + ] + }, + { + "id": "q0322", + "category": "fact_start_year", + "question": "V ktorom roku začal študovať Oleh Poiasnik?", + "expected_answer_contains": [ + "2022" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Oleh Poiasnik", + "start_year": 2022, + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "basis": "explicit_text" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "sha256": "c2f7dc3f61f8b4a5b43ded1e3ae4c7d3e7f874439ee6b17dd9c8d45dabc25bc8" + } + ] + }, + { + "id": "q0190", + "category": "fact_start_year", + "question": "V ktorom roku začal študovať Alina Vitko?", + "expected_answer_contains": [ + "2019" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Alina Vitko", + "start_year": 2019, + "document_path": "pages/students/2019/alina_vitko/README.md", + "basis": "explicit_text" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2019/alina_vitko/README.md", + "sha256": "9b071fe2aff470c275ae027be236fe836f741f774cabb9cbbb55304a8d5102cd" + } + ] + }, + { + "id": "q0072", + "category": "fact_start_year", + "question": "V ktorom roku začal študovať Dárius Lindvai?", + "expected_answer_contains": [ + "2016" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Dárius Lindvai", + "start_year": 2016, + "document_path": "pages/students/2016/darius_lindvai/README.md", + "basis": "explicit_text" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/darius_lindvai/README.md", + "sha256": "fcaa47bb9708a6439eec5c4b7f7b4ed6c7359e423e1d940e1aed9f44957e6d62" + } + ] + }, + { + "id": "q0060", + "category": "disambiguation_year", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Lukáš Pokrývka.", + "expected_answer_contains": [ + "2016", + "2021" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Lukáš Pokrývka", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/lukas_pokryvka/README.md", + "heading_line": 18, + "title_line": 14, + "year": 2021, + "start_year": 2016, + "start_year_basis": "explicit_text", + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/lukas_pokryvka/README.md", + "sha256": "f279456c5c277ff1ee0dd89a3464ed9f9f67b44bc185a5da9811ef5cfb675002" + } + ] + }, + { + "id": "q0402", + "category": "disambiguation_year", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Ján Malinovský.", + "expected_answer_contains": [ + "2022", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Ján Malinovský", + "work_type": "diploma_project", + "document_path": "pages/students/2022/jan_malinovsky/README.md", + "heading_line": 13, + "title_line": 18, + "year": 2026, + "start_year": 2022, + "start_year_basis": "explicit_text", + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/jan_malinovsky/README.md", + "sha256": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab" + } + ] + }, + { + "id": "q0325", + "category": "disambiguation_year", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Oleh Poiasnik.", + "expected_answer_contains": [ + "2022", + "2027" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Oleh Poiasnik", + "work_type": "diploma_thesis", + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "heading_line": 14, + "title_line": 16, + "year": 2027, + "start_year": 2022, + "start_year_basis": "explicit_text", + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "sha256": "c2f7dc3f61f8b4a5b43ded1e3ae4c7d3e7f874439ee6b17dd9c8d45dabc25bc8" + } + ] + }, + { + "id": "q0356", + "category": "disambiguation_year", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Vladyslav Yanchenko.", + "expected_answer_contains": [ + "2022", + "2025" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/vladyslav_yanchenko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Vladyslav Yanchenko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2022/vladyslav_yanchenko/README.md", + "heading_line": 13, + "title_line": 17, + "year": 2025, + "start_year": 2022, + "start_year_basis": "explicit_text", + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/vladyslav_yanchenko/README.md", + "sha256": "4d1ce216e3b526ee437ddbc658e51d7bb83530087e43628c096a569a3854a47f" + } + ] + }, + { + "id": "q0222", + "category": "disambiguation_year", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Martin Šarišský.", + "expected_answer_contains": [ + "2021", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Martin Šarišský", + "work_type": "diploma_thesis", + "document_path": "pages/students/2021/martin_sarissky/README.md", + "heading_line": 12, + "title_line": 18, + "year": 2026, + "start_year": 2021, + "start_year_basis": "explicit_text", + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/martin_sarissky/README.md", + "sha256": "8e97b45a253745faa55f63c844e0b5dfe680aeaf6eb01fb8f9d06ac6e8ee8bcf" + } + ] + }, + { + "id": "q0296", + "category": "disambiguation_year", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Andrii Pervashov.", + "expected_answer_contains": [ + "2022", + "2027" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/andrii_pervashov" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Andrii Pervashov", + "work_type": "diploma_thesis", + "document_path": "pages/students/2022/andrii_pervashov/README.md", + "heading_line": 13, + "title_line": 17, + "year": 2027, + "start_year": 2022, + "start_year_basis": "explicit_text", + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/andrii_pervashov/README.md", + "sha256": "319dcaa1cc2e29dd7e3a710bab26cb3ac6944ab71b0b97e1de076b1199122334" + } + ] + }, + { + "id": "q0518", + "category": "disambiguation_year", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Samuel Vasiľ.", + "expected_answer_contains": [ + "2022", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/samuel_vasil" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Samuel Vasiľ", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2022/samuel_vasil/README.md", + "heading_line": 14, + "title_line": 20, + "year": 2026, + "start_year": 2022, + "start_year_basis": "explicit_text", + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/samuel_vasil/README.md", + "sha256": "67c807049147aacd04f3050f4ff0233a8b3f0a239ab173b430e13dfe113ad0a5" + } + ] + }, + { + "id": "q0460", + "category": "disambiguation_year", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Ronald Zalacko.", + "expected_answer_contains": [ + "2023", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/ronald_zalacko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Ronald Zalacko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/ronald_zalacko/README.md", + "heading_line": 17, + "title_line": 22, + "year": 2026, + "start_year": 2023, + "start_year_basis": "explicit_text", + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/ronald_zalacko/README.md", + "sha256": "724b0f8e6e641236dba9bddbd77f4546e09514076c7dfb151807aaf4b882c96f" + } + ] + }, + { + "id": "q1147", + "category": "multi_document", + "question": "Porovnaj záznamy osôb Ján Holp a Ondrej Megela. Pri osobe Ján Holp použi záznam typu „diplomová práca“ a pri osobe Ondrej Megela záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "expected_answer_contains": [ + "Systém získavania informácií v slovenskom jazyku", + "2021", + "Neurónové jazykové modelovanie typu BERT", + "2020" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Ján Holp", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/jan_holp/README.md", + "heading_line": 13, + "title_line": 15, + "title": "Systém získavania informácií v slovenskom jazyku.", + "year": 2021, + "year_basis": "heading" + }, + { + "person": "Ondrej Megela", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2018/ondrej_megela/README.md", + "heading_line": 265, + "title_line": 268, + "title": "Neurónové jazykové modelovanie typu BERT.", + "year": 2020, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/jan_holp/README.md", + "sha256": "80d6a8492766e5bfa61673a372e5f059d30c8e335de39842866c51485960e4d3" + }, + { + "document_path": "pages/students/2018/ondrej_megela/README.md", + "sha256": "969ff2e1a436b8f957e7168cf5eb678196b99aeda17701c502b016441f644065" + } + ] + }, + { + "id": "q1148", + "category": "multi_document", + "question": "Porovnaj záznamy osôb Márk Fehér a Martin Wencel. Pri osobe Márk Fehér použi záznam typu „diplomová práca“ a pri osobe Martin Wencel záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "expected_answer_contains": [ + "Klasifikácia textu metódami strojového učenia", + "2022", + "Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy", + "2020" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/mark_feher", + "https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Márk Fehér", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/mark_feher/README.md", + "heading_line": 9, + "title_line": 11, + "title": "Klasifikácia textu metódami strojového učenia", + "year": 2022, + "year_basis": "heading" + }, + { + "person": "Martin Wencel", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2018/martin_wencel/README.md", + "heading_line": 17, + "title_line": 19, + "title": "Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy", + "year": 2020, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/mark_feher/README.md", + "sha256": "bec37c0319fe0869a0377d771db23d090dbda93c27328f5697cdaeb3bbccfe9b" + }, + { + "document_path": "pages/students/2018/martin_wencel/README.md", + "sha256": "0e6a7a981689f22304f8baa5de97d4315c0695c94c9277de7851943eb012253b" + } + ] + }, + { + "id": "q1154", + "category": "multi_document", + "question": "Porovnaj záznamy osôb Dominik Nagy a Valerii Kutsenko. Pri osobe Dominik Nagy použi záznam typu „diplomová práca“ a pri osobe Valerii Kutsenko záznam typu „diplomová práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "expected_answer_contains": [ + "Neurónový strojový preklad pomocou knižnice Fairseq", + "2022", + "Zlepšenie vyhľadávania pomocou znalostných grafov", + "2027" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy", + "https://zp.kemt.fei.tuke.sk/students/2022/valerii_kutsenko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Dominik Nagy", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/dominik_nagy/README.md", + "heading_line": 13, + "title_line": 17, + "title": "Neurónový strojový preklad pomocou knižnice Fairseq", + "year": 2022, + "year_basis": "heading" + }, + { + "person": "Valerii Kutsenko", + "work_type": "diploma_thesis", + "document_path": "pages/students/2022/valerii_kutsenko/README.md", + "heading_line": 12, + "title_line": 14, + "title": "Zlepšenie vyhľadávania pomocou znalostných grafov.", + "year": 2027, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/dominik_nagy/README.md", + "sha256": "1c4037a158d84c45fa705a0c38e47142a9d46e226f0c138bc2d56d7c38f6d74d" + }, + { + "document_path": "pages/students/2022/valerii_kutsenko/README.md", + "sha256": "dc764c609ad3c1df59cfea997d4db82a591b1b1e0764453e319d841115507e4f" + } + ] + }, + { + "id": "q1149", + "category": "multi_document", + "question": "Porovnaj záznamy osôb Martin Šarišský a Samuel Džurina. Pri osobe Martin Šarišský použi záznam typu „diplomová práca“ a pri osobe Samuel Džurina záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "expected_answer_contains": [ + "Agentové systémy pomocou veľkého jazykového modelu", + "2026", + "Dialógový systém pre podporu firemných procesov", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky", + "https://zp.kemt.fei.tuke.sk/students/2023/samuel_dzurina" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Martin Šarišský", + "work_type": "diploma_thesis", + "document_path": "pages/students/2021/martin_sarissky/README.md", + "heading_line": 12, + "title_line": 18, + "title": "Agentové systémy pomocou veľkého jazykového modelu", + "year": 2026, + "year_basis": "heading" + }, + { + "person": "Samuel Džurina", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/samuel_dzurina/README.md", + "heading_line": 14, + "title_line": 19, + "title": "Dialógový systém pre podporu firemných procesov", + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/martin_sarissky/README.md", + "sha256": "8e97b45a253745faa55f63c844e0b5dfe680aeaf6eb01fb8f9d06ac6e8ee8bcf" + }, + { + "document_path": "pages/students/2023/samuel_dzurina/README.md", + "sha256": "4818191d044b91219c796a35010c95da5cf2093e050d494f30b219b91e13fa0e" + } + ] + }, + { + "id": "q1156", + "category": "multi_document", + "question": "Porovnaj záznamy osôb Samuel Vasiľ a Oleksandr Dorybohov. Pri osobe Samuel Vasiľ použi záznam typu „bakalárska práca“ a pri osobe Oleksandr Dorybohov záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "expected_answer_contains": [ + "Orchestrácia veľkých jazykových modelov pomocou model context protokolu s využitím OpenWebUI", + "2026", + "Automatické odpovede na otázky v právnej oblasti", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/samuel_vasil", + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Samuel Vasiľ", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2022/samuel_vasil/README.md", + "heading_line": 14, + "title_line": 20, + "title": "Orchestrácia veľkých jazykových modelov pomocou model context protokolu s využitím OpenWebUI", + "year": 2026, + "year_basis": "heading" + }, + { + "person": "Oleksandr Dorybohov", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "heading_line": 14, + "title_line": 20, + "title": "Automatické odpovede na otázky v právnej oblasti", + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/samuel_vasil/README.md", + "sha256": "67c807049147aacd04f3050f4ff0233a8b3f0a239ab173b430e13dfe113ad0a5" + }, + { + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "sha256": "c24d6f941de0dafb4459cea8259cf147821cafc86ba65be5b528f419f734bc18" + } + ] + }, + { + "id": "q1162", + "category": "multi_document", + "question": "Porovnaj záznamy osôb Vladimír Ferko a Oleksandr Dorybohov. Pri osobe Vladimír Ferko použi záznam typu „bakalárska práca“ a pri osobe Oleksandr Dorybohov záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "expected_answer_contains": [ + "Anotácia a vyhodnotenie slovenskej databázy nenávistnej reči", + "2024", + "Automatické odpovede na otázky v právnej oblasti", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko", + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Vladimír Ferko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2021/vladimir_ferko/README.md", + "heading_line": 14, + "title_line": 21, + "title": "Anotácia a vyhodnotenie slovenskej databázy nenávistnej reči", + "year": 2024, + "year_basis": "heading" + }, + { + "person": "Oleksandr Dorybohov", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "heading_line": 14, + "title_line": 20, + "title": "Automatické odpovede na otázky v právnej oblasti", + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/vladimir_ferko/README.md", + "sha256": "68e7d311896eda2171e5eec1ec3a8da90d326ed91acaace8058f68664a50d271" + }, + { + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "sha256": "c24d6f941de0dafb4459cea8259cf147821cafc86ba65be5b528f419f734bc18" + } + ] + }, + { + "id": "q1166", + "category": "multi_document", + "question": "Porovnaj záznamy osôb Matej Ščišľak a Vladimír Ferko. Pri osobe Matej Ščišľak použi záznam typu „diplomová práca“ a pri osobe Vladimír Ferko záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "expected_answer_contains": [ + "Znalostné grafy pre zlepšenie vyhľadávania právnych informácií", + "2026", + "Anotácia a vyhodnotenie slovenskej databázy nenávistnej reči", + "2024" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/matej_scislak", + "https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Matej Ščišľak", + "work_type": "diploma_thesis", + "document_path": "pages/students/2021/matej_scislak/README.md", + "heading_line": 13, + "title_line": 17, + "title": "Znalostné grafy pre zlepšenie vyhľadávania právnych informácií.", + "year": 2026, + "year_basis": "heading" + }, + { + "person": "Vladimír Ferko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2021/vladimir_ferko/README.md", + "heading_line": 14, + "title_line": 21, + "title": "Anotácia a vyhodnotenie slovenskej databázy nenávistnej reči", + "year": 2024, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2021/matej_scislak/README.md", + "sha256": "b9a32cce8605e044a4f2828ff2d91a2995a62ef3a841e62f0a225c97221f0cc8" + }, + { + "document_path": "pages/students/2021/vladimir_ferko/README.md", + "sha256": "68e7d311896eda2171e5eec1ec3a8da90d326ed91acaace8058f68664a50d271" + } + ] + }, + { + "id": "q1168", + "category": "multi_document", + "question": "Porovnaj záznamy osôb Dárius Lindvai a Tetiana Mohorian. Pri osobe Dárius Lindvai použi záznam typu „diplomová práca“ a pri osobe Tetiana Mohorian záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "expected_answer_contains": [ + "Obnovenie interpunkcie pomocou hlbokých neurónových sietí", + "2021", + "Rozpoznávanie nenávistnej reči pomocou veľkých jazykových modelov", + "2025" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai", + "https://zp.kemt.fei.tuke.sk/students/2022/tetiana_mohorian" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Dárius Lindvai", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/darius_lindvai/README.md", + "heading_line": null, + "title_line": 14, + "title": "Obnovenie interpunkcie pomocou hlbokých neurónových sietí", + "year": 2021, + "year_basis": "frontmatter_category" + }, + { + "person": "Tetiana Mohorian", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2022/tetiana_mohorian/README.md", + "heading_line": 14, + "title_line": 21, + "title": "Rozpoznávanie nenávistnej reči pomocou veľkých jazykových modelov.", + "year": 2025, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/darius_lindvai/README.md", + "sha256": "fcaa47bb9708a6439eec5c4b7f7b4ed6c7359e423e1d940e1aed9f44957e6d62" + }, + { + "document_path": "pages/students/2022/tetiana_mohorian/README.md", + "sha256": "c9efdc632bf1ca89a0bfde0dc00be7aaac8a85da3d92e2ce8506236c73e93cfb" + } + ] + }, + { + "id": "q1024", + "category": "single_document_synthesis", + "question": "V dokumente osoby Jakub Maruniak, ktorý súvisí s témou „pomenované entity“, uveď názov záznamu typu „diplomová práca“ a rok, pri ktorom je táto práca uvedená.", + "expected_answer_contains": [ + "Anotácia a rozpoznávanie pomenovaných entít v slovenskom jazyku", + "2021" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Jakub Maruniak", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/jakub_maruniak/README.md", + "heading_line": 33, + "title_line": 15, + "title": "Anotácia a rozpoznávanie pomenovaných entít v slovenskom jazyku.", + "year": 2021, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/jakub_maruniak/README.md", + "sha256": "6734c41a95b9d0c1e3d226b393ad109ee4a7b82ebfffa73df2d7e0a743fefa14" + } + ] + }, + { + "id": "q1049", + "category": "single_document_synthesis", + "question": "V dokumente osoby Oleh Poiasnik, ktorý súvisí s témou „lekáreň“, uveď názov záznamu typu „diplomová práca“ a rok, pri ktorom je táto práca uvedená.", + "expected_answer_contains": [ + "Expertný agentový systém na podporu rozhodovania v lekárni", + "2027" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Oleh Poiasnik", + "work_type": "diploma_thesis", + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "heading_line": 14, + "title_line": 16, + "title": "Expertný agentový systém na podporu rozhodovania v lekárni", + "year": 2027, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "sha256": "c2f7dc3f61f8b4a5b43ded1e3ae4c7d3e7f874439ee6b17dd9c8d45dabc25bc8" + } + ] + }, + { + "id": "q1059", + "category": "single_document_synthesis", + "question": "V dokumente osoby Samuel Džurina, ktorý súvisí s témou „agent“, uveď názov záznamu typu „bakalárska práca“ a rok, pri ktorom je táto práca uvedená.", + "expected_answer_contains": [ + "Dialógový systém pre podporu firemných procesov", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/samuel_dzurina" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Samuel Džurina", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/samuel_dzurina/README.md", + "heading_line": 14, + "title_line": 19, + "title": "Dialógový systém pre podporu firemných procesov", + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/samuel_dzurina/README.md", + "sha256": "4818191d044b91219c796a35010c95da5cf2093e050d494f30b219b91e13fa0e" + } + ] + }, + { + "id": "q1035", + "category": "single_document_synthesis", + "question": "V dokumente osoby Alina Vitko, ktorý súvisí s témou „Question Answering“, uveď názov záznamu typu „bakalárska práca“ a rok, pri ktorom je táto práca uvedená.", + "expected_answer_contains": [ + "Systémy pre generovanie odpovede na otázku v prirodzenom jazyku", + "2022" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Alina Vitko", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2019/alina_vitko/README.md", + "heading_line": 12, + "title_line": 14, + "title": "Systémy pre generovanie odpovede na otázku v prirodzenom jazyku", + "year": 2022, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2019/alina_vitko/README.md", + "sha256": "9b071fe2aff470c275ae027be236fe836f741f774cabb9cbbb55304a8d5102cd" + } + ] + }, + { + "id": "q1063", + "category": "single_document_synthesis", + "question": "V dokumente osoby Dominik Nagy, ktorý súvisí s témou „MBART“, uveď názov záznamu typu „diplomová práca“ a rok, pri ktorom je táto práca uvedená.", + "expected_answer_contains": [ + "Neurónový strojový preklad pomocou knižnice Fairseq", + "2022" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Dominik Nagy", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/dominik_nagy/README.md", + "heading_line": 13, + "title_line": 17, + "title": "Neurónový strojový preklad pomocou knižnice Fairseq", + "year": 2022, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2016/dominik_nagy/README.md", + "sha256": "1c4037a158d84c45fa705a0c38e47142a9d46e226f0c138bc2d56d7c38f6d74d" + } + ] + }, + { + "id": "q1034", + "category": "single_document_synthesis", + "question": "V dokumente osoby Ondrej Megela, ktorý súvisí s témou „jazykové modelovanie“, uveď názov záznamu typu „bakalárska práca“ a rok, pri ktorom je táto práca uvedená.", + "expected_answer_contains": [ + "Neurónové jazykové modelovanie typu BERT", + "2020" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Ondrej Megela", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2018/ondrej_megela/README.md", + "heading_line": 265, + "title_line": 268, + "title": "Neurónové jazykové modelovanie typu BERT.", + "year": 2020, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2018/ondrej_megela/README.md", + "sha256": "969ff2e1a436b8f957e7168cf5eb678196b99aeda17701c502b016441f644065" + } + ] + }, + { + "id": "q1058", + "category": "single_document_synthesis", + "question": "V dokumente osoby Rostyslav Rodzhuk, ktorý súvisí s témou „agent“, uveď názov záznamu typu „bakalárska práca“ a rok, pri ktorom je táto práca uvedená.", + "expected_answer_contains": [ + "Inteligentný agent pre podporu práce špeciálneho pedagóga", + "2026" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/rostyslav_rodzhuk" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Rostyslav Rodzhuk", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2023/rostyslav_rodzhuk/README.md", + "heading_line": 15, + "title_line": 20, + "title": "Inteligentný agent pre podporu práce špeciálneho pedagóga", + "year": 2026, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2023/rostyslav_rodzhuk/README.md", + "sha256": "de356dd7427c8a0716c165a0f88ae47b691a86ee7a388f60e2d2e9306298415c" + } + ] + }, + { + "id": "q1073", + "category": "single_document_synthesis", + "question": "V dokumente osoby Martin Wencel, ktorý súvisí s témou „pomenované entity“, uveď názov záznamu typu „bakalárska práca“ a rok, pri ktorom je táto práca uvedená.", + "expected_answer_contains": [ + "Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy", + "2020" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2018/martin_wencel" + ], + "should_answer": true, + "expected_bindings": [ + { + "person": "Martin Wencel", + "work_type": "bachelor_thesis", + "document_path": "pages/students/2018/martin_wencel/README.md", + "heading_line": 17, + "title_line": 19, + "title": "Demonštrácia spracovania slovenského prirodzeného jazyka pomocou knižnice Spacy", + "year": 2020, + "year_basis": "heading" + } + ], + "source_checks": [ + { + "document_path": "pages/students/2018/martin_wencel/README.md", + "sha256": "0e6a7a981689f22304f8baa5de97d4315c0695c94c9277de7851943eb012253b" + } + ] + }, + { + "id": "q0065", + "category": "specific_fact", + "question": "Aký dataset používal Lukáš Pokrývka pri experimentoch s detekciou rakoviny pľúc?", + "expected_answer_contains": [ + "LUNA" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2016/lukas_pokryvka/README.md", + "sha256": "f279456c5c277ff1ee0dd89a3464ed9f9f67b44bc185a5da9811ef5cfb675002" + } + ] + }, + { + "id": "q0066", + "category": "specific_fact", + "question": "Aký backend použil Lukáš Pokrývka pri trénovaní na dvoch strojoch?", + "expected_answer_contains": [ + "NCCL" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2016/lukas_pokryvka/README.md", + "sha256": "f279456c5c277ff1ee0dd89a3464ed9f9f67b44bc185a5da9811ef5cfb675002" + } + ] + }, + { + "id": "q0523", + "category": "specific_fact", + "question": "Aké rozhranie má Samuel Vasiľ nasadiť pomocou Docker Compose?", + "expected_answer_contains": [ + "OpenWebUI" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/samuel_vasil" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2022/samuel_vasil/README.md", + "sha256": "67c807049147aacd04f3050f4ff0233a8b3f0a239ab173b430e13dfe113ad0a5" + } + ] + }, + { + "id": "q0408", + "category": "specific_fact", + "question": "Aké príklady oblastí obsahu sú uvedené pri práci Jána Malinovského?", + "expected_answer_contains": [ + "výchovné opatrenia", + "osnovy", + "asistent pedagóga" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [] + }, + { + "id": "q0479", + "category": "specific_fact", + "question": "Pre akú doménu má agent Oleksandra Dorybohova odpovedať na otázky?", + "expected_answer_contains": [ + "právnej" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [] + }, + { + "id": "q0198", + "category": "specific_fact", + "question": "Aké webové rozhranie mala Alina Vitko použiť pre demo aplikáciu?", + "expected_answer_contains": [ + "Streamlit" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2019/alina_vitko/README.md", + "sha256": "9b071fe2aff470c275ae027be236fe836f741f774cabb9cbbb55304a8d5102cd" + } + ] + }, + { + "id": "q0631", + "category": "specific_fact", + "question": "Aké tri prístupy alebo nástroje sú uvedené pri morfologickej analýze slovenského jazyka?", + "expected_answer_contains": [ + "Hunspell", + "Byte Pair Encoding", + "Morfessor" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/topics/morphology" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [] + }, + { + "id": "q0567", + "category": "specific_fact", + "question": "Aký anotačný nástroj si Sevval Bulburu naštudovala?", + "expected_answer_contains": [ + "Prodigy" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/sevval_bulburu" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/interns/sevval_bulburu/README.md", + "sha256": "ff2a9d66253802f1af0fdfd74832e40fae41b6df23c5e61dc266f6cd0cc16c4d" + } + ] + }, + { + "id": "q0614", + "category": "specific_fact", + "question": "Ktorí ľudia sú uvedení pri projekte podpory slovenčiny v Spacy?", + "expected_answer_contains": [ + "Maroš Harahus", + "Kristián Sopkovič" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/topics/spacy" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [] + }, + { + "id": "q0566", + "category": "specific_fact", + "question": "Aký webový framework použila Sevval Bulburu pre aplikáciu?", + "expected_answer_contains": [ + "Django" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/sevval_bulburu" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/interns/sevval_bulburu/README.md", + "sha256": "ff2a9d66253802f1af0fdfd74832e40fae41b6df23c5e61dc266f6cd0cc16c4d" + } + ] + }, + { + "id": "q0421", + "category": "keyword_grounding", + "question": "Ktorý dokument alebo študent súvisí s témou znalostný graf a osobou Maksym Zatirka?", + "expected_answer_contains": [ + "Maksym Zatirka", + "znalostný graf" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/maksym_zatirka" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2023/maksym_zatirka/README.md", + "sha256": "7a7d2cbe276c352dacfbc6ba6de5f15303d30ecf34845ff8da8d085f3d67c3dd" + } + ] + }, + { + "id": "q0476", + "category": "keyword_grounding", + "question": "Ktorý dokument alebo študent súvisí s témou právo a osobou Oleksandr Dorybohov?", + "expected_answer_contains": [ + "Oleksandr Dorybohov", + "právo" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2023/oleksandr_dorybohov/README.md", + "sha256": "c24d6f941de0dafb4459cea8259cf147821cafc86ba65be5b528f419f734bc18" + } + ] + }, + { + "id": "q0544", + "category": "keyword_grounding", + "question": "Ktorá stáž osoby Youssef Ressaissi súvisela s témou Slovak?", + "expected_answer_contains": [ + "Youssef Ressaissi", + "Slovak" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/interns/yussef_ressaissi" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/interns/yussef_ressaissi/README.md", + "sha256": "f997a65699da0db3222ef7a3c37652d05dfc3419ad4ec74683bafc2c897bb987" + } + ] + }, + { + "id": "q0404", + "category": "keyword_grounding", + "question": "Ktorý dokument alebo študent súvisí s témou RAG a osobou Ján Malinovský?", + "expected_answer_contains": [ + "Ján Malinovský", + "RAG" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2022/jan_malinovsky/README.md", + "sha256": "8af021bfb131fa005a7aec3e2f2f12fc17fa5dc14e2675311adbbfc445a4bcab" + } + ] + }, + { + "id": "q0816", + "category": "keyword_grounding", + "question": "Ktorý záznam v ZP Wiki spája osobu Dávid Omasta s témou „generatívne modely“?", + "expected_answer_contains": [ + "Dávid Omasta", + "generatívne modely" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2018/david_omasta" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2018/david_omasta/README.md", + "sha256": "4164051870e1e1cf8949ed18a74196e9ea789dfbe4fee24bdeb736ca42848327" + } + ] + }, + { + "id": "q0195", + "category": "keyword_grounding", + "question": "Ktorý dokument alebo študent súvisí s témou Question Answering a osobou Alina Vitko?", + "expected_answer_contains": [ + "Alina Vitko", + "Question Answering" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/alina_vitko" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2019/alina_vitko/README.md", + "sha256": "9b071fe2aff470c275ae027be236fe836f741f774cabb9cbbb55304a8d5102cd" + } + ] + }, + { + "id": "q0078", + "category": "keyword_grounding", + "question": "Ktorý dokument alebo študent súvisí s témou BiLSTM a osobou Dárius Lindvai?", + "expected_answer_contains": [ + "Dárius Lindvai", + "BiLSTM" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2016/darius_lindvai/README.md", + "sha256": "fcaa47bb9708a6439eec5c4b7f7b4ed6c7359e423e1d940e1aed9f44957e6d62" + } + ] + }, + { + "id": "q0328", + "category": "keyword_grounding", + "question": "Ktorý dokument alebo študent súvisí s témou znalostný graf a osobou Oleh Poiasnik?", + "expected_answer_contains": [ + "Oleh Poiasnik", + "znalostný graf" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/oleh_poiasnik" + ], + "should_answer": true, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2022/oleh_poiasnik/README.md", + "sha256": "c2f7dc3f61f8b4a5b43ded1e3ae4c7d3e7f874439ee6b17dd9c8d45dabc25bc8" + } + ] + }, + { + "id": "q0888", + "category": "unanswerable_missing_fact", + "question": "Aký konkrétny finančný rozpočet bol uvedený pre projekt osoby Tomáš Kuchárik?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2016/tomas_kucharik/README.md", + "sha256": "6a1a215433eb1857660177b4cb2ecec03c6a86743d3a04cd6d5c9d9682c9e63b" + } + ] + }, + { + "id": "q0913", + "category": "unanswerable_missing_fact", + "question": "Aký konkrétny model procesora alebo CPU používal Kristián Sopkovič pri práci?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/kristian_sopkovic" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2019/kristian_sopkovic/README.md", + "sha256": "3286960fb30f584df35455f9b6dfae81ca4827a1d711d2bcd6e48c3f150c9279" + } + ] + }, + { + "id": "q0921", + "category": "unanswerable_missing_fact", + "question": "Koľko strán mala finálna záverečná práca osoby Michal Stromko?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2019/michal_stromko/README.md", + "sha256": "3ae1dd9b6638a6a249a94be3f3846286555666aefa225bb3a2c7288b35900ce7" + } + ] + }, + { + "id": "q0910", + "category": "unanswerable_missing_fact", + "question": "Aká konkrétna IP adresa bola použitá pre nasadený systém osoby Dávid Omasta?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2018/david_omasta" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2018/david_omasta/README.md", + "sha256": "4164051870e1e1cf8949ed18a74196e9ea789dfbe4fee24bdeb736ca42848327" + } + ] + }, + { + "id": "q0908", + "category": "unanswerable_missing_fact", + "question": "Akú výslednú známku z obhajoby získal Dávid Omasta?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2018/david_omasta" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2018/david_omasta/README.md", + "sha256": "4164051870e1e1cf8949ed18a74196e9ea789dfbe4fee24bdeb736ca42848327" + } + ] + }, + { + "id": "q0901", + "category": "unanswerable_missing_fact", + "question": "Aký konkrétny model procesora alebo CPU používal Patrik Pavlišin pri práci?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2016/patrik_pavlisin/README.md", + "sha256": "dc1c648d7cd9b0f0e021c57f9b98189e37c100871439e7e304390715913f2763" + } + ] + }, + { + "id": "q0928", + "category": "unanswerable_missing_fact", + "question": "Aká konkrétna IP adresa bola použitá pre nasadený systém osoby Artem Yatsenko?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/artem_yatsenko" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2019/artem_yatsenko/README.md", + "sha256": "328b101e46c51c959e792cf7ca226942f81d2ab8fe9648ad6d0ecd07016429e2" + } + ] + }, + { + "id": "q0892", + "category": "unanswerable_missing_fact", + "question": "Aká konkrétna IP adresa bola použitá pre nasadený systém osoby Jakub Maruniak?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2016/jakub_maruniak/README.md", + "sha256": "6734c41a95b9d0c1e3d226b393ad109ee4a7b82ebfffa73df2d7e0a743fefa14" + } + ] + }, + { + "id": "q0922", + "category": "unanswerable_missing_fact", + "question": "Aká konkrétna IP adresa bola použitá pre nasadený systém osoby Michal Stromko?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2019/michal_stromko/README.md", + "sha256": "3ae1dd9b6638a6a249a94be3f3846286555666aefa225bb3a2c7288b35900ce7" + } + ] + }, + { + "id": "q0929", + "category": "unanswerable_missing_fact", + "question": "Akú presnú veľkosť v GB mala hlavná dátová množina použitá osobou Artem Yatsenko?", + "expected_answer_contains": [], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2019/artem_yatsenko" + ], + "should_answer": false, + "expected_bindings": [], + "source_checks": [ + { + "document_path": "pages/students/2019/artem_yatsenko/README.md", + "sha256": "328b101e46c51c959e792cf7ca226942f81d2ab8fe9648ad6d0ecd07016429e2" + } + ] + } + ], + "limitations": [ + "Known regression questions; not a new independent accuracy estimate.", + "Dárius Lindvai: the diploma year 2021 is inferred from taxonomy category dp2021; no explicit diploma heading. Runtime checked RAG deliberately does not use category years as work-year proof.", + "Source review is automated against original Markdown; independent human annotation has not been performed." + ] +} diff --git a/tools/rag/evaluation/results/verified_rag_2026-10-27/summary.json b/tools/rag/evaluation/results/verified_rag_2026-10-27/summary.json new file mode 100644 index 0000000..7d22a61 --- /dev/null +++ b/tools/rag/evaluation/results/verified_rag_2026-10-27/summary.json @@ -0,0 +1,447 @@ +{ + "date": "2026-10-27", + "scope": "Classic checked RAG regression and operational report organization. GraphRAG baseline/results unchanged.", + "status": { + "local_wider_verification": "complete", + "reports_and_logs": "complete", + "wider_80_model_answers": "awaiting_explicit_consent" + }, + "dataset": { + "path": "tools/rag/evaluation/json_files/verified_rag_regression.json", + "sha256": "07d6dcaaf546c1df5a2a6c8074ad7502122175d4226a194cd782f5b5031a0b07" + }, + "method": "80 predeclared known regression questions from nine strata, including 10 missing-fact negatives. No gold sent to production model. Not an independent TEST.", + "retrieval_and_typed_evidence": { + "total": 80, + "answers_evaluated": 0, + "passed": 0, + "success_percent": null, + "false_no_answers": 0, + "blocked_deliveries": 0, + "first_attempt_accepted": 0, + "failure_reasons": {}, + "source_coverage": 80, + "answerable_source_coverage": { + "passed": 70, + "total": 70 + }, + "typed_binding_coverage": { + "passed": 50, + "total": 52 + }, + "categories": { + "disambiguation_year": { + "total": 8, + "passed": 0 + }, + "fact_start_year": { + "total": 8, + "passed": 0 + }, + "fact_title": { + "total": 10, + "passed": 0 + }, + "fact_work_year": { + "total": 10, + "passed": 0 + }, + "keyword_grounding": { + "total": 8, + "passed": 0 + }, + "multi_document": { + "total": 8, + "passed": 0 + }, + "single_document_synthesis": { + "total": 8, + "passed": 0 + }, + "specific_fact": { + "total": 10, + "passed": 0 + }, + "unanswerable_missing_fact": { + "total": 10, + "passed": 0 + } + } + }, + "unresolved_source_conventions": [ + { + "id": "q0074", + "question": "V ktorom roku je pri osobe Dárius Lindvai uvedená diplomová práca?", + "missing_bindings": [ + { + "person": "Dárius Lindvai", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/darius_lindvai/README.md", + "heading_line": null, + "title_line": 14, + "year": 2021, + "year_basis": "frontmatter_category" + } + ] + }, + { + "id": "q1168", + "question": "Porovnaj záznamy osôb Dárius Lindvai a Tetiana Mohorian. Pri osobe Dárius Lindvai použi záznam typu „diplomová práca“ a pri osobe Tetiana Mohorian záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "missing_bindings": [ + { + "person": "Dárius Lindvai", + "work_type": "diploma_thesis", + "document_path": "pages/students/2016/darius_lindvai/README.md", + "heading_line": null, + "title_line": 14, + "title": "Obnovenie interpunkcie pomocou hlbokých neurónových sietí", + "year": 2021, + "year_basis": "frontmatter_category" + } + ] + } + ], + "source_note": "Darius Lindvai diploma year 2021 comes from frontmatter category dp2021 without an explicit diploma heading. Runtime deliberately does not infer academic year from taxonomy.", + "guard_replay": { + "summary": { + "checks": 360, + "passed": 360, + "contexts": 80, + "llm_calls": 0, + "note": "Controlled fault injection, not an LLM accuracy measurement." + }, + "by_check": { + "accept_supported_tuple": { + "passed": 50, + "total": 50 + }, + "reject_false_refusal": { + "passed": 50, + "total": 50 + }, + "reject_wrong_year": { + "passed": 50, + "total": 50 + }, + "reject_wrong_title": { + "passed": 50, + "total": 50 + }, + "reject_wrong_person": { + "passed": 50, + "total": 50 + }, + "reject_wrong_work_type": { + "passed": 50, + "total": 50 + }, + "reject_wrong_citation": { + "passed": 50, + "total": 50 + }, + "accept_supported_missing_fact_refusal": { + "passed": 10, + "total": 10 + } + } + }, + "authorized_five_model_smoke": { + "summary": { + "total": 5, + "answers_evaluated": 5, + "passed": 5, + "success_percent": 100.0, + "false_no_answers": 0, + "blocked_deliveries": 0, + "first_attempt_accepted": 5, + "failure_reasons": {}, + "source_coverage": 5, + "answerable_source_coverage": { + "passed": 4, + "total": 4 + }, + "typed_binding_coverage": { + "passed": 3, + "total": 3 + }, + "categories": { + "disambiguation_year": { + "total": 1, + "passed": 1 + }, + "fact_title": { + "total": 1, + "passed": 1 + }, + "multi_document": { + "total": 1, + "passed": 1 + }, + "source_url": { + "total": 1, + "passed": 1 + }, + "unanswerable_missing_fact": { + "total": 1, + "passed": 1 + } + } + }, + "model": "model120-fast", + "provider": "https://ui.tukekemt.xyz", + "method": "Five actual /answer requests with at most one repair, using previously authorized questions. Same stored responses regraded after correcting evaluator false positives on additional supported academic works. No new generation during regrading.", + "results": [ + { + "id": "q0053", + "question": "Aký je názov diplomová práca osoby Lukáš Pokrývka?", + "passed": true, + "issues": [], + "answer": "Lukáš Pokrývka — diplomová práca; názov práce: Paralelné trénovanie neurónových sietí; rok práce: 2021.\nVýpis je neúplný.\nZdroje: https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka", + "claims": [ + { + "person": "Lukáš Pokrývka", + "title": "Paralelné trénovanie neurónových sietí", + "year": 2021, + "work_type": "diploma_thesis" + } + ], + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka" + ], + "accepted_for_delivery": true + }, + { + "id": "q0060", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Lukáš Pokrývka.", + "passed": true, + "issues": [], + "answer": "Lukáš Pokrývka — diplomová práca; rok práce: 2021; začiatok štúdia: 2016.\nLukáš Pokrývka — tímový projekt; rok práce: 2019; začiatok štúdia: 2016.\nLukáš Pokrývka — diplomový projekt; rok práce: 2020; začiatok štúdia: 2016.\nVýpis je neúplný.\nZdroje: https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka", + "claims": [ + { + "person": "Lukáš Pokrývka", + "year": 2021, + "start_year": 2016, + "work_type": "diploma_thesis" + }, + { + "person": "Lukáš Pokrývka", + "year": 2019, + "start_year": 2016, + "work_type": "team_project" + }, + { + "person": "Lukáš Pokrývka", + "year": 2020, + "start_year": 2016, + "work_type": "diploma_project" + } + ], + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka" + ], + "accepted_for_delivery": true + }, + { + "id": "q0608", + "question": "Aký source_url patrí k stránke „Podpora slovenčiny v knižnici Spacy“?", + "passed": true, + "issues": [], + "answer": "Zo zdroja Podpora slovenčiny v knižnici Spacy: „Source URL: https://zp.kemt.fei.tuke.sk/topics/spacy“\nVýpis je neúplný.\nZdroje: https://zp.kemt.fei.tuke.sk/topics/spacy", + "claims": [], + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/topics/spacy" + ], + "accepted_for_delivery": true + }, + { + "id": "q0888", + "question": "Aký konkrétny finančný rozpočet bol uvedený pre projekt osoby Tomáš Kuchárik?", + "passed": true, + "issues": [], + "answer": "V dostupných podkladoch sa požadovanú informáciu nepodarilo spoľahlivo nájsť.\nVýpis je neúplný.", + "claims": [], + "source_urls": [], + "accepted_for_delivery": true + }, + { + "id": "q1147", + "question": "Porovnaj záznamy osôb Ján Holp a Ondrej Megela. Pri osobe Ján Holp použi záznam typu „diplomová práca“ a pri osobe Ondrej Megela záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.", + "passed": true, + "issues": [], + "answer": "Ján Holp — diplomová práca; názov práce: Systém získavania informácií v slovenskom jazyku; rok práce: 2021.\nOndrej Megela — bakalárska práca; názov práce: Neurónové jazykové modelovanie typu BERT; rok práce: 2020.\nVýpis je neúplný.\nZdroje: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp, https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela", + "claims": [ + { + "person": "Ján Holp", + "title": "Systém získavania informácií v slovenskom jazyku", + "year": 2021, + "work_type": "diploma_thesis" + }, + { + "person": "Ondrej Megela", + "title": "Neurónové jazykové modelovanie typu BERT", + "year": 2020, + "work_type": "bachelor_thesis" + } + ], + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela" + ], + "accepted_for_delivery": true + } + ] + }, + "broad_model_accuracy": null, + "full_verification": { + "passed": true, + "checks": { + "python_and_shell": { + "passed": true, + "details": { + "compiled_python_files": 150, + "checked_shell_scripts": 4 + }, + "seconds": 0.5 + }, + "git_diff": { + "passed": true, + "details": "", + "seconds": 0.01 + }, + "git_integrity": { + "passed": true, + "details": "dangling tree 7a73ecae52355619e782cb822622ac3f69de7ae0\ndangling blob 5574a6fec5472a6e8f5e743a49da177b376eda10\ndangling tree fbb4fbc9fe2f927d118b4b3b97e886a316d1d8f1\ndangling blob a186d97f5ef3b6da4a0ef094f7d05c11b47d9350\ndangling tree a626128497ed8c0d3dc200285901748474ac7dd2\ndangling tree 3337aecd889bcdac7142dcb00e46f1c71aa0cf92\ndangling blob c0c87b808a2454b64badf9d191260458aa576653\ndangling tree ef38b2cbe6f6e24898cefe8a83450122dea5d4c6\ndangling blob f208469c41d2b62812c34bc4ace4f25fdbd627cf\ndangling blob 6489d7ce0bb130e424bbe8da6c73300a2e167b94\ndangling blob c949aad31632b94da867885110890f0a5c9ca2a5\ndangling tree 625a23eefd539c3844b515e71d0ac6929b7babe0\ndangling tree b4eaffde38b667198523282947dae5e4a4e4fb03\ndangling blob f54b96ee587ee0d697902604536f3803e6001ad2\ndangling blob c54c2eb64f9a1d522f8c7e335222fdac15b1d02b\ndangling blob c14d08a27284af6a3b91414aaf9f5a1773ed9240", + "seconds": 0.05 + }, + "compose_configuration": { + "passed": true, + "details": "", + "seconds": 0.11 + }, + "api_and_both_indexes": { + "passed": true, + "details": { + "ready": true, + "index_version": { + "version": "497fadb8f38c4af18d7d9e3b06cdb0a0", + "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e", + "created_at_utc": "2026-10-26T18:15:17.983336+00:00", + "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363", + "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52", + "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9", + "embedding_model": "intfloat/multilingual-e5-small", + "publication_policy": "explicit-false-hidden-v1", + "embedding_fingerprint": "d066fb9a266a1a0c77c65da75dcf538cfab5c87d5c605ff26b1fd2da8a12aa92" + }, + "tools": { + "rag": true, + "neo4j": true, + "graphrag": true + } + }, + "seconds": 7.91 + }, + "sqlite_integrity": { + "passed": true, + "details": { + "documents": 114, + "chunks": 887, + "chunks_fts": 887, + "chunk_embeddings": 892 + }, + "seconds": 0.06 + }, + "container_readiness": { + "passed": true, + "details": "", + "seconds": 1.2 + }, + "wiki_git_ssh": { + "passed": true, + "details": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e\trefs/heads/master", + "seconds": 0.83 + }, + "openwebui_tool_packages": { + "passed": true, + "details": { + "packages": [ + { + "package_id": "zpwiki_rag", + "archive": "/home/janko/DP/zp-agent/data/tool_packages/zpwiki_rag.zip", + "sha256": "84c66cefbed33dbbabbe033f5fe5dc5e71cda380c9824b2ae7f15b30e304fdd6" + }, + { + "package_id": "zpwiki_graphrag", + "archive": "/home/janko/DP/zp-agent/data/tool_packages/zpwiki_graphrag.zip", + "sha256": "6d6ab36bf8fe936a1ae9a1c1debb46cedb04116c860ee9d3216ca720e65cf2fe" + } + ], + "live_retrieval": true, + "same_data_version": true, + "negative_query": true, + "authorization": true, + "external_llm_calls": false + }, + "seconds": 0.75 + }, + "full_local_regression": { + "passed": true, + "details": { + "passed": 734, + "log": "/home/janko/DP/zp-agent/data/logs/verification/20261027T005852Z-ef38a940.pytest.log", + "external_network_blocked": true + }, + "seconds": 89.48 + } + }, + "deployed_python_modules_match_workspace": 125, + "offline_assets_build_probe": true, + "post_regression_both_tools_ready": true, + "index_version_unchanged": "497fadb8f38c4af18d7d9e3b06cdb0a0" + }, + "report_organization": { + "moved_files": 26, + "deleted_files": 0, + "operational_json_hashes_unchanged": true, + "reports": "data/reports//", + "logs": "data/logs/", + "legacy_reports": "data/reports/archive/legacy/", + "legacy_logs": "data/logs/archive/legacy/", + "retention": "Keep five latest completed project verifications active, archive older reports and paired logs, preserve incomplete and experimental results. Never auto-delete.", + "journal": { + "path": "data/reports/migrations/20261027T004801Z-dfb5c510.json", + "sha256": "e6ff162d8d8164bdac07cb3c1781fc9d113990079d9d636203a8ed1ab48fce40" + } + }, + "artifacts": { + "local_contexts": { + "path": "data/reports/rag_verified/20261027T005720Z-3186724c.json", + "sha256": "516276ce1f91f5b3e30c8d09d0a008021c777f067d74ef8fbd89faeba259af9d" + }, + "guard_replay": { + "path": "data/reports/rag_guards/20261027T005846Z-cfbc62a7.json", + "sha256": "d2539d13a205af724a2aa95f7d11bc72d9a2e1f4176147eddcea78a6ab173e42" + }, + "original_model_outputs": { + "path": "data/reports/rag_verified/20261027T005425Z-2e5ca17c.json", + "sha256": "e98bd32ccc9f5010fb09b674499cb7decf0417781f3b1d92d7da077050eca9e8" + }, + "regraded_model_outputs": { + "path": "data/reports/rag_verified/20261027T005639Z-bb118881.json", + "sha256": "7383b72083e965c670cda2caadae268894eb8caead4185f48abde26c477e21c5" + }, + "project_verification": { + "path": "data/reports/verification/20261027T005852Z-ef38a940.json", + "sha256": "eac6ef0caace933d42fd41c540a10c646699843aec6bc2bab28ef8be4727aa8f" + } + }, + "limitations": [ + "Known regression questions; not a new independent accuracy estimate.", + "Dárius Lindvai: the diploma year 2021 is inferred from taxonomy category dp2021; no explicit diploma heading. Runtime checked RAG deliberately does not use category years as work-year proof.", + "Source review is automated against original Markdown; independent human annotation has not been performed.", + "No 80-case model accuracy has been measured. Existing explicit permission covers five school-model questions; broader export approval is pending.", + "360 guard checks are controlled fault injection, not 360 generated model responses.", + "Finding the expected document or extracting a supported tuple does not by itself measure answer correctness.", + "Automatic quote relevance uses known expected-value coverage and literal support; independent human semantic review is still absent." + ] +} diff --git a/tools/runtime/report_files.py b/tools/runtime/report_files.py new file mode 100644 index 0000000..8dae1e1 --- /dev/null +++ b/tools/runtime/report_files.py @@ -0,0 +1,124 @@ +"""Locations and non-destructive retention for local operational reports.""" +from __future__ import annotations + +import hashlib +import json +import shutil +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +from tools.rag.common import DATA_DIR + +CORE_JSON = {'documents.json', 'chunks.json', 'index_version.json'} + + +def run_id() -> str: + return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid4().hex[:8] + + +def report_output(requested: Path | None, category: str, *, data_dir: Path = DATA_DIR) -> Path: + if not category.replace('_', '').isalnum(): + raise ValueError('Invalid report category') + if requested is None: + target = data_dir / 'reports' / category / (run_id() + '.json') + else: + target = requested.resolve() + if target.parent == data_dir.resolve(): + if target.name in CORE_JSON or target.name.startswith('.') or target.suffix != '.json': + raise ValueError('Report cannot overwrite operational data') + target = data_dir / 'reports' / category / target.name + if target.exists(): + raise FileExistsError(f'Report already exists: {target}') + target.parent.mkdir(parents=True, exist_ok=True) + return target + + +def report_log(report: Path, *, data_dir: Path = DATA_DIR) -> Path: + try: + relative = report.resolve().relative_to((data_dir / 'reports').resolve()) + except ValueError: + identity = hashlib.sha256(str(report.absolute()).encode()).hexdigest()[:8] + relative = Path('custom') / (report.stem + '-' + identity + '.json') + target = data_dir / 'logs' / relative.with_suffix('.pytest.log') + if target.exists(): + raise FileExistsError(f'Log already exists: {target}') + target.parent.mkdir(parents=True, exist_ok=True) + return target + + +def resolve_report_input(path: Path, *, data_dir: Path = DATA_DIR) -> Path: + if path.exists(): + return path + mappings = {} + for record in sorted((data_dir / 'reports/migrations').glob('*.json')): + for move in json.loads(record.read_text()).get('moves', []): + mappings[Path(move['source']).resolve()] = Path(move['destination']) + visited = set() + while not path.exists() and path.resolve() in mappings and path.resolve() not in visited: + visited.add(path.resolve()) + path = mappings[path.resolve()] + return path + + +def migration_plan(data_dir: Path = DATA_DIR) -> list[dict]: + moves = [] + for path in sorted(data_dir.iterdir()): + if not path.is_file() or path.name.startswith('.') or path.name in CORE_JSON: + continue + if path.suffix == '.json': + target = data_dir / 'reports/archive/legacy' / path.name + elif path.suffix == '.log': + target = data_dir / 'logs/archive/legacy' / path.name + else: + continue + moves.append({'source': str(path.absolute()), 'destination': str(target.absolute()), + 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}) + return moves + + +def retention_plan(data_dir: Path = DATA_DIR, *, keep: int = 5) -> list[dict]: + if keep < 1: + raise ValueError('Keep at least one verification') + completed = [] + for path in (data_dir / 'reports/verification').glob('*.json'): + try: + data = json.loads(path.read_text()) + if isinstance(data, dict) and type(data.get('passed')) is bool: + completed.append(path) + except (ValueError, OSError): + continue + moves = [] + for path in sorted(completed, key=lambda p: (p.stat().st_mtime_ns, p.name), reverse=True)[keep:]: + target = data_dir / 'reports/archive/verification' / path.name + moves.append({'source': str(path.absolute()), 'destination': str(target.absolute()), + 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}) + log = data_dir / 'logs/verification' / path.with_suffix('.pytest.log').name + if log.exists(): + target = data_dir / 'logs/archive/verification' / log.name + moves.append({'source': str(log.absolute()), 'destination': str(target.absolute()), + 'sha256': hashlib.sha256(log.read_bytes()).hexdigest()}) + return moves + + +def apply_moves(moves: list[dict], *, data_dir: Path = DATA_DIR) -> Path: + # Validate the entire plan before moving anything. Never replace existing files. + for move in moves: + source, destination = Path(move['source']), Path(move['destination']) + if destination.exists() or hashlib.sha256(source.read_bytes()).hexdigest() != move['sha256']: + raise RuntimeError('Report migration conflict; no files moved') + journal = data_dir / 'reports/migrations' / (run_id() + '.json') + journal.parent.mkdir(parents=True, exist_ok=True) + record = {'method': 'Move without rewriting content or deleting historical evidence', 'moves': []} + def write_journal(): + temporary = journal.with_suffix('.tmp') + temporary.write_text(json.dumps(record, indent=2) + '\n') + temporary.replace(journal) + write_journal() + for move in moves: + destination = Path(move['destination']) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.move(move['source'], destination) + record['moves'].append(move) + write_journal() + return journal