From 66a00686ed46d2ed87290f142285a6593b3fa85f Mon Sep 17 00:00:00 2001 From: jp170na Date: Mon, 2 Nov 2026 16:44:29 +0100 Subject: [PATCH] =?UTF-8?q?=C3=9Apln=C3=A9=20v=C3=BDpisy=20pr=C3=A1c?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 10 +- test/test_graph_work_listing.py | 222 ++++++++++++++++++ tools/answer_evaluation/answer_format.py | 3 + tools/answer_evaluation/context_policy.py | 30 ++- .../answer_evaluation/graph_answer_prompt.txt | 3 +- tools/answer_evaluation/graph_context.py | 17 +- tools/answer_evaluation/render_answer.py | 12 +- tools/graphrag/evaluation/reproducibility.py | 3 +- tools/graphrag/graph_search.py | 24 +- 9 files changed, 306 insertions(+), 18 deletions(-) create mode 100644 test/test_graph_work_listing.py diff --git a/README.md b/README.md index f332235..fdbd0da 100644 --- a/README.md +++ b/README.md @@ -984,7 +984,7 @@ grafové vyhľadávanie je implementované v `tools/graphrag/graph_search.py`. GraphRAG zatiaľ nezasahuje do stabilného klasického `/rag` toku; ďalšia etapa spojí grafové výsledky s klasickým RAG a OpenWebUI nástrojom. -Aktuálna evaluácia GraphRAG **v7** má DEV **1 046/1 046** a nový TEST +Zmrazená evaluácia GraphRAG **v7** má DEV **1 046/1 046** a nový TEST **80/80** pre grafové vyhľadávanie aj finálne odpovede. Modelový DEV má **46/46**; pri novom TESTe prešlo na prvý pokus **77/80 (96,25 %)**, po jednej povolenej oprave **80/80 (100 %)**. Prirodzený text sa skladá @@ -1007,6 +1007,14 @@ v1–v4 zostávajú zachované. Aktuálne skóre nie je zárukou všeobecnej pra V7 benchmark nie je nezávisle anotovaný človekom. Nové ladenie na známych otázkach sa označuje ako regresia, nie ako nový nezávislý TEST. +Otázky typu „Aké práce má Ján Holp a v ktorých rokoch?“ vracajú jednotlivé +práce aj projekty s názvom, typom a rokom. Záznamy bez samostatného názvu +sa zachovajú a odpoveď uvedie, že názov v zdroji chýba. Dva projekty z rovnakého +roku sa nezlučujú. Kontrola pred doručením vyžaduje pokrytie všetkých dodaných +záznamov vrátane ich vzájomných väzieb a zdrojov; samotný zoznam rokov nestačí. +Otázka pýtajúca sa iba na roky naďalej vracia iba roky. Táto neskoršia oprava +nemení historické výsledky ani manifest v7; ich skóre sa neprenáša na novú verziu. + ## Claude Skills a webový MCP konektor Táto vetva `claude-tools` zachováva spoločný RAG/GraphRAG backend, diff --git a/test/test_graph_work_listing.py b/test/test_graph_work_listing.py new file mode 100644 index 0000000..627db76 --- /dev/null +++ b/test/test_graph_work_listing.py @@ -0,0 +1,222 @@ +"""Regression for work listings losing titles/types and merging same-year work.""" +import copy +import json +from pathlib import Path + +import pytest + +from tools.answer_evaluation.answer_format import parse_answer +from tools.answer_evaluation.checked_generation import generate_checked_answer +from tools.answer_evaluation.context_policy import answer_context_issues +from tools.answer_evaluation.graph_context import graph_context +from tools.answer_evaluation.render_answer import render_graph_answer +from tools.graphrag import graph_search as gs +from tools.graphrag.tests.test_graph_search import FakeSession, sample_catalog + + +QUESTION = 'Aké práce má Ján Holp a v ktorých rokoch?' +URL = 'https://example.test/jan' + + +@pytest.mark.parametrize('question', [ + QUESTION, 'Aké práce má Ján Holp?', 'Ktoré práce má Ján Holp?', + 'Vypíš práce Jána Holpa.', 'Uveď všetky práce Jána Holpa.', + 'Zoznam prác Jána Holpa', 'Prehľad projektov Jána Holpa', + 'Aký diplomový projekt mal Ján Holp v roku 2020?', + 'Aké má Ján Holp práce a v ktorých rokoch?', +]) +def test_work_listing_intent_without_explicit_title_word(question): + assert gs.extract_query_constraints(question)['mentions_work_listing'] + + +@pytest.mark.parametrize('question', [ + 'V ktorých rokoch má Ján Holp práce?', 'Aký je rok práce Jána Holpa?', + 'Aký je názov diplomovej práce Jána Holpa?', + 'Koľko prác má Ján Holp?', 'Ktoré osoby majú práce z roku 2020?', + 'Aké kľúčové slová má Ján Holp?', +]) +def test_other_projections_are_not_changed_into_work_listings(question): + assert not gs.extract_query_constraints(question)['mentions_work_listing'] + + +@pytest.fixture +def retrieved(): + common = {'person': 'Ján Holp', 'document_path': 'p.md', 'source_url': URL} + theses = [ + {'thesis_id': 'thesis:bp', 'thesis_type': 'bachelor_thesis', 'year': 2019, + 'heading': 'Bakalárska práca 2019', 'title': 'Vyhľadávač na slovenskom internete'}, + {'thesis_id': 'thesis:dp', 'thesis_type': 'diploma_thesis', 'year': 2021, + 'heading': 'Diplomová práca 2021', 'title': 'Systém získavania informácií v slovenskom jazyku.'}] + projects = [ + {'work_id': 'work:tp', 'work_type': 'team_project', 'year': 2019, + 'heading': 'Tímový projekt 2019', 'title': None}, + {'work_id': 'work:dp', 'work_type': 'diploma_project', 'year': 2020, + 'heading': 'Diplomový projekt 2020', 'title': None}, + {'work_id': 'work:dp2', 'work_type': 'diploma_project', 'year': 2020, + 'heading': 'Diplomový projekt 2 2020', 'title': None}] + return {'status': 'ok', 'reason': None, 'complete': True, 'truncated': False, + 'truncation_reasons': [], 'index_version': {'version': 'fixture'}, 'aggregation': None, + 'constraints': gs.extract_query_constraints(QUESTION), + 'sources': [{'document_path': 'p.md', 'source_url': URL}], + 'facts': [{**common, **row, 'fact_type': 'person_thesis'} for row in theses] + + [{**common, **row, 'fact_type': 'person_work'} for row in projects]} + + +def listing_answer(context): + keys = ['person', 'record_id', 'title', 'year', 'work_type', 'work_label'] + return {'status': 'ok', 'answer': '', 'source_urls': [URL], 'count': None, + 'claims': [{key: row[key] for key in keys if row.get(key) is not None} + for row in context['required_academic_records']]} + + +def completion(answer): + return {'choices': [{'message': {'content': json.dumps(answer, ensure_ascii=False)}}]} + + +def test_search_without_year_or_title_fetches_both_academic_kinds(): + fact = {'person': 'Jakub Maruniak', 'person_element_id': 'p2', 'work_id': 'work:dp2', + 'work_element_id': 'w2', 'heading': 'Diplomový projekt 2 2020', 'year': 2020, + 'work_type': 'diploma_project', 'title': None} + session = FakeSession({gs.CATALOG_QUERY: sample_catalog(), gs.WORK_FACTS_QUERY: [fact]}) + result = gs.search_graph_with_session(session, 'Aké práce má Jakub Maruniak?') + calls = {query for query, _ in session.calls} + assert {gs.THESIS_FACTS_QUERY, gs.WORK_FACTS_QUERY} <= calls + assert result['facts'][0]['work_id'] == 'work:dp2' + assert result['facts'][0]['title'] is None + + +@pytest.mark.parametrize('question,expected,excluded', [ + ('Aké projekty má Jakub Maruniak?', gs.WORK_FACTS_QUERY, gs.THESIS_FACTS_QUERY), + ('Aké záverečné práce má Jakub Maruniak?', gs.THESIS_FACTS_QUERY, gs.WORK_FACTS_QUERY), +]) +def test_listing_preserves_requested_domain(question, expected, excluded): + session = FakeSession({gs.CATALOG_QUERY: sample_catalog()}) + gs.search_graph_with_session(session, question) + calls = {query for query, _ in session.calls} + assert expected in calls and excluded not in calls + + +def test_context_preserves_five_records_instead_of_three_years(retrieved): + context = graph_context(retrieved) + assert len(context['required_academic_records']) == 5 + assert len(context['facts']) == 5 + assert {r['record_id'] for r in context['facts']} == {'thesis:bp', 'thesis:dp', 'work:tp', 'work:dp', 'work:dp2'} + assert [r['title'] for r in context['facts']].count(None) == 3 + assert context['complete'] is True + + +@pytest.mark.parametrize('change', ['years_only', 'no_types', 'missing_project', 'no_record_ids']) +def test_guard_rejects_incomplete_response_even_when_retrieval_is_complete(retrieved, change): + context = graph_context(retrieved) + answer = listing_answer(context) + if change == 'years_only': + answer['claims'] = [{'person': 'Ján Holp', 'year': year} for year in (2019, 2020, 2021)] + elif change == 'no_types': + for claim in answer['claims']: claim.pop('work_type') + elif change == 'missing_project': + answer['claims'].pop() + else: + for claim in answer['claims']: claim.pop('record_id') + assert 'missing_academic_record_details' in answer_context_issues(answer, context, check_text=False) + + +@pytest.mark.parametrize('field,value', [('year', 2020), ('work_type', 'team_project'), + ('record_id', 'work:dp2'), ('record_id', 'thesis/dp'), + ('work_label', 'Diplomový projekt 2020')]) +def test_record_fields_cannot_be_mixed_between_works(retrieved, field, value): + context = graph_context(retrieved) + answer = listing_answer(context) + answer['claims'][1][field] = value + assert 'unsupported_academic_binding' in answer_context_issues(answer, context, check_text=False) + + +def test_citation_must_support_the_whole_record_not_separate_field_occurrences(retrieved): + context = graph_context(retrieved) + claim = {'person': 'Ján Holp', 'title': 'A', 'year': 2021} + context.pop('required_academic_records') + other = 'https://example.test/other' + context['sources'].append({'source_url': other}) + context['facts'] = [ + {'person': 'Ján Holp', 'title': 'A', 'year': 2021, 'source_url': URL}, + {'person': 'Ján Holp', 'title': 'A', 'year': 2020, 'source_url': other}, + {'person': 'Ján Holp', 'title': 'B', 'year': 2021, 'source_url': other}] + answer = {'status': 'ok', 'answer': '', 'claims': [claim], 'source_urls': [other]} + assert 'missing_academic_record_source' in answer_context_issues(answer, context, check_text=False) + answer['source_urls'].append(URL) + assert answer_context_issues(answer, context, check_text=False) == [] + + +def test_missing_title_is_reported_without_invention_and_ids_stay_private(retrieved): + context = graph_context(retrieved) + answer, _ = parse_answer(json.dumps(listing_answer(context))) + assert answer_context_issues(answer, context, check_text=False) == [] + rendered = render_graph_answer(answer, context) + assert answer_context_issues(rendered, context) == [] + assert rendered['answer'].count('názov v zdroji nie je uvedený') == 3 + assert 'Diplomový projekt 2 2020' in rendered['answer'] + assert 'Systém získavania informácií' in rendered['answer'] + assert 'typ práce: Bakalárska práca' in rendered['answer'] + assert all(claim['record_id'] not in rendered['answer'] for claim in answer['claims']) + answer['claims'][2]['title'] = 'Vymyslený názov projektu' + assert 'unsupported_academic_binding' in answer_context_issues(answer, context, check_text=False) + + +def test_unknown_year_is_not_filled_from_heading_or_another_record(retrieved): + retrieved['facts'][2]['year'] = None + retrieved['facts'][2]['heading'] = 'Tímový projekt' + context = graph_context(retrieved) + answer = listing_answer(context) + rendered = render_graph_answer(answer, context) + assert 'rok v zdroji nie je uvedený' in rendered['answer'] + assert answer_context_issues(rendered, context) == [] + answer['claims'][2]['year'] = 2019 + assert 'unsupported_academic_binding' in answer_context_issues(answer, context, check_text=False) + + +def test_known_record_cannot_be_refused(retrieved): + context = graph_context(retrieved) + answer = {'status': 'no_answer', 'answer': '', 'claims': [], 'source_urls': []} + assert 'rejected_supported_academic_listing' in answer_context_issues(answer, context, check_text=False) + + +def test_listing_does_not_require_records_outside_truncated_context(retrieved): + retrieved.update(complete=False, truncated=True, truncation_reasons=['result_limit']) + retrieved['facts'] = retrieved['facts'][:2] + context = graph_context(retrieved) + rendered = render_graph_answer(listing_answer(context), context) + assert answer_context_issues(rendered, context) == [] + assert 'Výpis je neúplný.' in rendered['answer'] + + +def test_year_only_question_still_returns_only_years(retrieved): + retrieved['constraints'] = gs.extract_query_constraints('V ktorých rokoch má Ján Holp práce?') + context = graph_context(retrieved) + assert 'required_academic_records' not in context + assert len(context['facts']) == 3 + assert all(set(row) == {'person', 'year', 'document_path', 'source_url'} for row in context['facts']) + + +def test_production_delivery_repairs_year_only_answer_and_renders_all_records(monkeypatch, retrieved): + from tools.answers import answer_delivery as delivery + monkeypatch.setattr(delivery, 'search_graph', lambda query: retrieved) + good = listing_answer(graph_context(retrieved)) + bad = {**good, 'claims': [{'person': 'Ján Holp', 'year': year} for year in (2019, 2020, 2021)]} + replies = iter([bad, good]); messages = [] + def complete(rows): + messages.append(copy.deepcopy(rows)) + return completion(next(replies)) + result = delivery.deliver_answer(QUESTION, engine='graphrag', complete=complete) + assert result['accepted_for_delivery'] + assert len(result['claims']) == 5 and len(messages) == 2 + assert 'missing_academic_record_details' in result['attempts'][0]['context_issues'] + assert 'Diplomový projekt 2 2020' in result['answer'] + assert 'expected_bindings' not in json.dumps(messages) + + +def test_two_year_only_attempts_are_never_delivered(retrieved): + context = graph_context(retrieved) + answer = {'status': 'ok', 'answer': '', 'source_urls': [URL], + 'claims': [{'person': 'Ján Holp', 'year': 2021}]} + prompt = Path('tools/answer_evaluation/graph_answer_prompt.txt').read_text() + result = generate_checked_answer(QUESTION, context, prompt, lambda _: completion(answer)) + assert not result['accepted_for_delivery'] and len(result['attempts']) == 2 diff --git a/tools/answer_evaluation/answer_format.py b/tools/answer_evaluation/answer_format.py index 17071e9..f671ba8 100644 --- a/tools/answer_evaluation/answer_format.py +++ b/tools/answer_evaluation/answer_format.py @@ -14,6 +14,9 @@ class Claim(BaseModel): keyword: str | None = None category: str | None = None document_author: str | None = None + record_id: str | None = None + work_type: str | None = None + work_label: str | None = None class Answer(BaseModel): diff --git a/tools/answer_evaluation/context_policy.py b/tools/answer_evaluation/context_policy.py index 71a55ee..39bc7bd 100644 --- a/tools/answer_evaluation/context_policy.py +++ b/tools/answer_evaluation/context_policy.py @@ -131,6 +131,11 @@ def text_contains_value(text: str, value: str, *, person: bool = False) -> bool: return bool(re.search(pattern, text)) +def _same_claim_value(key: str, actual, expected) -> bool: + # Opaque identifiers must not collapse paths/punctuation as prose does. + return actual == expected if key == 'record_id' else normalized(actual) == normalized(expected) + + def answer_context_issues(answer: dict, context: dict, *, check_text: bool = True) -> list[str]: issues=[] if not citation_check(answer,allowed_urls(context))['citations_ok']: @@ -160,20 +165,37 @@ def answer_context_issues(answer: dict, context: dict, *, check_text: bool = Tru for key in ['keyword','category','document_author','start_year']: if claim.get(key) is not None and not any(normalized(row.get(key))==normalized(claim[key]) for row in rows): issues.append('unsupported_'+key) - for key in ['title', 'year', 'start_year', 'keyword', 'category', 'document_author']: + for key in ['title', 'year', 'start_year', 'keyword', 'category', 'document_author', + 'record_id', 'work_type', 'work_label']: if claim.get(key) is None: continue evidence = {row.get('source_url') for row in rows - if normalized(row.get(key)) == normalized(claim[key]) and row.get('source_url')} + if _same_claim_value(key, row.get(key), claim[key]) and row.get('source_url')} if evidence and not evidence & cited: issues.append('missing_claim_source') - academic={k:claim[k] for k in ['title','year'] if claim.get(k) is not None} - if academic and not any(all(normalized(row.get(k))==normalized(v) for k,v in academic.items()) for row in rows): + academic={k:claim[k] for k in ['title','year','record_id','work_type','work_label'] if claim.get(k) is not None} + if academic and not any(all(_same_claim_value(k, row.get(k), v) for k,v in academic.items()) for row in rows): issues.append('unsupported_academic_binding') + if academic: + bound_rows = [row for row in rows if all( + _same_claim_value(k, row.get(k), v) for k, v in academic.items())] + if bound_rows and not any(row.get('source_url') in cited for row in bound_rows): + issues.append('missing_academic_record_source') if claim.get('category') is not None and claim.get('document_author') is not None: if not any(a.get('document_path') and a.get('document_path')==b.get('document_path') for a in rows if normalized(a.get('category'))==normalized(claim['category']) for b in rows if normalized(b.get('document_author'))==normalized(claim['document_author'])): issues.append('disconnected_author_category') + required_records = context.get('required_academic_records', []) + if context.get('status') == 'ok' and required_records: + if answer.get('status') != 'ok': + issues.append('rejected_supported_academic_listing') + fields = ['person', 'record_id', 'title', 'year', 'work_type', 'work_label'] + for record in required_records: + if not any(all(_same_claim_value(key, claim.get(key), record[key]) + for key in fields if record.get(key) is not None) + for claim in claims): + issues.append('missing_academic_record_details') + break candidates=context.get('answer_candidates',[]) if context.get('status')=='ok' and candidates: if answer.get('status')=='no_answer': diff --git a/tools/answer_evaluation/graph_answer_prompt.txt b/tools/answer_evaluation/graph_answer_prompt.txt index 4b4aeb0..f593d5a 100644 --- a/tools/answer_evaluation/graph_answer_prompt.txt +++ b/tools/answer_evaluation/graph_answer_prompt.txt @@ -1,6 +1,7 @@ Vyber iba pravdivé štruktúrované tvrdenia z dodaného graph_context. Nepoužívaj vlastné vedomosti, zdrojový oracle ani údaje z otázky ako dôkaz. Vráť jediný platný JSON objekt: {"answer":"","status":"ok","claims":[],"source_urls":[],"count":null}. Answer MUSÍ zostať prázdny reťazec. Prirodzený text, citácie a upozornenie na úplnosť zostaví aplikácia z overených polí. Nepíš voľný text ani Markdown mimo JSON. Všetky reťazce ohraničuj ASCII dvojitými úvodzovkami; úvodzovky v hodnotách escapuj. -Každá claim obsahuje person a len požadované polia, ktoré sú priamo doložené faktami: title, year, start_year, keyword, category, document_author. Nepovinné polia vynechaj. Roky a count sú celé čísla. Presne zachovaj mená, názvy, tagy a kódy kategórií. Nevytváraj nové osoby, hodnoty ani typy tvrdení. +Každá claim obsahuje person a len požadované polia, ktoré sú priamo doložené faktami: title, year, start_year, keyword, category, document_author, record_id, work_type, work_label. Nepovinné polia vynechaj. Roky a count sú celé čísla. Presne zachovaj mená, názvy, tagy a kódy kategórií. Nevytváraj nové osoby, hodnoty ani typy tvrdení. +Ak kontext obsahuje required_academic_records, otázka žiada jednotlivé práce/projekty. Pre každý dodaný záznam vytvor samostatnú claim so všetkými jeho známymi poľami person, record_id, title, year, work_type, work_label. Iba zoznam rokov nestačí. Nezlučuj rôzne záznamy s rovnakým rokom alebo typom. Ak title alebo year chýba, vynechaj ho; názov nevymýšľaj a nenahrádzaj ho nadpisom. Work_label je pôvodný nadpis sekcie a work_type je presný kód typu toho istého záznamu. Record_id je technický identifikátor dôkazu, nie text odpovede. Pri orezanom kontexte pokry všetky dodané záznamy; neodhaduj chýbajúce. Title je názov akademickej práce/projektu, nikdy názov profilovej stránky alebo generický odkaz na rešerš. Year je rok tej istej práce ako title. Start_year je explicitne doložený začiatok štúdia; rok adresára nie je dôkaz. Keyword je skutočný tag osoby, nikdy typ práce ako diplomový projekt. Category je presný taxonomy.category kód. Document_author je autor dokumentu, nie automaticky študent. Všetky podmienky jednej claim patria tej istej osobe. Title a year patria rovnakému akademickému záznamu. Category a document_author musia byť doložené tým istým document_path. Tag patrí osobe cez jej profil aj dcérske dokumenty. Pre kombináciu názvu/roku a tagu nevyžaduj rovnaký dokument. Answer_candidates sú už doložené spojenia cez tú istú osobu; zachovaj ich person, year, keyword a prípadne title spolu v jednej claim. Neodmietaj toto spojenie len pre odlišné document_path. diff --git a/tools/answer_evaluation/graph_context.py b/tools/answer_evaluation/graph_context.py index f5c8e6c..faf0972 100644 --- a/tools/answer_evaluation/graph_context.py +++ b/tools/answer_evaluation/graph_context.py @@ -9,6 +9,8 @@ def graph_context(result: dict) -> dict: Values/subjects/source paths are copied without inventing or rewriting facts. """ facts = []; intent = result['constraints'] + work_listing = intent.get('mentions_work_listing', False) + academic_records = [] for row in result['facts']: kind = row.get('fact_type', '') if kind.startswith('shared_'): @@ -23,13 +25,22 @@ def graph_context(result: dict) -> dict: if not row.get('person'): continue allowed = {'person', 'document_path', 'source_url', 'keyword', 'category', 'document_author'} - if intent['mentions_title']: + if intent['mentions_title'] or work_listing: allowed.add('title') - if intent['mentions_title'] or intent['mentions_academic_year'] or intent['years']: + if intent['mentions_title'] or intent['mentions_academic_year'] or intent['years'] or work_listing: allowed.add('year') if intent['mentions_start_year']: allowed.add('start_year') fact = {k: v for k, v in row.items() if k in allowed and v is not None} + if work_listing and kind in {'person_thesis', 'person_work'}: + # Stable source record identity distinguishes two untitled projects + # by the same person in the same year. Never use Neo4j element IDs. + fact.update(record_id=row.get('thesis_id') or row.get('work_id'), + work_type=row.get('thesis_type') or row.get('work_type'), + work_label=row.get('heading'), title=row.get('title')) + fact = {k: v for k, v in fact.items() if v is not None or k == 'title'} + if fact not in academic_records: + academic_records.append(fact) if fact not in facts: facts.append(fact) sources=result.get('sources') @@ -41,5 +52,7 @@ def graph_context(result: dict) -> dict: 'truncation_reasons':result.get('truncation_reasons',[]), 'index_version':result.get('index_version')} context['answer_candidates']=joined_candidates(facts,intent) + if work_listing: + context['required_academic_records'] = academic_records return context diff --git a/tools/answer_evaluation/render_answer.py b/tools/answer_evaluation/render_answer.py index 6ac2cd7..044be6b 100644 --- a/tools/answer_evaluation/render_answer.py +++ b/tools/answer_evaluation/render_answer.py @@ -1,15 +1,23 @@ """Render only validated structured facts; never infer facts or read gold.""" from copy import deepcopy +from tools.graphrag.graphrag import WORK_TYPE_LABELS + def render_graph_answer(answer: dict, context: dict) -> dict: out = deepcopy(answer) lines = [] if out['status'] == 'ok': - labels = {'title': 'názov práce', 'year': 'rok práce', 'start_year': 'začiatok štúdia', + labels = {'work_label': 'záznam', 'work_type': 'typ práce', + 'title': 'názov práce', 'year': 'rok práce', 'start_year': 'začiatok štúdia', 'keyword': 'kľúčové slovo', 'category': 'kategória', 'document_author': 'autor dokumentu'} for claim in out['claims']: - values = [f'{label}: {claim[key]}' for key, label in labels.items() if claim.get(key) is not None] + values = [f'{label}: {WORK_TYPE_LABELS.get(claim[key], claim[key]) if key == "work_type" else claim[key]}' + for key, label in labels.items() if claim.get(key) is not None] + if claim.get('record_id') and not claim.get('title'): + values.append('názov v zdroji nie je uvedený') + if claim.get('record_id') and claim.get('year') is None: + values.append('rok v zdroji nie je uvedený') lines.append(claim['person'] + (' — ' + '; '.join(values) if values else '') + '.') if out.get('count') is not None: lines.append(f'Počet vyhovujúcich osôb: {out["count"]}.') diff --git a/tools/graphrag/evaluation/reproducibility.py b/tools/graphrag/evaluation/reproducibility.py index 8f07ddc..219a900 100644 --- a/tools/graphrag/evaluation/reproducibility.py +++ b/tools/graphrag/evaluation/reproducibility.py @@ -43,7 +43,8 @@ def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]: 'tools/sync/index_state.py', 'tools/rag/common.py', 'tools/sync/sync_indexes.py', 'tools/rag/rag_utils.py', 'tools/answer_evaluation/context_policy.py', 'tools/answer_evaluation/checked_generation.py', 'tools/answer_evaluation/answer_format.py', - 'tools/answer_evaluation/render_answer.py', + 'tools/answer_evaluation/render_answer.py', 'tools/answer_evaluation/graph_context.py', + 'tools/answer_evaluation/graph_answer_prompt.txt', 'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py']) files = sorted(set(files)) return { diff --git a/tools/graphrag/graph_search.py b/tools/graphrag/graph_search.py index d82b2ae..1311910 100644 --- a/tools/graphrag/graph_search.py +++ b/tools/graphrag/graph_search.py @@ -973,6 +973,16 @@ def extract_query_constraints( ) ) + # Asking *which works* requests records, even without the word "title". + # Keep this separate from title-only lookup: untitled projects must survive. + academic_noun = r"(?:prac(?:a|e|u)?|projekt(?:y|ov|u)?)" + modifiers = r"(?:(?:vsetk\w*|(?:akademick|zaverecn|bakalarsk|diplomov|timov|vyskumn|dizertacn)\w*)\s+)*" + mentions_work_listing = bool(re.search( + rf"\b(?:ake|aku|aky|ktore|ktoru|ktory|vypis|vymenuj|uved|zobraz|ukaz|zoznam|prehlad)\s+{modifiers}{academic_noun}\b", + normalized, + ) or re.search(r"\b(?:ake|ktore)\s+(?:ma|mal|maju|mali)\b.*?\b(?:prace|projekty)\b", normalized) + ) and not bool(token_set & {"kolko", "pocet", "spocitaj"}) and not mentions_person_answer + mentions_thesis = ( bool( thesis_types @@ -986,9 +996,7 @@ def extract_query_constraints( ) ) - mentions_work = bool( - work_types - ) + mentions_work = bool(work_types) or (mentions_work_listing and has_project_word and not has_work_word) target_kinds: list[str] = [] @@ -1026,7 +1034,7 @@ def extract_query_constraints( ) if ( - mentions_title + (mentions_title or mentions_work_listing) and not mentions_thesis and not mentions_work ): @@ -1085,6 +1093,7 @@ def extract_query_constraints( mentions_title ), "mentions_academic_year": mentions_academic_year, + "mentions_work_listing": mentions_work_listing, "keyword_phrase": explicit_keyword_phrase(query), "mentions_count": bool(token_set & {"kolko", "pocet", "spocitaj"}), "mentions_shared": ( @@ -2408,13 +2417,14 @@ def search_graph_with_session( author_ids = ids("DocumentAuthor") year_specific = bool(constraints["years"] or constraints["mentions_academic_year"]) title_specific = constraints["mentions_title"] + academic_specific = title_specific or year_specific or constraints["mentions_work_listing"] thesis_specific = bool( constraints["mentions_thesis"] or thesis_ids - or ((title_specific or year_specific) and not constraints["mentions_work"]) + or (academic_specific and not constraints["mentions_work"]) ) and not constraints["mentions_start_year"] work_specific = bool( constraints["mentions_work"] or work_ids - or ((title_specific or year_specific) and not constraints["mentions_thesis"]) + or (academic_specific and not constraints["mentions_thesis"]) ) and not constraints["mentions_start_year"] shared_keyword = bool( constraints["mentions_shared"] and len(person_ids) >= 2 @@ -2491,7 +2501,7 @@ def search_graph_with_session( work_rows = fetch(work_specific, WORK_FACTS_QUERY, work_ids=work_ids, years=constraints["years"], work_types=constraints["work_types"]) - if title_specific: + if title_specific and not constraints["mentions_work_listing"]: thesis_rows = [row for row in thesis_rows if row.get("title")] work_rows = [row for row in work_rows if row.get("title")] start_rows = fetch(constraints["mentions_start_year"], START_YEAR_FACTS_QUERY,