From 16a3f92db5a4ad3040eb410c54c8b7290ebf4cdd Mon Sep 17 00:00:00 2001 From: jp170na Date: Sun, 8 Nov 2026 23:23:51 +0100 Subject: [PATCH] Preserve independent semantic candidates in hybrid RAG search --- README.md | 4 +- tools/answers/answer_delivery.py | 2 +- tools/answers/question_plan.py | 23 +++- tools/answers/rag_evidence.py | 2 + tools/answers/verified_rag.py | 3 + tools/knowledge/query.py | 6 + tools/rag/evaluation/retrieval_runner.py | 24 ++-- tools/rag/rag_utils.py | 27 +++- tools/rag/search_core.py | 14 +- tools/rag/search_scope.py | 91 +++++++++++++ tools/rag/search_utils.py | 37 ++---- tools/rag/tests/test_hybrid_candidates.py | 154 ++++++++++++++++++++++ tools/rag/tests/test_search.py | 10 +- tools/retrieval/person_names.py | 7 +- 14 files changed, 351 insertions(+), 53 deletions(-) create mode 100644 tools/rag/search_scope.py create mode 100644 tools/rag/tests/test_hybrid_candidates.py diff --git a/README.md b/README.md index 97a1804..0f2408e 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ z rovnakého snapshotu a aktivujú spoločne. Pri chybe zostáva predchádzajúc ## Vyhľadávanie **RAG** kombinuje fulltextové vyhľadávanie BM25 so sémantickým vyhľadávaním -pomocou embeddingov. Výsledky spája metódou RRF. Je vhodný na otázky -na obsah, zadanie, metódy, priebeh a výsledky práce. +pomocou embeddingov. Samostatné výsledky spája metódou RRF; osobu, typ a rok filtruje záväzne. +Je vhodný na otázky na obsah, zadanie, metódy, priebeh a výsledky práce. **GraphRAG** využíva znalostný graf osôb, dokumentov, prác, projektov, tém, tagov a kategórií. Umožňuje vyhľadávať vzťahy, spoločné vlastnosti diff --git a/tools/answers/answer_delivery.py b/tools/answers/answer_delivery.py index 54e618f..d8e423c 100644 --- a/tools/answers/answer_delivery.py +++ b/tools/answers/answer_delivery.py @@ -138,7 +138,7 @@ def _deliver_answer( if engine == "rag": try: with index_snapshot(db_file.parent, db_file=db_file): - retrieved = build_rag_context(db_file, question, limit=limit) + retrieved = build_rag_context(db_file, question, limit=limit, question_plan=plan.spec) if retrieved.get("status") == "error": return retrieval_failure(engine) if failure := name_failure(retrieved, engine): diff --git a/tools/answers/question_plan.py b/tools/answers/question_plan.py index e92a853..76f38e4 100644 --- a/tools/answers/question_plan.py +++ b/tools/answers/question_plan.py @@ -60,7 +60,7 @@ class Period(Strict): return self class Condition(Strict): - kind: Literal['tag','category','document_author','quantity','exclusion','current_status','other'] + kind: Literal['tag','category','document_author','quantity','exclusion','current_status','year_verification','other'] quote: str class QuestionPlan(Strict): @@ -71,7 +71,7 @@ class QuestionPlan(Strict): years: list[int] = Field(default_factory=list) period: Period | None = None conditions: list[Condition] = Field(default_factory=list) - fields: list[Literal['title','year','work_type','keyword','category','document_author','supervisor','consultant','status','date']] = Field(default_factory=list) + fields: list[Literal['title','year','start_year','work_type','keyword','category','document_author','supervisor','consultant','status','date']] = Field(default_factory=list) facets: list[Facet] = Field(default_factory=list) scope: Literal['across_works','single_work','selected_works'] = Field(default='across_works',description='Scope is PER PERSON. single_work means one work per student, including a comparison of two students. selected_works means multiple explicitly requested works for the same person. Preserve the baseline scope.') presentation: Literal['brief','overview','list','comparison','table','count'] = 'overview' @@ -206,7 +206,25 @@ def direct_spec(question, resolution): conditions.extend(Condition(kind=kind,quote=m[0]) for m in re.finditer(pattern,n)) if mode and mode!='metadata' and conditions:unsupported.append('unsupported_content_join') conditions.extend(Condition(kind='current_status',quote=m[0]) for m in re.finditer(r'\b(?:aktual\w*|teraz|dnes)\b',n)) + study_start = bool(re.search(r'\b(?:zaciat\w* stud\w*|zacal\w* stud\w*)\b', n)) + year_verification = bool(re.search(r'\b(?:je|bol)\s+(?:rok\s+)?[12]\d{3}\s+rokom\b', n)) + if study_start or year_verification: + # A value to verify is not an asserted work-year restriction. Keep + # the literal so an LLM cannot erase the verification. + verification_values = {int(m[1]) for m in re.finditer(r'\b(?:je|bol)\s+(?:rok\s+)?([12]\d{3})\s+rokom\b', n)} + study_values = {int(m[1]) for m in re.finditer(r'\b(?:zaciat\w* stud\w*|zacal\w* stud\w*)[^\d.;?]{0,35}([12]\d{3})\b', n)} + if study_start and not study_values: + if len(years) == 1: + study_values = set(years) + elif years and not year_verification: + unsupported.append('ambiguous_time_roles') + for work in works: + work.years = [year for year in work.years if year not in study_values | verification_values] + if year_verification: + conditions.append(Condition(kind='year_verification', quote=q)) fields=[field for field,pattern in [('title',r'\bnaz(?:ov|v)\w*'),('year',r'\b(?:ktoreho roku|ktorych rokoch|rok\w*)'),('work_type',r'\btyp\w*'),('keyword',r'\b(?:tag\w*|klucov\w*)'),('category',r'\bkategor\w*'),('document_author',r'\bautor\w* dokument\w*'),('supervisor',r'\b(?:vedie|veduc\w*|veden\w*)'),('consultant',r'\bkonzult(?:ant|oval)\w*'),('status',r'\b(?:ukoncil\w*|obhaj\w*|skoncil\w*)'),('date',r'\b(?:kedy|datum\w*)')] if re.search(pattern,n)] + if study_start: + fields.append('start_year') return QuestionPlan(fields=fields,question=q,intent=mode or 'metadata',people=people,works=works,years=years,period=period, conditions=conditions,facets=facets,scope=scope,presentation=presentation,unsupported=list(dict.fromkeys(unsupported))) @@ -268,6 +286,7 @@ def build_question_plan(question, catalog, *, complete=None, verify_complete=Non 'period_needs_dates':'Pre toto obdobie uveď konkrétny interval dátumov; semester nebudem odhadovať.', 'invalid_period':'Uveď platný dátum alebo interval dátumov.', 'invalid_year':'Uveď rok štyrmi číslicami; túto podmienku nevynechám.', + 'ambiguous_time_roles':'Rozlíš, ktorý rok označuje začiatok štúdia a ktorý rok konkrétnej práce.', 'unsupported_content_join':'Otázka spája obsah s metadátovou podmienkou, ktorú táto cesta nepodporuje. Podmienku nevynechám.'} message=messages.get(code) or constraint_message(code) return bound.model_copy(update={'reason':code if code in messages else 'unsupported_constraint','clarification':message}) diff --git a/tools/answers/rag_evidence.py b/tools/answers/rag_evidence.py index 2f0109b..939634a 100644 --- a/tools/answers/rag_evidence.py +++ b/tools/answers/rag_evidence.py @@ -805,6 +805,8 @@ def rag_evidence_context(question: str, payload: dict) -> dict: requirements["years"] = ( requested_years(question) if academic and not requirements["start_year"] else [] ) + if plan and any(c.kind == 'year_verification' for c in plan.conditions): + requirements['years'] = [] requirements["unsupported_year"] = any(len(value) != 4 for value in stated_years) requirements["project_number"] = project_number(question) requirements["exact_work_label"] = selector.labels[0] if len(selector.labels) == 1 else None diff --git a/tools/answers/verified_rag.py b/tools/answers/verified_rag.py index 6688848..553b0c4 100644 --- a/tools/answers/verified_rag.py +++ b/tools/answers/verified_rag.py @@ -352,6 +352,9 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: cited = set(answer.get("source_urls", [])) blocks = {row["source_id"]: row for row in context["source_blocks"]} for quote in quotes: + from tools.knowledge.query import activity_supported + if not activity_supported(context['question'], {'text': quote['quote']}): + issues.append('unsupported_requested_activity') source = blocks.get(quote["source_id"]) if not source or " ".join(quote["quote"].split()) not in " ".join( source["full_text"].split() diff --git a/tools/knowledge/query.py b/tools/knowledge/query.py index 966eb40..ca5a22a 100644 --- a/tools/knowledge/query.py +++ b/tools/knowledge/query.py @@ -174,6 +174,12 @@ def activity_supported(question, row): named_entities = bool( re.search(r"\bpomenovan\w* entit\w*\b|\bnamed entit\w*\b|\bner\b", wording) ) + if named_entities and re.search(r'\brozpoznav\w*|\brozpoznava\w*|\bnamed entity recognition\b|\bner\b', wording): + text = normalized(row['text']) + if re.search(r'\banot\w*\b', text) and not re.search( + r'\brozpoznav\w*|\brozpoznava\w*|\bnamed entity recognition\b', text + ): + return False if re.search(r"\banot\w*\b", wording) and named_entities: text = normalized(row["text"]) if not re.search(r"\banot\w*\b", text): diff --git a/tools/rag/evaluation/retrieval_runner.py b/tools/rag/evaluation/retrieval_runner.py index 33c9040..0cacdcb 100644 --- a/tools/rag/evaluation/retrieval_runner.py +++ b/tools/rag/evaluation/retrieval_runner.py @@ -224,6 +224,10 @@ def retrieve_all_modes( conn.execute("PRAGMA query_only = ON") verify_search_schema(conn) + from tools.rag.search_scope import prepare_search_scope + scope = prepare_search_scope(conn, db_file, clean_query, published_only=published_only) + if scope.failure: + return {mode: [] for mode in EVALUATION_MODES} # ------------------------- # FTS5 @@ -242,6 +246,7 @@ def retrieve_all_modes( match_query, candidate_limit, published_only, + eligible_chunk_ids=scope.eligible_ids, ) if not rows: @@ -259,7 +264,7 @@ def retrieve_all_modes( fts_results = add_fts_metadata( conn, clean_query, - fts_candidates, + scope.project_rows(fts_candidates), ) # ------------------------- @@ -271,32 +276,21 @@ def retrieve_all_modes( clean_query, candidate_limit, published_only, + eligible_chunk_ids=scope.eligible_ids, ) vector_results = add_vector_metadata( conn, - vector_candidates, + scope.project_rows(vector_candidates), ) # ------------------------- # Hybrid # ------------------------- - hybrid_vector_results = vector_results - - if used_strategies and used_strategies[0] in { - "all_terms", - "prefix_terms", - }: - fts_chunk_ids = {item["chunk_id"] for item in fts_results} - - hybrid_vector_results = [ - item for item in vector_results if item["chunk_id"] in fts_chunk_ids - ] - hybrid_results = fuse_hybrid_results( fts_results, - hybrid_vector_results, + vector_results, ) return { diff --git a/tools/rag/rag_utils.py b/tools/rag/rag_utils.py index bb7afe5..cdb2534 100644 --- a/tools/rag/rag_utils.py +++ b/tools/rag/rag_utils.py @@ -491,6 +491,7 @@ def build_source_text(result: dict[str, Any]) -> str: seen, label="ZAČIATOK RELEVANTNEJ SEKCIE", text=str(result.get("section_lead_text") or ""), + section=result.get('heading_paths') or [], ) _append_unique_block( @@ -498,6 +499,7 @@ def build_source_text(result: dict[str, Any]) -> str: seen, label="NAJRELEVANTNEJŠÍ NÁJDENÝ ÚSEK", text=primary, + section=result.get('heading_paths') or [], ) if not blocks: @@ -655,6 +657,7 @@ def build_rag_context( limit: int = 5, published_only: bool = DEFAULT_PUBLISHED_ONLY, max_per_document: int = 1, + question_plan=None, ) -> dict[str, Any]: from tools.retrieval.person_names import resolve_person_names, sqlite_people @@ -681,7 +684,7 @@ def build_rag_context( from tools.retrieval.question_constraints import unsupported_constraint, constraint_message unsupported = unsupported_constraint(query) - if unsupported: + if unsupported and question_plan is None: return { **result_fields( status="ambiguous", sources=[], reason="unsupported_constraint", complete=False @@ -702,8 +705,16 @@ def build_rag_context( limit, published_only=published_only, max_per_document=max_per_document, + **({'question_plan': question_plan} if question_plan is not None else {}), ) + if response.get('status') == 'ambiguous': + return {**{key: response.get(key) for key in + ['status','reason','sources','complete','truncated','truncation_reasons','index_version','clarification','name_resolution']}, + 'query': original_query, 'engine': response['engine'], 'strategies': [], + 'source_count': 0, 'context': '', 'instructions': RAG_INSTRUCTIONS, + 'answer_format': ANSWER_FORMAT} + results = [dict(item) for item in response["results"]] results = _expand_exact_document( @@ -749,6 +760,20 @@ def build_rag_context( for row in results if row.get("document_path") not in profile_paths or row["document_path"] in identities ] + # Expansions are candidates too: they must not restore another work/year + # that was excluded before the BM25/vector candidate budgets. + if db_path.exists(): + from tools.answers.question_plan import build_question_plan, QuestionPlan + from tools.rag.search_scope import SearchScope + spec = (question_plan if isinstance(question_plan, QuestionPlan) else + QuestionPlan.model_validate(question_plan)) if question_plan is not None else build_question_plan(query, catalog).spec + # Render expansions once, then filter the combined evidence. Strip the + # raw expansion texts so build_source cannot append excluded works again. + text_fields = {'exact_document_text', 'family_evidence_blocks', 'query_evidence_blocks', + 'query_focus_text', 'query_evidence_text', 'section_lead_text'} + results = SearchScope(spec).project_rows([ + {**{key: value for key, value in row.items() if key not in text_fields}, + 'text': build_source_text(row)} for row in results]) sources = [build_source(result, index) for index, result in enumerate(results, start=1)] context = build_context_text(sources) coverage = { diff --git a/tools/rag/search_core.py b/tools/rag/search_core.py index 859bcff..6fc6f30 100644 --- a/tools/rag/search_core.py +++ b/tools/rag/search_core.py @@ -307,6 +307,8 @@ def run_fts_query( match_query: str, candidate_limit: int, published_only: bool, + *, + eligible_chunk_ids: set[str] | None = None, ) -> list[dict[str, Any]]: rows = conn.execute( f""" @@ -341,6 +343,7 @@ def run_fts_query( ? = 0 OR COALESCE(chunks.published, 1) = 1 ) + AND (? IS NULL OR chunks.chunk_id IN (SELECT value FROM json_each(?))) ORDER BY chunks_fts.rank ASC, chunks.id ASC @@ -349,6 +352,8 @@ def run_fts_query( ( match_query, 1 if published_only else 0, + None if eligible_chunk_ids is None else json.dumps(sorted(eligible_chunk_ids)), + None if eligible_chunk_ids is None else json.dumps(sorted(eligible_chunk_ids)), candidate_limit, ), ).fetchall() @@ -361,7 +366,11 @@ def run_vector_query( query: str, candidate_limit: int, published_only: bool, + *, + eligible_chunk_ids: set[str] | None = None, ) -> list[dict[str, Any]]: + if eligible_chunk_ids is not None and not eligible_chunk_ids: + return [] model_name, dimensions = embedding_index_info(conn) query_vector = embed_query( @@ -396,8 +405,11 @@ def run_vector_query( ? = 0 OR COALESCE(chunks.published, 1) = 1 ) + AND (? IS NULL OR chunks.chunk_id IN (SELECT value FROM json_each(?))) """, - (1 if published_only else 0,), + (1 if published_only else 0, + None if eligible_chunk_ids is None else json.dumps(sorted(eligible_chunk_ids)), + None if eligible_chunk_ids is None else json.dumps(sorted(eligible_chunk_ids))), ).fetchall() candidates: list[dict[str, Any]] = [] diff --git a/tools/rag/search_scope.py b/tools/rag/search_scope.py new file mode 100644 index 0000000..65024c5 --- /dev/null +++ b/tools/rag/search_scope.py @@ -0,0 +1,91 @@ +"""Hard question filters, independent of lexical/semantic relevance scores.""" +from __future__ import annotations + +from dataclasses import dataclass +import json + +from tools.answers.question_plan import QuestionPlan, build_question_plan +from tools.answers.rag_evidence import academic_windows_metadata +from tools.answers.context_policy import normalized +from tools.retrieval.person_names import sqlite_people +from tools.retrieval.retrieval_contract import result_fields + + +@dataclass +class SearchScope: + plan: QuestionPlan + failure: dict | None = None + eligible_ids: set[str] | None = None + + def project(self, row: dict) -> dict | None: + """Clip mixed-work chunks; a date/name elsewhere is not a work binding.""" + students = [p for p in self.plan.people if p.role == 'student'] + path = row.get('document_path', '') + owners = [p for p in students if p.source_document and ( + path == p.source_document or path.startswith(p.source_document.removesuffix('README.md')))] + if students and not owners: + return None + authors = [p.name for p in self.plan.people if p.role == 'document_author'] + if authors and normalized(row.get('author') or '') not in {normalized(p) for p in authors}: + return None + constrained = any(w.work_type or w.years or w.labels or w.excluded_labels or + w.numbers or w.excluded_numbers or w.unnumbered for w in self.plan.works) + constrained |= bool(self.plan.period and self.plan.period.field == 'work_year') + if not constrained: + return dict(row) + text = row.get('text') or '' + windows = academic_windows_metadata(text) + if not any(scope for scope, _, _ in windows): + # Chunk ancestry can be the only explicit work heading. Never use + # a profile directory or taxonomy year as an academic work year. + paths = row.get('heading_paths') + if paths is None: + paths = json.loads(row.get('heading_paths_json') or '[]') + first = paths[0] if paths else '' + heading = ' > '.join(first) if isinstance(first, list) else str(first) + body = '\n'.join(line for line in text.splitlines() if not line.startswith('Dokument:')) + windows = academic_windows_metadata('Sekcia: ' + heading + '\n' + body) + names = [p.name for p in owners] or [None] + selected = [(scope, label, body) for scope, label, body in windows if scope and any( + self.plan.matches_work(name, label, scope[0], scope[1]) for name in names)] + if not selected: + return None + # Preserve scoped source headings for the evidence verifier, but omit + # other works even if one chunk or an expansion spans several works. + clipped = '\n\n'.join('Sekcia: ' + label + '\n' + body for _, label, body in selected) + if 'start_year' in self.plan.fields: + # Study metadata belongs to the profile, independently of work + # sections. Keep the explicit line when both values were requested. + study_lines = [line for scope, _, body in windows if scope is None + for line in body.splitlines() if normalized(line).startswith('rok zaciatku studia ')] + clipped = '\n'.join(study_lines) + '\n\n' + clipped if study_lines else clipped + headings = [[label] for _, label, _ in selected] + return {**row, 'text': clipped, 'text_length': len(clipped), + 'snippet': clipped[:320], 'heading_paths': headings, + 'heading_paths_json': json.dumps(headings, ensure_ascii=False)} + + def project_rows(self, rows): + return [value for row in rows if (value := self.project(row)) is not None] + + +def prepare_search_scope(conn, db_file, query, *, published_only, question_plan=None): + if question_plan is None: + bound = build_question_plan(query, sqlite_people(db_file, published_only=published_only)) + plan = bound.spec + if bound.clarification: + status = 'no_answer' if bound.resolution.get('status') == 'no_answer' and bound.reason == bound.resolution.get('reason') else 'ambiguous' + return SearchScope(plan, failure={ + **result_fields(status=status, sources=[], reason=bound.reason, complete=False), + 'engine': 'hybrid_fts5_embeddings', 'results': [], 'strategies': [], + 'clarification': bound.clarification, 'name_resolution': bound.resolution}) + else: + plan = question_plan if isinstance(question_plan, QuestionPlan) else QuestionPlan.model_validate(question_plan) + if normalized(plan.question) != normalized(query) or plan.unsupported: + raise ValueError('Search plan does not match the supported question') + scope = SearchScope(plan) + if plan.people or any(w.work_type or w.years or w.labels or w.excluded_labels or + w.numbers or w.excluded_numbers or w.unnumbered for w in plan.works) or plan.period: + rows = conn.execute('SELECT * FROM chunks WHERE (?=0 OR COALESCE(published,1)=1)', + (int(published_only),)).fetchall() + scope.eligible_ids = {row['chunk_id'] for row in rows if scope.project(dict(row)) is not None} + return scope diff --git a/tools/rag/search_utils.py b/tools/rag/search_utils.py index 8629e3b..a8b0dfd 100644 --- a/tools/rag/search_utils.py +++ b/tools/rag/search_utils.py @@ -54,6 +54,8 @@ def search_database( limit: int = 10, published_only: bool = DEFAULT_PUBLISHED_ONLY, max_per_document: int = 3, + *, + question_plan=None, ) -> dict[str, Any]: if limit < 1 or max_per_document < 0: raise ValueError("Limit musí byť kladný a max_per_document nezáporné.") @@ -86,6 +88,11 @@ def search_database( conn.execute("PRAGMA query_only = ON") verify_search_schema(conn) + from tools.rag.search_scope import prepare_search_scope + scope = prepare_search_scope(conn, db_file, clean_query, + published_only=published_only, question_plan=question_plan) + if scope.failure: + return scope.failure # ------------------------- # FTS5 kandidáti @@ -105,6 +112,7 @@ def search_database( match_query, candidate_limit + 1, published_only, + eligible_chunk_ids=scope.eligible_ids, ) if not rows: @@ -126,7 +134,7 @@ def search_database( fts_results = add_fts_metadata( conn, clean_query, - fts_candidates, + scope.project_rows(fts_candidates), ) # ------------------------- @@ -138,6 +146,7 @@ def search_database( clean_query, candidate_limit + 1, published_only, + eligible_chunk_ids=scope.eligible_ids, ) vector_capped = len(vector_candidates) > candidate_limit @@ -145,29 +154,9 @@ def search_database( vector_results = add_vector_metadata( conn, - vector_candidates, + scope.project_rows(vector_candidates), ) - # ------------------------- - # Presné FTS výsledky - # ------------------------- - # - # Pri all_terms alebo prefix_terms - # embeddingy iba preradia chunky, - # ktoré už našiel FTS. - # - # Tým sa zabráni pridávaniu - # sémanticky podobných, ale - # lexikálne nesúvisiacich výsledkov. - - if used_strategies and used_strategies[0] in { - "all_terms", - "prefix_terms", - }: - fts_chunk_ids = {item["chunk_id"] for item in fts_results} - - vector_results = [item for item in vector_results if item["chunk_id"] in fts_chunk_ids] - # ------------------------- # Hybrid fusion # ------------------------- @@ -190,9 +179,7 @@ def search_database( reasons = [] if fts_candidates and fts_capped: reasons.append("candidate_limit") - if vector_capped and ( - not used_strategies or used_strategies[0] not in {"all_terms", "prefix_terms"} - ): + if vector_capped: reasons.append("candidate_limit") eligible = diversify_results(hybrid_results, max(1, len(hybrid_results)), max_per_document) if len(eligible) < len(hybrid_results): diff --git a/tools/rag/tests/test_hybrid_candidates.py b/tools/rag/tests/test_hybrid_candidates.py new file mode 100644 index 0000000..c1c10d9 --- /dev/null +++ b/tools/rag/tests/test_hybrid_candidates.py @@ -0,0 +1,154 @@ +"""Independent candidate recall, hard scope before top-k and verified activities.""" +import json +import sqlite3 + +import numpy as np +import pytest + +from tools.rag import build_sqlite_index as indexer, search_core as core, search_utils as search +from tools.rag.evaluation.retrieval_runner import retrieve_all_modes +from tools.rag.search_scope import SearchScope +from tools.answers.question_plan import build_question_plan + +EVA = 'pages/students/2022/eva_nova/README.md' +ADAM = 'pages/students/2022/adam_stary/README.md' + + +@pytest.fixture +def corpus(tmp_path, monkeypatch): + docs = [(EVA, 'Eva Nová'), (ADAM, 'Adam Starý'), + ('pages/topics/literal/README.md', 'Lexikálny príklad'), + ('pages/topics/semantic/README.md', 'Sémantický príklad'), + ('pages/topics/hidden/README.md', 'Skrytý príklad')] + records = [ + ('eva-bp', EVA, 'Bakalárska práca 2020', 'Používala indexovanie dokumentov.', .5), + ('eva-dp', EVA, 'Diplomová práca 2021', 'Používala embeddingy dokumentov.', .9), + ('adam-bp', ADAM, 'Bakalárska práca 2020', 'Používal indexovanie dokumentov.', 1.), + ('adam-dp', ADAM, 'Diplomová práca 2021', 'Používal embeddingy dokumentov.', .99), + ('literal', docs[2][0], '', 'retrieval je lexikálny záchytný výraz.', .6), + ('semantic', docs[3][0], '', 'Získavanie informácií z rozsiahlych zbierok textov.', .98), + ('hidden', docs[4][0], '', 'retrieval tajné informácie.', 1.), + ] + documents = [{'path': path, 'title': name, 'published': path != docs[4][0]} for path, name in docs] + chunks = [] + vectors = {} + for number, (key, path, heading, text, similarity) in enumerate(records): + title = dict(docs)[path] + body = f'Dokument: {title}\n' + (f'Sekcia: {heading}\n' if heading else '') + text + vectors[key] = [similarity, (1 - similarity**2)**.5, 0., 0.] + chunks.append({'chunk_id': key, 'document_path': path, 'title': title, + 'text': body, 'published': key != 'hidden', 'chunk_index': number, + 'heading_paths': [[heading]] if heading else [], 'tags': [], 'categories': []}) + monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.asarray([ + vectors[next(row['chunk_id'] for row in chunks if row['text'] in text)] for text in texts], dtype=np.float32)) + monkeypatch.setattr(core, 'embed_query', lambda *a, **kw: np.asarray([1., 0., 0., 0.], dtype=np.float32)) + for name, value in [('documents', documents), ('chunks', chunks)]: + (tmp_path / (name+'.json')).write_text(json.dumps(value, ensure_ascii=False)) + db = tmp_path/'zp_index.sqlite' + indexer.build_database(documents_file=tmp_path/'documents.json', chunks_file=tmp_path/'chunks.json', db_file=db) + return db + + +@pytest.mark.parametrize('query,strategy', [('retrieval','all_terms'), ('retrievalxyz','prefix_terms')]) +def test_semantic_only_candidate_survives_successful_all_terms_and_rrf(corpus, query, strategy): + result = search.search_database(corpus, query, limit=10) + assert result['strategies'] == [strategy] + rows = {row['chunk_id']: row for row in result['results']} + assert rows['literal']['fts_rank'] == 1 + assert rows['semantic']['fts_rank'] is None and rows['semantic']['vector_rank'] is not None + assert rows['semantic']['hybrid_score'] > 0 and 'hidden' not in rows + assert rows['semantic']['hybrid_score'] == pytest.approx( + core.VECTOR_RRF_WEIGHT / (core.RRF_K + rows['semantic']['vector_rank']), abs=1e-8) + # Fusion is the union, not an intersection, with one result per chunk. + assert len(rows) == len(result['results']) + + +def test_evaluator_uses_the_same_union_as_production(corpus): + result = retrieve_all_modes(corpus, 'retrieval', limit=10, published_only=True, max_per_document=0) + assert {r['chunk_id'] for r in result['fts']} == {'literal'} + assert 'semantic' in {r['chunk_id'] for r in result['vector']} + assert {r['chunk_id'] for r in result['hybrid']} == { + r['chunk_id'] for r in search.search_database(corpus, 'retrieval', limit=10, max_per_document=0)['results']} + + +def test_person_type_and_year_filter_before_both_candidate_budgets(corpus, monkeypatch): + monkeypatch.setattr(search, 'MIN_CANDIDATES', 1) + monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1) + value = search.search_database(corpus, + 'Aké metódy používala Eva Nová pri bakalárskej práci v roku 2020?', limit=1) + assert [r['chunk_id'] for r in value['results']] == ['eva-bp'] + assert value['results'][0]['vector_rank'] == 1 # Stronger wrong-person/year vectors were excluded first. + assert value['truncated'] is False + + +def test_wrong_year_has_no_semantic_fallback_to_another_work(corpus): + value = search.search_database(corpus, 'Akú bakalársku prácu mala Eva Nová v roku 1999?') + assert value['status'] == 'no_answer' and value['results'] == [] + + +def test_unknown_person_never_runs_global_embedding_search(corpus, monkeypatch): + monkeypatch.setattr(search, 'run_vector_query', lambda *a, **kw: pytest.fail('Global search for unknown identity')) + value = search.search_database(corpus, 'Aké práce má Xaver Neexistujúci?') + assert value['status'] == 'no_answer' and value['reason'] == 'unknown_person' + + +def test_vector_cap_is_reported_even_when_fts_succeeds(corpus, monkeypatch): + monkeypatch.setattr(search, 'MIN_CANDIDATES', 1) + monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1) + value = search.search_database(corpus, 'retrieval', limit=1) + assert value['truncated'] and 'candidate_limit' in value['truncation_reasons'] + + +def test_one_chunk_cannot_license_another_work_year_or_taxonomy(corpus): + bound = build_question_plan('Aký názov má bakalárska práca Evy Novej z roku 2020?', + [{'name': 'Eva Nová', 'kind': 'Person', 'source_document': EVA}]) + scope = SearchScope(bound.spec) + row = {'document_path': EVA, 'text': '# Bakalárska práca 2020\nNázov: Správny názov.\n' + '# Diplomová práca 2021\nNázov: Nesprávny názov.', 'heading_paths': []} + projected = scope.project(row) + assert projected and 'Správny názov' in projected['text'] and 'Nesprávny názov' not in projected['text'] + assert core.parse_heading_paths(dict(projected)) == [['Bakalárska práca 2020']] + assert scope.project({**row, 'text': 'Názov: Iba kategória bp2020.', 'categories': ['bp2020']}) is None + + +def test_study_year_is_not_a_work_year_filter(corpus): + value = search.search_database(corpus, 'Je pri osobe Eva Nová uvedený rok začiatku štúdia 2022?') + assert value['status'] == 'ok' and {r['document_path'] for r in value['results']} == {EVA} + bound = build_question_plan('Aký názov má bakalárska práca Evy Novej z roku 2020 a aký je rok začiatku štúdia 2022?', + [{'name': 'Eva Nová', 'kind': 'Person', 'source_document': EVA}]) + assert bound.spec.works[0].years == [2020] and 'start_year' in bound.spec.fields + row = {'document_path': EVA, 'heading_paths': [], + 'text': '*Rok začiatku štúdia*: 2022\n# Bakalárska práca 2020\nNázov: Overená práca.'} + assert 'Rok začiatku štúdia' in SearchScope(bound.spec).project(row)['text'] + + +def test_verifying_a_year_keeps_the_evidence_needed_to_refute_it(corpus): + value = search.search_database(corpus, 'Je rok 1999 rokom bakalárskej práce Evy Novej?') + assert value['status'] == 'ok' and {r['chunk_id'] for r in value['results']} == {'eva-bp'} + + +@pytest.mark.parametrize('q',[ + 'Kto sa venuje práci s názvom „Získavanie informácií“?', + 'Ktorý dokument alebo študent súvisí s témou indexovanie a osobou Eva Nová?', + 'Pri osobe Eva Nová uveď názov práce patriacej do kategórie „bp2020“.', + 'Pri osobe Eva Nová uveď názov práce patriacej do kategórie „bp2020“ a rok vyjadrený touto kategóriou.', +]) +def test_query_grammar_is_not_an_unknown_person(corpus,q): + value = search.search_database(corpus,q) + assert value['status'] != 'no_answer' or value['reason'] != 'unknown_person' + + +@pytest.mark.parametrize('question,quote',[ + ('Kto anotoval pomenované entity?', 'Implementoval rozpoznávanie pomenovaných entít.'), + ('Kto sa venoval rozpoznávaniu pomenovaných entít?', 'Anotoval pomenované entity v korpuse.'), +]) +def test_related_tasks_are_rejected_by_the_final_rag_guard(question, quote): + from tools.answers.rag_evidence import rag_evidence_context + from tools.answers.verified_rag import rag_answer_issues + from tools.rag.rag_utils import build_context_text + sources = [{'source_id': 'S1', 'title': 'Príklad', 'document_path': 'pages/topics/example/README.md', + 'source_url': 'https://example.test/source', 'text': quote}] + context = rag_evidence_context(question, {'sources': sources, 'context': build_context_text(sources)}) + answer = {'status': 'ok', 'claims': [], 'answer': '', 'source_urls': ['https://example.test/source'], + 'evidence_quotes': [{'source_id': 'S1', 'quote': quote}]} + assert 'unsupported_requested_activity' in rag_answer_issues(answer, context) diff --git a/tools/rag/tests/test_search.py b/tools/rag/tests/test_search.py index 6d7b4e9..7366f9b 100644 --- a/tools/rag/tests/test_search.py +++ b/tools/rag/tests/test_search.py @@ -30,7 +30,7 @@ def build_search_database( documents = [ { - "path": "pages/students/jan_ptak/README.md", + "path": "pages/students/2022/jan_ptak/README.md", "title": "Ján Pták", "author": "Daniel Hladek", "published": True, @@ -38,7 +38,7 @@ def build_search_database( "metadata": {}, }, { - "path": "pages/students/jan_holp/README.md", + "path": "pages/students/2016/jan_holp/README.md", "title": "Ján Holp", "author": "Daniel Hladek", "published": True, @@ -100,7 +100,7 @@ def build_search_database( chunks = [ chunk( "jan-ptak::0", - "pages/students/jan_ptak/README.md", + "pages/students/2022/jan_ptak/README.md", "Ján Pták", "Dokument: Ján Pták. Agent pre manažment záverečných prác.", True, @@ -109,7 +109,7 @@ def build_search_database( ), chunk( "jan-holp::0", - "pages/students/jan_holp/README.md", + "pages/students/2016/jan_holp/README.md", "Ján Holp", "Dokument: Ján Holp. Získavanie informácií a PageRank.", True, @@ -344,7 +344,7 @@ def test_public_result_format_has_no_internal_id_and_uses_boolean( assert result["published"] is True assert result["chunk_id"] == "jan-ptak::0" - assert result["source_url"].endswith("students/jan_ptak") + assert result["source_url"].endswith("students/2022/jan_ptak") def test_empty_query_returns_empty_result( diff --git a/tools/retrieval/person_names.py b/tools/retrieval/person_names.py index 22dfd06..4322350 100644 --- a/tools/retrieval/person_names.py +++ b/tools/retrieval/person_names.py @@ -31,6 +31,8 @@ def name_token_variants(token: str) -> set[str]: "frantisek": ("frantiska", "frantiskovi", "frantiskom"), } variants.update(irregular.get(token, ())) + if token.endswith(('ii', 'yi')): + variants.add(token + 'ho') if token.endswith("ova"): variants.update(token[:-1] + s for s in ("ej", "u", "ou")) elif token.endswith("y"): @@ -91,7 +93,7 @@ def _score(words: list[str], canonical: list[str]) -> int | None: _STOP = set( "a aj ale alebo ak ako ake aky aku ktore ktori ktora ktory kto kolko ci co obaja obidvaja " - "ma maju mal mali je su bol bola bolo boli budu bude sa s so pri pre z zo na v od do roku rokoch " + "ma maju mal mali je su bol bola bolo boli budu bude sa s so pri pre z zo na v od do rok roku rokoch " "praca prace pracu projekt projekty dokument dokumenty nazov nazvy nazv " "tag tagy klucove slova slovo temu temy kategoriu kategorie " "uved uvedte vypis vyber najdi zisti porovnaj popis opis čomu komu vymenuj pomenuj " @@ -111,6 +113,9 @@ def _generic(word: str) -> bool: "timov", "zaverecn", "dokument", + "praci", + "suvisi", + "patriac", "osob", "student", "kandidat",