Preserve independent semantic candidates in hybrid RAG search
This commit is contained in:
parent
d43071e7e6
commit
16a3f92db5
@ -28,8 +28,8 @@ z rovnakého snapshotu a aktivujú spoločne. Pri chybe zostáva predchádzajúc
|
||||
## Vyhľadávanie
|
||||
|
||||
**RAG** kombinuje fulltextové vyhľadávanie BM25 so sémantickým vyhľadávaním
|
||||
pomocou embeddingov. Výsledky spája metódou RRF. Je vhodný na otázky
|
||||
na obsah, zadanie, metódy, priebeh a výsledky práce.
|
||||
pomocou embeddingov. Samostatné výsledky spája metódou RRF; osobu, typ a rok filtruje záväzne.
|
||||
Je vhodný na otázky na obsah, zadanie, metódy, priebeh a výsledky práce.
|
||||
|
||||
**GraphRAG** využíva znalostný graf osôb, dokumentov, prác, projektov,
|
||||
tém, tagov a kategórií. Umožňuje vyhľadávať vzťahy, spoločné vlastnosti
|
||||
|
||||
@ -138,7 +138,7 @@ def _deliver_answer(
|
||||
if engine == "rag":
|
||||
try:
|
||||
with index_snapshot(db_file.parent, db_file=db_file):
|
||||
retrieved = build_rag_context(db_file, question, limit=limit)
|
||||
retrieved = build_rag_context(db_file, question, limit=limit, question_plan=plan.spec)
|
||||
if retrieved.get("status") == "error":
|
||||
return retrieval_failure(engine)
|
||||
if failure := name_failure(retrieved, engine):
|
||||
|
||||
@ -60,7 +60,7 @@ class Period(Strict):
|
||||
return self
|
||||
|
||||
class Condition(Strict):
|
||||
kind: Literal['tag','category','document_author','quantity','exclusion','current_status','other']
|
||||
kind: Literal['tag','category','document_author','quantity','exclusion','current_status','year_verification','other']
|
||||
quote: str
|
||||
|
||||
class QuestionPlan(Strict):
|
||||
@ -71,7 +71,7 @@ class QuestionPlan(Strict):
|
||||
years: list[int] = Field(default_factory=list)
|
||||
period: Period | None = None
|
||||
conditions: list[Condition] = Field(default_factory=list)
|
||||
fields: list[Literal['title','year','work_type','keyword','category','document_author','supervisor','consultant','status','date']] = Field(default_factory=list)
|
||||
fields: list[Literal['title','year','start_year','work_type','keyword','category','document_author','supervisor','consultant','status','date']] = Field(default_factory=list)
|
||||
facets: list[Facet] = Field(default_factory=list)
|
||||
scope: Literal['across_works','single_work','selected_works'] = Field(default='across_works',description='Scope is PER PERSON. single_work means one work per student, including a comparison of two students. selected_works means multiple explicitly requested works for the same person. Preserve the baseline scope.')
|
||||
presentation: Literal['brief','overview','list','comparison','table','count'] = 'overview'
|
||||
@ -206,7 +206,25 @@ def direct_spec(question, resolution):
|
||||
conditions.extend(Condition(kind=kind,quote=m[0]) for m in re.finditer(pattern,n))
|
||||
if mode and mode!='metadata' and conditions:unsupported.append('unsupported_content_join')
|
||||
conditions.extend(Condition(kind='current_status',quote=m[0]) for m in re.finditer(r'\b(?:aktual\w*|teraz|dnes)\b',n))
|
||||
study_start = bool(re.search(r'\b(?:zaciat\w* stud\w*|zacal\w* stud\w*)\b', n))
|
||||
year_verification = bool(re.search(r'\b(?:je|bol)\s+(?:rok\s+)?[12]\d{3}\s+rokom\b', n))
|
||||
if study_start or year_verification:
|
||||
# A value to verify is not an asserted work-year restriction. Keep
|
||||
# the literal so an LLM cannot erase the verification.
|
||||
verification_values = {int(m[1]) for m in re.finditer(r'\b(?:je|bol)\s+(?:rok\s+)?([12]\d{3})\s+rokom\b', n)}
|
||||
study_values = {int(m[1]) for m in re.finditer(r'\b(?:zaciat\w* stud\w*|zacal\w* stud\w*)[^\d.;?]{0,35}([12]\d{3})\b', n)}
|
||||
if study_start and not study_values:
|
||||
if len(years) == 1:
|
||||
study_values = set(years)
|
||||
elif years and not year_verification:
|
||||
unsupported.append('ambiguous_time_roles')
|
||||
for work in works:
|
||||
work.years = [year for year in work.years if year not in study_values | verification_values]
|
||||
if year_verification:
|
||||
conditions.append(Condition(kind='year_verification', quote=q))
|
||||
fields=[field for field,pattern in [('title',r'\bnaz(?:ov|v)\w*'),('year',r'\b(?:ktoreho roku|ktorych rokoch|rok\w*)'),('work_type',r'\btyp\w*'),('keyword',r'\b(?:tag\w*|klucov\w*)'),('category',r'\bkategor\w*'),('document_author',r'\bautor\w* dokument\w*'),('supervisor',r'\b(?:vedie|veduc\w*|veden\w*)'),('consultant',r'\bkonzult(?:ant|oval)\w*'),('status',r'\b(?:ukoncil\w*|obhaj\w*|skoncil\w*)'),('date',r'\b(?:kedy|datum\w*)')] if re.search(pattern,n)]
|
||||
if study_start:
|
||||
fields.append('start_year')
|
||||
return QuestionPlan(fields=fields,question=q,intent=mode or 'metadata',people=people,works=works,years=years,period=period,
|
||||
conditions=conditions,facets=facets,scope=scope,presentation=presentation,unsupported=list(dict.fromkeys(unsupported)))
|
||||
|
||||
@ -268,6 +286,7 @@ def build_question_plan(question, catalog, *, complete=None, verify_complete=Non
|
||||
'period_needs_dates':'Pre toto obdobie uveď konkrétny interval dátumov; semester nebudem odhadovať.',
|
||||
'invalid_period':'Uveď platný dátum alebo interval dátumov.',
|
||||
'invalid_year':'Uveď rok štyrmi číslicami; túto podmienku nevynechám.',
|
||||
'ambiguous_time_roles':'Rozlíš, ktorý rok označuje začiatok štúdia a ktorý rok konkrétnej práce.',
|
||||
'unsupported_content_join':'Otázka spája obsah s metadátovou podmienkou, ktorú táto cesta nepodporuje. Podmienku nevynechám.'}
|
||||
message=messages.get(code) or constraint_message(code)
|
||||
return bound.model_copy(update={'reason':code if code in messages else 'unsupported_constraint','clarification':message})
|
||||
|
||||
@ -805,6 +805,8 @@ def rag_evidence_context(question: str, payload: dict) -> dict:
|
||||
requirements["years"] = (
|
||||
requested_years(question) if academic and not requirements["start_year"] else []
|
||||
)
|
||||
if plan and any(c.kind == 'year_verification' for c in plan.conditions):
|
||||
requirements['years'] = []
|
||||
requirements["unsupported_year"] = any(len(value) != 4 for value in stated_years)
|
||||
requirements["project_number"] = project_number(question)
|
||||
requirements["exact_work_label"] = selector.labels[0] if len(selector.labels) == 1 else None
|
||||
|
||||
@ -352,6 +352,9 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
||||
cited = set(answer.get("source_urls", []))
|
||||
blocks = {row["source_id"]: row for row in context["source_blocks"]}
|
||||
for quote in quotes:
|
||||
from tools.knowledge.query import activity_supported
|
||||
if not activity_supported(context['question'], {'text': quote['quote']}):
|
||||
issues.append('unsupported_requested_activity')
|
||||
source = blocks.get(quote["source_id"])
|
||||
if not source or " ".join(quote["quote"].split()) not in " ".join(
|
||||
source["full_text"].split()
|
||||
|
||||
@ -174,6 +174,12 @@ def activity_supported(question, row):
|
||||
named_entities = bool(
|
||||
re.search(r"\bpomenovan\w* entit\w*\b|\bnamed entit\w*\b|\bner\b", wording)
|
||||
)
|
||||
if named_entities and re.search(r'\brozpoznav\w*|\brozpoznava\w*|\bnamed entity recognition\b|\bner\b', wording):
|
||||
text = normalized(row['text'])
|
||||
if re.search(r'\banot\w*\b', text) and not re.search(
|
||||
r'\brozpoznav\w*|\brozpoznava\w*|\bnamed entity recognition\b', text
|
||||
):
|
||||
return False
|
||||
if re.search(r"\banot\w*\b", wording) and named_entities:
|
||||
text = normalized(row["text"])
|
||||
if not re.search(r"\banot\w*\b", text):
|
||||
|
||||
@ -224,6 +224,10 @@ def retrieve_all_modes(
|
||||
conn.execute("PRAGMA query_only = ON")
|
||||
|
||||
verify_search_schema(conn)
|
||||
from tools.rag.search_scope import prepare_search_scope
|
||||
scope = prepare_search_scope(conn, db_file, clean_query, published_only=published_only)
|
||||
if scope.failure:
|
||||
return {mode: [] for mode in EVALUATION_MODES}
|
||||
|
||||
# -------------------------
|
||||
# FTS5
|
||||
@ -242,6 +246,7 @@ def retrieve_all_modes(
|
||||
match_query,
|
||||
candidate_limit,
|
||||
published_only,
|
||||
eligible_chunk_ids=scope.eligible_ids,
|
||||
)
|
||||
|
||||
if not rows:
|
||||
@ -259,7 +264,7 @@ def retrieve_all_modes(
|
||||
fts_results = add_fts_metadata(
|
||||
conn,
|
||||
clean_query,
|
||||
fts_candidates,
|
||||
scope.project_rows(fts_candidates),
|
||||
)
|
||||
|
||||
# -------------------------
|
||||
@ -271,32 +276,21 @@ def retrieve_all_modes(
|
||||
clean_query,
|
||||
candidate_limit,
|
||||
published_only,
|
||||
eligible_chunk_ids=scope.eligible_ids,
|
||||
)
|
||||
|
||||
vector_results = add_vector_metadata(
|
||||
conn,
|
||||
vector_candidates,
|
||||
scope.project_rows(vector_candidates),
|
||||
)
|
||||
|
||||
# -------------------------
|
||||
# Hybrid
|
||||
# -------------------------
|
||||
|
||||
hybrid_vector_results = vector_results
|
||||
|
||||
if used_strategies and used_strategies[0] in {
|
||||
"all_terms",
|
||||
"prefix_terms",
|
||||
}:
|
||||
fts_chunk_ids = {item["chunk_id"] for item in fts_results}
|
||||
|
||||
hybrid_vector_results = [
|
||||
item for item in vector_results if item["chunk_id"] in fts_chunk_ids
|
||||
]
|
||||
|
||||
hybrid_results = fuse_hybrid_results(
|
||||
fts_results,
|
||||
hybrid_vector_results,
|
||||
vector_results,
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
@ -491,6 +491,7 @@ def build_source_text(result: dict[str, Any]) -> str:
|
||||
seen,
|
||||
label="ZAČIATOK RELEVANTNEJ SEKCIE",
|
||||
text=str(result.get("section_lead_text") or ""),
|
||||
section=result.get('heading_paths') or [],
|
||||
)
|
||||
|
||||
_append_unique_block(
|
||||
@ -498,6 +499,7 @@ def build_source_text(result: dict[str, Any]) -> str:
|
||||
seen,
|
||||
label="NAJRELEVANTNEJŠÍ NÁJDENÝ ÚSEK",
|
||||
text=primary,
|
||||
section=result.get('heading_paths') or [],
|
||||
)
|
||||
|
||||
if not blocks:
|
||||
@ -655,6 +657,7 @@ def build_rag_context(
|
||||
limit: int = 5,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
max_per_document: int = 1,
|
||||
question_plan=None,
|
||||
) -> dict[str, Any]:
|
||||
from tools.retrieval.person_names import resolve_person_names, sqlite_people
|
||||
|
||||
@ -681,7 +684,7 @@ def build_rag_context(
|
||||
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
|
||||
|
||||
unsupported = unsupported_constraint(query)
|
||||
if unsupported:
|
||||
if unsupported and question_plan is None:
|
||||
return {
|
||||
**result_fields(
|
||||
status="ambiguous", sources=[], reason="unsupported_constraint", complete=False
|
||||
@ -702,8 +705,16 @@ def build_rag_context(
|
||||
limit,
|
||||
published_only=published_only,
|
||||
max_per_document=max_per_document,
|
||||
**({'question_plan': question_plan} if question_plan is not None else {}),
|
||||
)
|
||||
|
||||
if response.get('status') == 'ambiguous':
|
||||
return {**{key: response.get(key) for key in
|
||||
['status','reason','sources','complete','truncated','truncation_reasons','index_version','clarification','name_resolution']},
|
||||
'query': original_query, 'engine': response['engine'], 'strategies': [],
|
||||
'source_count': 0, 'context': '', 'instructions': RAG_INSTRUCTIONS,
|
||||
'answer_format': ANSWER_FORMAT}
|
||||
|
||||
results = [dict(item) for item in response["results"]]
|
||||
|
||||
results = _expand_exact_document(
|
||||
@ -749,6 +760,20 @@ def build_rag_context(
|
||||
for row in results
|
||||
if row.get("document_path") not in profile_paths or row["document_path"] in identities
|
||||
]
|
||||
# Expansions are candidates too: they must not restore another work/year
|
||||
# that was excluded before the BM25/vector candidate budgets.
|
||||
if db_path.exists():
|
||||
from tools.answers.question_plan import build_question_plan, QuestionPlan
|
||||
from tools.rag.search_scope import SearchScope
|
||||
spec = (question_plan if isinstance(question_plan, QuestionPlan) else
|
||||
QuestionPlan.model_validate(question_plan)) if question_plan is not None else build_question_plan(query, catalog).spec
|
||||
# Render expansions once, then filter the combined evidence. Strip the
|
||||
# raw expansion texts so build_source cannot append excluded works again.
|
||||
text_fields = {'exact_document_text', 'family_evidence_blocks', 'query_evidence_blocks',
|
||||
'query_focus_text', 'query_evidence_text', 'section_lead_text'}
|
||||
results = SearchScope(spec).project_rows([
|
||||
{**{key: value for key, value in row.items() if key not in text_fields},
|
||||
'text': build_source_text(row)} for row in results])
|
||||
sources = [build_source(result, index) for index, result in enumerate(results, start=1)]
|
||||
context = build_context_text(sources)
|
||||
coverage = {
|
||||
|
||||
@ -307,6 +307,8 @@ def run_fts_query(
|
||||
match_query: str,
|
||||
candidate_limit: int,
|
||||
published_only: bool,
|
||||
*,
|
||||
eligible_chunk_ids: set[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
@ -341,6 +343,7 @@ def run_fts_query(
|
||||
? = 0
|
||||
OR COALESCE(chunks.published, 1) = 1
|
||||
)
|
||||
AND (? IS NULL OR chunks.chunk_id IN (SELECT value FROM json_each(?)))
|
||||
ORDER BY
|
||||
chunks_fts.rank ASC,
|
||||
chunks.id ASC
|
||||
@ -349,6 +352,8 @@ def run_fts_query(
|
||||
(
|
||||
match_query,
|
||||
1 if published_only else 0,
|
||||
None if eligible_chunk_ids is None else json.dumps(sorted(eligible_chunk_ids)),
|
||||
None if eligible_chunk_ids is None else json.dumps(sorted(eligible_chunk_ids)),
|
||||
candidate_limit,
|
||||
),
|
||||
).fetchall()
|
||||
@ -361,7 +366,11 @@ def run_vector_query(
|
||||
query: str,
|
||||
candidate_limit: int,
|
||||
published_only: bool,
|
||||
*,
|
||||
eligible_chunk_ids: set[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
if eligible_chunk_ids is not None and not eligible_chunk_ids:
|
||||
return []
|
||||
model_name, dimensions = embedding_index_info(conn)
|
||||
|
||||
query_vector = embed_query(
|
||||
@ -396,8 +405,11 @@ def run_vector_query(
|
||||
? = 0
|
||||
OR COALESCE(chunks.published, 1) = 1
|
||||
)
|
||||
AND (? IS NULL OR chunks.chunk_id IN (SELECT value FROM json_each(?)))
|
||||
""",
|
||||
(1 if published_only else 0,),
|
||||
(1 if published_only else 0,
|
||||
None if eligible_chunk_ids is None else json.dumps(sorted(eligible_chunk_ids)),
|
||||
None if eligible_chunk_ids is None else json.dumps(sorted(eligible_chunk_ids))),
|
||||
).fetchall()
|
||||
|
||||
candidates: list[dict[str, Any]] = []
|
||||
|
||||
91
tools/rag/search_scope.py
Normal file
91
tools/rag/search_scope.py
Normal file
@ -0,0 +1,91 @@
|
||||
"""Hard question filters, independent of lexical/semantic relevance scores."""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
|
||||
from tools.answers.question_plan import QuestionPlan, build_question_plan
|
||||
from tools.answers.rag_evidence import academic_windows_metadata
|
||||
from tools.answers.context_policy import normalized
|
||||
from tools.retrieval.person_names import sqlite_people
|
||||
from tools.retrieval.retrieval_contract import result_fields
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchScope:
|
||||
plan: QuestionPlan
|
||||
failure: dict | None = None
|
||||
eligible_ids: set[str] | None = None
|
||||
|
||||
def project(self, row: dict) -> dict | None:
|
||||
"""Clip mixed-work chunks; a date/name elsewhere is not a work binding."""
|
||||
students = [p for p in self.plan.people if p.role == 'student']
|
||||
path = row.get('document_path', '')
|
||||
owners = [p for p in students if p.source_document and (
|
||||
path == p.source_document or path.startswith(p.source_document.removesuffix('README.md')))]
|
||||
if students and not owners:
|
||||
return None
|
||||
authors = [p.name for p in self.plan.people if p.role == 'document_author']
|
||||
if authors and normalized(row.get('author') or '') not in {normalized(p) for p in authors}:
|
||||
return None
|
||||
constrained = any(w.work_type or w.years or w.labels or w.excluded_labels or
|
||||
w.numbers or w.excluded_numbers or w.unnumbered for w in self.plan.works)
|
||||
constrained |= bool(self.plan.period and self.plan.period.field == 'work_year')
|
||||
if not constrained:
|
||||
return dict(row)
|
||||
text = row.get('text') or ''
|
||||
windows = academic_windows_metadata(text)
|
||||
if not any(scope for scope, _, _ in windows):
|
||||
# Chunk ancestry can be the only explicit work heading. Never use
|
||||
# a profile directory or taxonomy year as an academic work year.
|
||||
paths = row.get('heading_paths')
|
||||
if paths is None:
|
||||
paths = json.loads(row.get('heading_paths_json') or '[]')
|
||||
first = paths[0] if paths else ''
|
||||
heading = ' > '.join(first) if isinstance(first, list) else str(first)
|
||||
body = '\n'.join(line for line in text.splitlines() if not line.startswith('Dokument:'))
|
||||
windows = academic_windows_metadata('Sekcia: ' + heading + '\n' + body)
|
||||
names = [p.name for p in owners] or [None]
|
||||
selected = [(scope, label, body) for scope, label, body in windows if scope and any(
|
||||
self.plan.matches_work(name, label, scope[0], scope[1]) for name in names)]
|
||||
if not selected:
|
||||
return None
|
||||
# Preserve scoped source headings for the evidence verifier, but omit
|
||||
# other works even if one chunk or an expansion spans several works.
|
||||
clipped = '\n\n'.join('Sekcia: ' + label + '\n' + body for _, label, body in selected)
|
||||
if 'start_year' in self.plan.fields:
|
||||
# Study metadata belongs to the profile, independently of work
|
||||
# sections. Keep the explicit line when both values were requested.
|
||||
study_lines = [line for scope, _, body in windows if scope is None
|
||||
for line in body.splitlines() if normalized(line).startswith('rok zaciatku studia ')]
|
||||
clipped = '\n'.join(study_lines) + '\n\n' + clipped if study_lines else clipped
|
||||
headings = [[label] for _, label, _ in selected]
|
||||
return {**row, 'text': clipped, 'text_length': len(clipped),
|
||||
'snippet': clipped[:320], 'heading_paths': headings,
|
||||
'heading_paths_json': json.dumps(headings, ensure_ascii=False)}
|
||||
|
||||
def project_rows(self, rows):
|
||||
return [value for row in rows if (value := self.project(row)) is not None]
|
||||
|
||||
|
||||
def prepare_search_scope(conn, db_file, query, *, published_only, question_plan=None):
|
||||
if question_plan is None:
|
||||
bound = build_question_plan(query, sqlite_people(db_file, published_only=published_only))
|
||||
plan = bound.spec
|
||||
if bound.clarification:
|
||||
status = 'no_answer' if bound.resolution.get('status') == 'no_answer' and bound.reason == bound.resolution.get('reason') else 'ambiguous'
|
||||
return SearchScope(plan, failure={
|
||||
**result_fields(status=status, sources=[], reason=bound.reason, complete=False),
|
||||
'engine': 'hybrid_fts5_embeddings', 'results': [], 'strategies': [],
|
||||
'clarification': bound.clarification, 'name_resolution': bound.resolution})
|
||||
else:
|
||||
plan = question_plan if isinstance(question_plan, QuestionPlan) else QuestionPlan.model_validate(question_plan)
|
||||
if normalized(plan.question) != normalized(query) or plan.unsupported:
|
||||
raise ValueError('Search plan does not match the supported question')
|
||||
scope = SearchScope(plan)
|
||||
if plan.people or any(w.work_type or w.years or w.labels or w.excluded_labels or
|
||||
w.numbers or w.excluded_numbers or w.unnumbered for w in plan.works) or plan.period:
|
||||
rows = conn.execute('SELECT * FROM chunks WHERE (?=0 OR COALESCE(published,1)=1)',
|
||||
(int(published_only),)).fetchall()
|
||||
scope.eligible_ids = {row['chunk_id'] for row in rows if scope.project(dict(row)) is not None}
|
||||
return scope
|
||||
@ -54,6 +54,8 @@ def search_database(
|
||||
limit: int = 10,
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
max_per_document: int = 3,
|
||||
*,
|
||||
question_plan=None,
|
||||
) -> dict[str, Any]:
|
||||
if limit < 1 or max_per_document < 0:
|
||||
raise ValueError("Limit musí byť kladný a max_per_document nezáporné.")
|
||||
@ -86,6 +88,11 @@ def search_database(
|
||||
conn.execute("PRAGMA query_only = ON")
|
||||
|
||||
verify_search_schema(conn)
|
||||
from tools.rag.search_scope import prepare_search_scope
|
||||
scope = prepare_search_scope(conn, db_file, clean_query,
|
||||
published_only=published_only, question_plan=question_plan)
|
||||
if scope.failure:
|
||||
return scope.failure
|
||||
|
||||
# -------------------------
|
||||
# FTS5 kandidáti
|
||||
@ -105,6 +112,7 @@ def search_database(
|
||||
match_query,
|
||||
candidate_limit + 1,
|
||||
published_only,
|
||||
eligible_chunk_ids=scope.eligible_ids,
|
||||
)
|
||||
|
||||
if not rows:
|
||||
@ -126,7 +134,7 @@ def search_database(
|
||||
fts_results = add_fts_metadata(
|
||||
conn,
|
||||
clean_query,
|
||||
fts_candidates,
|
||||
scope.project_rows(fts_candidates),
|
||||
)
|
||||
|
||||
# -------------------------
|
||||
@ -138,6 +146,7 @@ def search_database(
|
||||
clean_query,
|
||||
candidate_limit + 1,
|
||||
published_only,
|
||||
eligible_chunk_ids=scope.eligible_ids,
|
||||
)
|
||||
|
||||
vector_capped = len(vector_candidates) > candidate_limit
|
||||
@ -145,29 +154,9 @@ def search_database(
|
||||
|
||||
vector_results = add_vector_metadata(
|
||||
conn,
|
||||
vector_candidates,
|
||||
scope.project_rows(vector_candidates),
|
||||
)
|
||||
|
||||
# -------------------------
|
||||
# Presné FTS výsledky
|
||||
# -------------------------
|
||||
#
|
||||
# Pri all_terms alebo prefix_terms
|
||||
# embeddingy iba preradia chunky,
|
||||
# ktoré už našiel FTS.
|
||||
#
|
||||
# Tým sa zabráni pridávaniu
|
||||
# sémanticky podobných, ale
|
||||
# lexikálne nesúvisiacich výsledkov.
|
||||
|
||||
if used_strategies and used_strategies[0] in {
|
||||
"all_terms",
|
||||
"prefix_terms",
|
||||
}:
|
||||
fts_chunk_ids = {item["chunk_id"] for item in fts_results}
|
||||
|
||||
vector_results = [item for item in vector_results if item["chunk_id"] in fts_chunk_ids]
|
||||
|
||||
# -------------------------
|
||||
# Hybrid fusion
|
||||
# -------------------------
|
||||
@ -190,9 +179,7 @@ def search_database(
|
||||
reasons = []
|
||||
if fts_candidates and fts_capped:
|
||||
reasons.append("candidate_limit")
|
||||
if vector_capped and (
|
||||
not used_strategies or used_strategies[0] not in {"all_terms", "prefix_terms"}
|
||||
):
|
||||
if vector_capped:
|
||||
reasons.append("candidate_limit")
|
||||
eligible = diversify_results(hybrid_results, max(1, len(hybrid_results)), max_per_document)
|
||||
if len(eligible) < len(hybrid_results):
|
||||
|
||||
154
tools/rag/tests/test_hybrid_candidates.py
Normal file
154
tools/rag/tests/test_hybrid_candidates.py
Normal file
@ -0,0 +1,154 @@
|
||||
"""Independent candidate recall, hard scope before top-k and verified activities."""
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tools.rag import build_sqlite_index as indexer, search_core as core, search_utils as search
|
||||
from tools.rag.evaluation.retrieval_runner import retrieve_all_modes
|
||||
from tools.rag.search_scope import SearchScope
|
||||
from tools.answers.question_plan import build_question_plan
|
||||
|
||||
EVA = 'pages/students/2022/eva_nova/README.md'
|
||||
ADAM = 'pages/students/2022/adam_stary/README.md'
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def corpus(tmp_path, monkeypatch):
|
||||
docs = [(EVA, 'Eva Nová'), (ADAM, 'Adam Starý'),
|
||||
('pages/topics/literal/README.md', 'Lexikálny príklad'),
|
||||
('pages/topics/semantic/README.md', 'Sémantický príklad'),
|
||||
('pages/topics/hidden/README.md', 'Skrytý príklad')]
|
||||
records = [
|
||||
('eva-bp', EVA, 'Bakalárska práca 2020', 'Používala indexovanie dokumentov.', .5),
|
||||
('eva-dp', EVA, 'Diplomová práca 2021', 'Používala embeddingy dokumentov.', .9),
|
||||
('adam-bp', ADAM, 'Bakalárska práca 2020', 'Používal indexovanie dokumentov.', 1.),
|
||||
('adam-dp', ADAM, 'Diplomová práca 2021', 'Používal embeddingy dokumentov.', .99),
|
||||
('literal', docs[2][0], '', 'retrieval je lexikálny záchytný výraz.', .6),
|
||||
('semantic', docs[3][0], '', 'Získavanie informácií z rozsiahlych zbierok textov.', .98),
|
||||
('hidden', docs[4][0], '', 'retrieval tajné informácie.', 1.),
|
||||
]
|
||||
documents = [{'path': path, 'title': name, 'published': path != docs[4][0]} for path, name in docs]
|
||||
chunks = []
|
||||
vectors = {}
|
||||
for number, (key, path, heading, text, similarity) in enumerate(records):
|
||||
title = dict(docs)[path]
|
||||
body = f'Dokument: {title}\n' + (f'Sekcia: {heading}\n' if heading else '') + text
|
||||
vectors[key] = [similarity, (1 - similarity**2)**.5, 0., 0.]
|
||||
chunks.append({'chunk_id': key, 'document_path': path, 'title': title,
|
||||
'text': body, 'published': key != 'hidden', 'chunk_index': number,
|
||||
'heading_paths': [[heading]] if heading else [], 'tags': [], 'categories': []})
|
||||
monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.asarray([
|
||||
vectors[next(row['chunk_id'] for row in chunks if row['text'] in text)] for text in texts], dtype=np.float32))
|
||||
monkeypatch.setattr(core, 'embed_query', lambda *a, **kw: np.asarray([1., 0., 0., 0.], dtype=np.float32))
|
||||
for name, value in [('documents', documents), ('chunks', chunks)]:
|
||||
(tmp_path / (name+'.json')).write_text(json.dumps(value, ensure_ascii=False))
|
||||
db = tmp_path/'zp_index.sqlite'
|
||||
indexer.build_database(documents_file=tmp_path/'documents.json', chunks_file=tmp_path/'chunks.json', db_file=db)
|
||||
return db
|
||||
|
||||
|
||||
@pytest.mark.parametrize('query,strategy', [('retrieval','all_terms'), ('retrievalxyz','prefix_terms')])
|
||||
def test_semantic_only_candidate_survives_successful_all_terms_and_rrf(corpus, query, strategy):
|
||||
result = search.search_database(corpus, query, limit=10)
|
||||
assert result['strategies'] == [strategy]
|
||||
rows = {row['chunk_id']: row for row in result['results']}
|
||||
assert rows['literal']['fts_rank'] == 1
|
||||
assert rows['semantic']['fts_rank'] is None and rows['semantic']['vector_rank'] is not None
|
||||
assert rows['semantic']['hybrid_score'] > 0 and 'hidden' not in rows
|
||||
assert rows['semantic']['hybrid_score'] == pytest.approx(
|
||||
core.VECTOR_RRF_WEIGHT / (core.RRF_K + rows['semantic']['vector_rank']), abs=1e-8)
|
||||
# Fusion is the union, not an intersection, with one result per chunk.
|
||||
assert len(rows) == len(result['results'])
|
||||
|
||||
|
||||
def test_evaluator_uses_the_same_union_as_production(corpus):
|
||||
result = retrieve_all_modes(corpus, 'retrieval', limit=10, published_only=True, max_per_document=0)
|
||||
assert {r['chunk_id'] for r in result['fts']} == {'literal'}
|
||||
assert 'semantic' in {r['chunk_id'] for r in result['vector']}
|
||||
assert {r['chunk_id'] for r in result['hybrid']} == {
|
||||
r['chunk_id'] for r in search.search_database(corpus, 'retrieval', limit=10, max_per_document=0)['results']}
|
||||
|
||||
|
||||
def test_person_type_and_year_filter_before_both_candidate_budgets(corpus, monkeypatch):
|
||||
monkeypatch.setattr(search, 'MIN_CANDIDATES', 1)
|
||||
monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1)
|
||||
value = search.search_database(corpus,
|
||||
'Aké metódy používala Eva Nová pri bakalárskej práci v roku 2020?', limit=1)
|
||||
assert [r['chunk_id'] for r in value['results']] == ['eva-bp']
|
||||
assert value['results'][0]['vector_rank'] == 1 # Stronger wrong-person/year vectors were excluded first.
|
||||
assert value['truncated'] is False
|
||||
|
||||
|
||||
def test_wrong_year_has_no_semantic_fallback_to_another_work(corpus):
|
||||
value = search.search_database(corpus, 'Akú bakalársku prácu mala Eva Nová v roku 1999?')
|
||||
assert value['status'] == 'no_answer' and value['results'] == []
|
||||
|
||||
|
||||
def test_unknown_person_never_runs_global_embedding_search(corpus, monkeypatch):
|
||||
monkeypatch.setattr(search, 'run_vector_query', lambda *a, **kw: pytest.fail('Global search for unknown identity'))
|
||||
value = search.search_database(corpus, 'Aké práce má Xaver Neexistujúci?')
|
||||
assert value['status'] == 'no_answer' and value['reason'] == 'unknown_person'
|
||||
|
||||
|
||||
def test_vector_cap_is_reported_even_when_fts_succeeds(corpus, monkeypatch):
|
||||
monkeypatch.setattr(search, 'MIN_CANDIDATES', 1)
|
||||
monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1)
|
||||
value = search.search_database(corpus, 'retrieval', limit=1)
|
||||
assert value['truncated'] and 'candidate_limit' in value['truncation_reasons']
|
||||
|
||||
|
||||
def test_one_chunk_cannot_license_another_work_year_or_taxonomy(corpus):
|
||||
bound = build_question_plan('Aký názov má bakalárska práca Evy Novej z roku 2020?',
|
||||
[{'name': 'Eva Nová', 'kind': 'Person', 'source_document': EVA}])
|
||||
scope = SearchScope(bound.spec)
|
||||
row = {'document_path': EVA, 'text': '# Bakalárska práca 2020\nNázov: Správny názov.\n'
|
||||
'# Diplomová práca 2021\nNázov: Nesprávny názov.', 'heading_paths': []}
|
||||
projected = scope.project(row)
|
||||
assert projected and 'Správny názov' in projected['text'] and 'Nesprávny názov' not in projected['text']
|
||||
assert core.parse_heading_paths(dict(projected)) == [['Bakalárska práca 2020']]
|
||||
assert scope.project({**row, 'text': 'Názov: Iba kategória bp2020.', 'categories': ['bp2020']}) is None
|
||||
|
||||
|
||||
def test_study_year_is_not_a_work_year_filter(corpus):
|
||||
value = search.search_database(corpus, 'Je pri osobe Eva Nová uvedený rok začiatku štúdia 2022?')
|
||||
assert value['status'] == 'ok' and {r['document_path'] for r in value['results']} == {EVA}
|
||||
bound = build_question_plan('Aký názov má bakalárska práca Evy Novej z roku 2020 a aký je rok začiatku štúdia 2022?',
|
||||
[{'name': 'Eva Nová', 'kind': 'Person', 'source_document': EVA}])
|
||||
assert bound.spec.works[0].years == [2020] and 'start_year' in bound.spec.fields
|
||||
row = {'document_path': EVA, 'heading_paths': [],
|
||||
'text': '*Rok začiatku štúdia*: 2022\n# Bakalárska práca 2020\nNázov: Overená práca.'}
|
||||
assert 'Rok začiatku štúdia' in SearchScope(bound.spec).project(row)['text']
|
||||
|
||||
|
||||
def test_verifying_a_year_keeps_the_evidence_needed_to_refute_it(corpus):
|
||||
value = search.search_database(corpus, 'Je rok 1999 rokom bakalárskej práce Evy Novej?')
|
||||
assert value['status'] == 'ok' and {r['chunk_id'] for r in value['results']} == {'eva-bp'}
|
||||
|
||||
|
||||
@pytest.mark.parametrize('q',[
|
||||
'Kto sa venuje práci s názvom „Získavanie informácií“?',
|
||||
'Ktorý dokument alebo študent súvisí s témou indexovanie a osobou Eva Nová?',
|
||||
'Pri osobe Eva Nová uveď názov práce patriacej do kategórie „bp2020“.',
|
||||
'Pri osobe Eva Nová uveď názov práce patriacej do kategórie „bp2020“ a rok vyjadrený touto kategóriou.',
|
||||
])
|
||||
def test_query_grammar_is_not_an_unknown_person(corpus,q):
|
||||
value = search.search_database(corpus,q)
|
||||
assert value['status'] != 'no_answer' or value['reason'] != 'unknown_person'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('question,quote',[
|
||||
('Kto anotoval pomenované entity?', 'Implementoval rozpoznávanie pomenovaných entít.'),
|
||||
('Kto sa venoval rozpoznávaniu pomenovaných entít?', 'Anotoval pomenované entity v korpuse.'),
|
||||
])
|
||||
def test_related_tasks_are_rejected_by_the_final_rag_guard(question, quote):
|
||||
from tools.answers.rag_evidence import rag_evidence_context
|
||||
from tools.answers.verified_rag import rag_answer_issues
|
||||
from tools.rag.rag_utils import build_context_text
|
||||
sources = [{'source_id': 'S1', 'title': 'Príklad', 'document_path': 'pages/topics/example/README.md',
|
||||
'source_url': 'https://example.test/source', 'text': quote}]
|
||||
context = rag_evidence_context(question, {'sources': sources, 'context': build_context_text(sources)})
|
||||
answer = {'status': 'ok', 'claims': [], 'answer': '', 'source_urls': ['https://example.test/source'],
|
||||
'evidence_quotes': [{'source_id': 'S1', 'quote': quote}]}
|
||||
assert 'unsupported_requested_activity' in rag_answer_issues(answer, context)
|
||||
@ -30,7 +30,7 @@ def build_search_database(
|
||||
|
||||
documents = [
|
||||
{
|
||||
"path": "pages/students/jan_ptak/README.md",
|
||||
"path": "pages/students/2022/jan_ptak/README.md",
|
||||
"title": "Ján Pták",
|
||||
"author": "Daniel Hladek",
|
||||
"published": True,
|
||||
@ -38,7 +38,7 @@ def build_search_database(
|
||||
"metadata": {},
|
||||
},
|
||||
{
|
||||
"path": "pages/students/jan_holp/README.md",
|
||||
"path": "pages/students/2016/jan_holp/README.md",
|
||||
"title": "Ján Holp",
|
||||
"author": "Daniel Hladek",
|
||||
"published": True,
|
||||
@ -100,7 +100,7 @@ def build_search_database(
|
||||
chunks = [
|
||||
chunk(
|
||||
"jan-ptak::0",
|
||||
"pages/students/jan_ptak/README.md",
|
||||
"pages/students/2022/jan_ptak/README.md",
|
||||
"Ján Pták",
|
||||
"Dokument: Ján Pták. Agent pre manažment záverečných prác.",
|
||||
True,
|
||||
@ -109,7 +109,7 @@ def build_search_database(
|
||||
),
|
||||
chunk(
|
||||
"jan-holp::0",
|
||||
"pages/students/jan_holp/README.md",
|
||||
"pages/students/2016/jan_holp/README.md",
|
||||
"Ján Holp",
|
||||
"Dokument: Ján Holp. Získavanie informácií a PageRank.",
|
||||
True,
|
||||
@ -344,7 +344,7 @@ def test_public_result_format_has_no_internal_id_and_uses_boolean(
|
||||
assert result["published"] is True
|
||||
assert result["chunk_id"] == "jan-ptak::0"
|
||||
|
||||
assert result["source_url"].endswith("students/jan_ptak")
|
||||
assert result["source_url"].endswith("students/2022/jan_ptak")
|
||||
|
||||
|
||||
def test_empty_query_returns_empty_result(
|
||||
|
||||
@ -31,6 +31,8 @@ def name_token_variants(token: str) -> set[str]:
|
||||
"frantisek": ("frantiska", "frantiskovi", "frantiskom"),
|
||||
}
|
||||
variants.update(irregular.get(token, ()))
|
||||
if token.endswith(('ii', 'yi')):
|
||||
variants.add(token + 'ho')
|
||||
if token.endswith("ova"):
|
||||
variants.update(token[:-1] + s for s in ("ej", "u", "ou"))
|
||||
elif token.endswith("y"):
|
||||
@ -91,7 +93,7 @@ def _score(words: list[str], canonical: list[str]) -> int | None:
|
||||
|
||||
_STOP = set(
|
||||
"a aj ale alebo ak ako ake aky aku ktore ktori ktora ktory kto kolko ci co obaja obidvaja "
|
||||
"ma maju mal mali je su bol bola bolo boli budu bude sa s so pri pre z zo na v od do roku rokoch "
|
||||
"ma maju mal mali je su bol bola bolo boli budu bude sa s so pri pre z zo na v od do rok roku rokoch "
|
||||
"praca prace pracu projekt projekty dokument dokumenty nazov nazvy nazv "
|
||||
"tag tagy klucove slova slovo temu temy kategoriu kategorie "
|
||||
"uved uvedte vypis vyber najdi zisti porovnaj popis opis čomu komu vymenuj pomenuj "
|
||||
@ -111,6 +113,9 @@ def _generic(word: str) -> bool:
|
||||
"timov",
|
||||
"zaverecn",
|
||||
"dokument",
|
||||
"praci",
|
||||
"suvisi",
|
||||
"patriac",
|
||||
"osob",
|
||||
"student",
|
||||
"kandidat",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user