"""Independent candidate recall, hard scope before top-k and verified activities.""" import json import sqlite3 import numpy as np import pytest from tools.rag import build_sqlite_index as indexer, search_core as core, search_utils as search from tools.rag.evaluation.retrieval_runner import retrieve_all_modes from tools.rag.search_scope import SearchScope from tools.answers.question_plan import build_question_plan EVA = 'pages/students/2022/eva_nova/README.md' ADAM = 'pages/students/2022/adam_stary/README.md' @pytest.fixture def corpus(tmp_path, monkeypatch): docs = [(EVA, 'Eva Nová'), (ADAM, 'Adam Starý'), ('pages/topics/literal/README.md', 'Lexikálny príklad'), ('pages/topics/semantic/README.md', 'Sémantický príklad'), ('pages/topics/hidden/README.md', 'Skrytý príklad')] records = [ ('eva-bp', EVA, 'Bakalárska práca 2020', 'Používala indexovanie dokumentov.', .5), ('eva-dp', EVA, 'Diplomová práca 2021', 'Používala embeddingy dokumentov.', .9), ('adam-bp', ADAM, 'Bakalárska práca 2020', 'Používal indexovanie dokumentov.', 1.), ('adam-dp', ADAM, 'Diplomová práca 2021', 'Používal embeddingy dokumentov.', .99), ('literal', docs[2][0], '', 'retrieval je lexikálny záchytný výraz.', .6), ('semantic', docs[3][0], '', 'Získavanie informácií z rozsiahlych zbierok textov.', .98), ('hidden', docs[4][0], '', 'retrieval tajné informácie.', 1.), ] documents = [{'path': path, 'title': name, 'published': path != docs[4][0]} for path, name in docs] chunks = [] vectors = {} for number, (key, path, heading, text, similarity) in enumerate(records): title = dict(docs)[path] body = f'Dokument: {title}\n' + (f'Sekcia: {heading}\n' if heading else '') + text vectors[key] = [similarity, (1 - similarity**2)**.5, 0., 0.] chunks.append({'chunk_id': key, 'document_path': path, 'title': title, 'text': body, 'published': key != 'hidden', 'chunk_index': number, 'heading_paths': [[heading]] if heading else [], 'tags': [], 'categories': []}) monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.asarray([ vectors[next(row['chunk_id'] for row in chunks if row['text'] in text)] for text in texts], dtype=np.float32)) monkeypatch.setattr(core, 'embed_query', lambda *a, **kw: np.asarray([1., 0., 0., 0.], dtype=np.float32)) for name, value in [('documents', documents), ('chunks', chunks)]: (tmp_path / (name+'.json')).write_text(json.dumps(value, ensure_ascii=False)) db = tmp_path/'zp_index.sqlite' indexer.build_database(documents_file=tmp_path/'documents.json', chunks_file=tmp_path/'chunks.json', db_file=db) return db @pytest.mark.parametrize('query,strategy', [('retrieval','all_terms'), ('retrievalxyz','prefix_terms')]) def test_semantic_only_candidate_survives_successful_all_terms_and_rrf(corpus, query, strategy): result = search.search_database(corpus, query, limit=10) assert result['strategies'] == [strategy] rows = {row['chunk_id']: row for row in result['results']} assert rows['literal']['fts_rank'] == 1 assert rows['semantic']['fts_rank'] is None and rows['semantic']['vector_rank'] is not None assert rows['semantic']['hybrid_score'] > 0 and 'hidden' not in rows assert rows['semantic']['hybrid_score'] == pytest.approx( core.VECTOR_RRF_WEIGHT / (core.RRF_K + rows['semantic']['vector_rank']), abs=1e-8) # Fusion is the union, not an intersection, with one result per chunk. assert len(rows) == len(result['results']) def test_evaluator_uses_the_same_union_as_production(corpus): result = retrieve_all_modes(corpus, 'retrieval', limit=10, published_only=True, max_per_document=0) assert {r['chunk_id'] for r in result['fts']} == {'literal'} assert 'semantic' in {r['chunk_id'] for r in result['vector']} assert {r['chunk_id'] for r in result['hybrid']} == { r['chunk_id'] for r in search.search_database(corpus, 'retrieval', limit=10, max_per_document=0)['results']} def test_person_type_and_year_filter_before_both_candidate_budgets(corpus, monkeypatch): monkeypatch.setattr(search, 'MIN_CANDIDATES', 1) monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1) value = search.search_database(corpus, 'Aké metódy používala Eva Nová pri bakalárskej práci v roku 2020?', limit=1) assert [r['chunk_id'] for r in value['results']] == ['eva-bp'] assert value['results'][0]['vector_rank'] == 1 # Stronger wrong-person/year vectors were excluded first. assert value['truncated'] is False def test_wrong_year_has_no_semantic_fallback_to_another_work(corpus): value = search.search_database(corpus, 'Akú bakalársku prácu mala Eva Nová v roku 1999?') assert value['status'] == 'no_answer' and value['results'] == [] def test_unknown_person_never_runs_global_embedding_search(corpus, monkeypatch): monkeypatch.setattr(search, 'run_vector_query', lambda *a, **kw: pytest.fail('Global search for unknown identity')) value = search.search_database(corpus, 'Aké práce má Xaver Neexistujúci?') assert value['status'] == 'no_answer' and value['reason'] == 'unknown_person' def test_vector_cap_is_reported_even_when_fts_succeeds(corpus, monkeypatch): monkeypatch.setattr(search, 'MIN_CANDIDATES', 1) monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1) value = search.search_database(corpus, 'retrieval', limit=1) assert value['truncated'] and 'candidate_limit' in value['truncation_reasons'] def test_one_chunk_cannot_license_another_work_year_or_taxonomy(corpus): bound = build_question_plan('Aký názov má bakalárska práca Evy Novej z roku 2020?', [{'name': 'Eva Nová', 'kind': 'Person', 'source_document': EVA}]) scope = SearchScope(bound.spec) row = {'document_path': EVA, 'text': '# Bakalárska práca 2020\nNázov: Správny názov.\n' '# Diplomová práca 2021\nNázov: Nesprávny názov.', 'heading_paths': []} projected = scope.project(row) assert projected and 'Správny názov' in projected['text'] and 'Nesprávny názov' not in projected['text'] assert core.parse_heading_paths(dict(projected)) == [['Bakalárska práca 2020']] assert scope.project({**row, 'text': 'Názov: Iba kategória bp2020.', 'categories': ['bp2020']}) is None def test_study_year_is_not_a_work_year_filter(corpus): value = search.search_database(corpus, 'Je pri osobe Eva Nová uvedený rok začiatku štúdia 2022?') assert value['status'] == 'ok' and {r['document_path'] for r in value['results']} == {EVA} bound = build_question_plan('Aký názov má bakalárska práca Evy Novej z roku 2020 a aký je rok začiatku štúdia 2022?', [{'name': 'Eva Nová', 'kind': 'Person', 'source_document': EVA}]) assert bound.spec.works[0].years == [2020] and 'start_year' in bound.spec.fields row = {'document_path': EVA, 'heading_paths': [], 'text': '*Rok začiatku štúdia*: 2022\n# Bakalárska práca 2020\nNázov: Overená práca.'} assert 'Rok začiatku štúdia' in SearchScope(bound.spec).project(row)['text'] def test_verifying_a_year_keeps_the_evidence_needed_to_refute_it(corpus): value = search.search_database(corpus, 'Je rok 1999 rokom bakalárskej práce Evy Novej?') assert value['status'] == 'ok' and {r['chunk_id'] for r in value['results']} == {'eva-bp'} @pytest.mark.parametrize('q',[ 'Kto sa venuje práci s názvom „Získavanie informácií“?', 'Ktorý dokument alebo študent súvisí s témou indexovanie a osobou Eva Nová?', 'Pri osobe Eva Nová uveď názov práce patriacej do kategórie „bp2020“.', 'Pri osobe Eva Nová uveď názov práce patriacej do kategórie „bp2020“ a rok vyjadrený touto kategóriou.', ]) def test_query_grammar_is_not_an_unknown_person(corpus,q): value = search.search_database(corpus,q) assert value['status'] != 'no_answer' or value['reason'] != 'unknown_person' @pytest.mark.parametrize('question,quote',[ ('Kto anotoval pomenované entity?', 'Implementoval rozpoznávanie pomenovaných entít.'), ('Kto sa venoval rozpoznávaniu pomenovaných entít?', 'Anotoval pomenované entity v korpuse.'), ]) def test_related_tasks_are_rejected_by_the_final_rag_guard(question, quote): from tools.answers.rag_evidence import rag_evidence_context from tools.answers.verified_rag import rag_answer_issues from tools.rag.rag_utils import build_context_text sources = [{'source_id': 'S1', 'title': 'Príklad', 'document_path': 'pages/topics/example/README.md', 'source_url': 'https://example.test/source', 'text': quote}] context = rag_evidence_context(question, {'sources': sources, 'context': build_context_text(sources)}) answer = {'status': 'ok', 'claims': [], 'answer': '', 'source_urls': ['https://example.test/source'], 'evidence_quotes': [{'source_id': 'S1', 'quote': quote}]} assert 'unsupported_requested_activity' in rag_answer_issues(answer, context)