155 lines
9.0 KiB
Python
155 lines
9.0 KiB
Python
"""Independent candidate recall, hard scope before top-k and verified activities."""
|
|
import json
|
|
import sqlite3
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from tools.rag import build_sqlite_index as indexer, search_core as core, search_utils as search
|
|
from tools.rag.evaluation.retrieval_runner import retrieve_all_modes
|
|
from tools.rag.search_scope import SearchScope
|
|
from tools.answers.question_plan import build_question_plan
|
|
|
|
EVA = 'pages/students/2022/eva_nova/README.md'
|
|
ADAM = 'pages/students/2022/adam_stary/README.md'
|
|
|
|
|
|
@pytest.fixture
|
|
def corpus(tmp_path, monkeypatch):
|
|
docs = [(EVA, 'Eva Nová'), (ADAM, 'Adam Starý'),
|
|
('pages/topics/literal/README.md', 'Lexikálny príklad'),
|
|
('pages/topics/semantic/README.md', 'Sémantický príklad'),
|
|
('pages/topics/hidden/README.md', 'Skrytý príklad')]
|
|
records = [
|
|
('eva-bp', EVA, 'Bakalárska práca 2020', 'Používala indexovanie dokumentov.', .5),
|
|
('eva-dp', EVA, 'Diplomová práca 2021', 'Používala embeddingy dokumentov.', .9),
|
|
('adam-bp', ADAM, 'Bakalárska práca 2020', 'Používal indexovanie dokumentov.', 1.),
|
|
('adam-dp', ADAM, 'Diplomová práca 2021', 'Používal embeddingy dokumentov.', .99),
|
|
('literal', docs[2][0], '', 'retrieval je lexikálny záchytný výraz.', .6),
|
|
('semantic', docs[3][0], '', 'Získavanie informácií z rozsiahlych zbierok textov.', .98),
|
|
('hidden', docs[4][0], '', 'retrieval tajné informácie.', 1.),
|
|
]
|
|
documents = [{'path': path, 'title': name, 'published': path != docs[4][0]} for path, name in docs]
|
|
chunks = []
|
|
vectors = {}
|
|
for number, (key, path, heading, text, similarity) in enumerate(records):
|
|
title = dict(docs)[path]
|
|
body = f'Dokument: {title}\n' + (f'Sekcia: {heading}\n' if heading else '') + text
|
|
vectors[key] = [similarity, (1 - similarity**2)**.5, 0., 0.]
|
|
chunks.append({'chunk_id': key, 'document_path': path, 'title': title,
|
|
'text': body, 'published': key != 'hidden', 'chunk_index': number,
|
|
'heading_paths': [[heading]] if heading else [], 'tags': [], 'categories': []})
|
|
monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.asarray([
|
|
vectors[next(row['chunk_id'] for row in chunks if row['text'] in text)] for text in texts], dtype=np.float32))
|
|
monkeypatch.setattr(core, 'embed_query', lambda *a, **kw: np.asarray([1., 0., 0., 0.], dtype=np.float32))
|
|
for name, value in [('documents', documents), ('chunks', chunks)]:
|
|
(tmp_path / (name+'.json')).write_text(json.dumps(value, ensure_ascii=False))
|
|
db = tmp_path/'zp_index.sqlite'
|
|
indexer.build_database(documents_file=tmp_path/'documents.json', chunks_file=tmp_path/'chunks.json', db_file=db)
|
|
return db
|
|
|
|
|
|
@pytest.mark.parametrize('query,strategy', [('retrieval','all_terms'), ('retrievalxyz','prefix_terms')])
|
|
def test_semantic_only_candidate_survives_successful_all_terms_and_rrf(corpus, query, strategy):
|
|
result = search.search_database(corpus, query, limit=10)
|
|
assert result['strategies'] == [strategy]
|
|
rows = {row['chunk_id']: row for row in result['results']}
|
|
assert rows['literal']['fts_rank'] == 1
|
|
assert rows['semantic']['fts_rank'] is None and rows['semantic']['vector_rank'] is not None
|
|
assert rows['semantic']['hybrid_score'] > 0 and 'hidden' not in rows
|
|
assert rows['semantic']['hybrid_score'] == pytest.approx(
|
|
core.VECTOR_RRF_WEIGHT / (core.RRF_K + rows['semantic']['vector_rank']), abs=1e-8)
|
|
# Fusion is the union, not an intersection, with one result per chunk.
|
|
assert len(rows) == len(result['results'])
|
|
|
|
|
|
def test_evaluator_uses_the_same_union_as_production(corpus):
|
|
result = retrieve_all_modes(corpus, 'retrieval', limit=10, published_only=True, max_per_document=0)
|
|
assert {r['chunk_id'] for r in result['fts']} == {'literal'}
|
|
assert 'semantic' in {r['chunk_id'] for r in result['vector']}
|
|
assert {r['chunk_id'] for r in result['hybrid']} == {
|
|
r['chunk_id'] for r in search.search_database(corpus, 'retrieval', limit=10, max_per_document=0)['results']}
|
|
|
|
|
|
def test_person_type_and_year_filter_before_both_candidate_budgets(corpus, monkeypatch):
|
|
monkeypatch.setattr(search, 'MIN_CANDIDATES', 1)
|
|
monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1)
|
|
value = search.search_database(corpus,
|
|
'Aké metódy používala Eva Nová pri bakalárskej práci v roku 2020?', limit=1)
|
|
assert [r['chunk_id'] for r in value['results']] == ['eva-bp']
|
|
assert value['results'][0]['vector_rank'] == 1 # Stronger wrong-person/year vectors were excluded first.
|
|
assert value['truncated'] is False
|
|
|
|
|
|
def test_wrong_year_has_no_semantic_fallback_to_another_work(corpus):
|
|
value = search.search_database(corpus, 'Akú bakalársku prácu mala Eva Nová v roku 1999?')
|
|
assert value['status'] == 'no_answer' and value['results'] == []
|
|
|
|
|
|
def test_unknown_person_never_runs_global_embedding_search(corpus, monkeypatch):
|
|
monkeypatch.setattr(search, 'run_vector_query', lambda *a, **kw: pytest.fail('Global search for unknown identity'))
|
|
value = search.search_database(corpus, 'Aké práce má Xaver Neexistujúci?')
|
|
assert value['status'] == 'no_answer' and value['reason'] == 'unknown_person'
|
|
|
|
|
|
def test_vector_cap_is_reported_even_when_fts_succeeds(corpus, monkeypatch):
|
|
monkeypatch.setattr(search, 'MIN_CANDIDATES', 1)
|
|
monkeypatch.setattr(search, 'DEFAULT_CANDIDATE_MULTIPLIER', 1)
|
|
value = search.search_database(corpus, 'retrieval', limit=1)
|
|
assert value['truncated'] and 'candidate_limit' in value['truncation_reasons']
|
|
|
|
|
|
def test_one_chunk_cannot_license_another_work_year_or_taxonomy(corpus):
|
|
bound = build_question_plan('Aký názov má bakalárska práca Evy Novej z roku 2020?',
|
|
[{'name': 'Eva Nová', 'kind': 'Person', 'source_document': EVA}])
|
|
scope = SearchScope(bound.spec)
|
|
row = {'document_path': EVA, 'text': '# Bakalárska práca 2020\nNázov: Správny názov.\n'
|
|
'# Diplomová práca 2021\nNázov: Nesprávny názov.', 'heading_paths': []}
|
|
projected = scope.project(row)
|
|
assert projected and 'Správny názov' in projected['text'] and 'Nesprávny názov' not in projected['text']
|
|
assert core.parse_heading_paths(dict(projected)) == [['Bakalárska práca 2020']]
|
|
assert scope.project({**row, 'text': 'Názov: Iba kategória bp2020.', 'categories': ['bp2020']}) is None
|
|
|
|
|
|
def test_study_year_is_not_a_work_year_filter(corpus):
|
|
value = search.search_database(corpus, 'Je pri osobe Eva Nová uvedený rok začiatku štúdia 2022?')
|
|
assert value['status'] == 'ok' and {r['document_path'] for r in value['results']} == {EVA}
|
|
bound = build_question_plan('Aký názov má bakalárska práca Evy Novej z roku 2020 a aký je rok začiatku štúdia 2022?',
|
|
[{'name': 'Eva Nová', 'kind': 'Person', 'source_document': EVA}])
|
|
assert bound.spec.works[0].years == [2020] and 'start_year' in bound.spec.fields
|
|
row = {'document_path': EVA, 'heading_paths': [],
|
|
'text': '*Rok začiatku štúdia*: 2022\n# Bakalárska práca 2020\nNázov: Overená práca.'}
|
|
assert 'Rok začiatku štúdia' in SearchScope(bound.spec).project(row)['text']
|
|
|
|
|
|
def test_verifying_a_year_keeps_the_evidence_needed_to_refute_it(corpus):
|
|
value = search.search_database(corpus, 'Je rok 1999 rokom bakalárskej práce Evy Novej?')
|
|
assert value['status'] == 'ok' and {r['chunk_id'] for r in value['results']} == {'eva-bp'}
|
|
|
|
|
|
@pytest.mark.parametrize('q',[
|
|
'Kto sa venuje práci s názvom „Získavanie informácií“?',
|
|
'Ktorý dokument alebo študent súvisí s témou indexovanie a osobou Eva Nová?',
|
|
'Pri osobe Eva Nová uveď názov práce patriacej do kategórie „bp2020“.',
|
|
'Pri osobe Eva Nová uveď názov práce patriacej do kategórie „bp2020“ a rok vyjadrený touto kategóriou.',
|
|
])
|
|
def test_query_grammar_is_not_an_unknown_person(corpus,q):
|
|
value = search.search_database(corpus,q)
|
|
assert value['status'] != 'no_answer' or value['reason'] != 'unknown_person'
|
|
|
|
|
|
@pytest.mark.parametrize('question,quote',[
|
|
('Kto anotoval pomenované entity?', 'Implementoval rozpoznávanie pomenovaných entít.'),
|
|
('Kto sa venoval rozpoznávaniu pomenovaných entít?', 'Anotoval pomenované entity v korpuse.'),
|
|
])
|
|
def test_related_tasks_are_rejected_by_the_final_rag_guard(question, quote):
|
|
from tools.answers.rag_evidence import rag_evidence_context
|
|
from tools.answers.verified_rag import rag_answer_issues
|
|
from tools.rag.rag_utils import build_context_text
|
|
sources = [{'source_id': 'S1', 'title': 'Príklad', 'document_path': 'pages/topics/example/README.md',
|
|
'source_url': 'https://example.test/source', 'text': quote}]
|
|
context = rag_evidence_context(question, {'sources': sources, 'context': build_context_text(sources)})
|
|
answer = {'status': 'ok', 'claims': [], 'answer': '', 'source_urls': ['https://example.test/source'],
|
|
'evidence_quotes': [{'source_id': 'S1', 'quote': quote}]}
|
|
assert 'unsupported_requested_activity' in rag_answer_issues(answer, context)
|