import json from pathlib import Path import pytest from tools.graphrag.graphrag import find_work_heading, extract_work_title from tools.graphrag.evaluation.source_oracle import academic_type from tools.graphrag.evaluation.source_validation import validate_returned_facts, claim_issues from tools.graphrag.evaluation.graphrag_evaluator import evaluate_result from tools.graphrag.evaluation.reproducibility import content_fingerprints from tools.answers.answer_format import parse_answer @pytest.mark.parametrize( "heading", ["Príprava na Diplomový projekt 2 2021", "Príprava k bakalárskej práci 2024"] ) def test_preparation_is_not_academic_work(heading): assert find_work_heading(heading) is None and academic_type(heading) is None def test_assignment_proposal_does_not_establish_an_explicit_title(): assert ( extract_work_title( ["Návrh na zadanie BP:\n\nDialógový systém pre otázky\n\n1. Vypracujte prehľad."] ) is None ) assert ( extract_work_title(["Názov práce:\n\nDialógový systém pre otázky"]) == "Dialógový systém pre otázky" ) def test_consultation_sentence_at_chunk_boundary_is_not_a_project_title(): assert ( extract_work_title( ["Odporúčaný čas konzultácie je piatok.\n\nMožné backendy:\n\n- OpenNMT"] ) is None ) @pytest.fixture def evidence(): oracle = { "people": { "Alena Nová": { "start_year": 2020, "documents": ["a.md"], "academics": [ { "year": 2024, "title": "Práca A", "academic_kind": "Thesis", "academic_type": "bachelor_thesis", "document_path": "a.md", } ], } }, "documents": { "a.md": { "source_url": "https://example.test/a", "tags": ["RAG"], "categories": ["bp2024"], "author": "Autor", } }, } fact = { "fact_type": "person_thesis", "person": "Alena Nová", "year": 2024, "title": "Práca A", "thesis_id": "thesis-a", "thesis_type": "bachelor_thesis", "document_path": "a.md", "source_url": "https://example.test/a", } question = { "graph": { "schema_version": 3, "expected_bindings": [{"person": "Alena Nová", "year": 2024}], } } return oracle, fact, question def test_explicit_incomplete_result_fails_even_without_truncation(evidence): oracle, fact, q = evidence result = {"status": "ok", "facts": [fact], "complete": False, "truncated": False} assert not evaluate_result(q, result, latency_seconds=0, source_oracle=oracle)["strict"] def test_wrong_extra_year_fails_even_when_required_fact_is_present(evidence): oracle, fact, q = evidence result = {"status": "ok", "facts": [fact], "complete": True} assert evaluate_result(q, result, latency_seconds=0, source_oracle=oracle)["strict"] result["facts"].append({**fact, "year": 1999, "thesis_id": "unrelated-wrong-extra"}) grade = evaluate_result(q, result, latency_seconds=0, source_oracle=oracle) assert grade["bindings_ok"] and not grade["strict"] assert grade["source_checked"] and not grade["all_facts_supported"] @pytest.mark.parametrize( "change", [ {"title": "Vymyslený názov"}, {"thesis_type": "doctoral_thesis"}, {"document_path": "other.md"}, {"source_url": "https://wrong.test"}, ], ) def test_every_extra_claim_has_correct_source_and_properties(evidence, change): oracle, fact, _ = evidence assert not validate_returned_facts({"facts": [{**fact, **change}]}, oracle)[ "all_facts_supported" ] def test_shared_metadata_validated_for_both_people(evidence): oracle, _, _ = evidence row = { "fact_type": "shared_keyword", "left_person": "Alena Nová", "right_person": "Unknown", "keyword": "RAG", "left_document_path": "a.md", "right_document_path": "a.md", } assert not validate_returned_facts({"facts": [row]}, oracle)["all_facts_supported"] def test_source_checker_rejects_unknown_fact_type(evidence): oracle, _, _ = evidence assert not validate_returned_facts({"facts": [{"year": 1999}]}, oracle)["all_facts_supported"] def test_metadata_from_different_documents_cannot_form_one_document_claim(evidence): oracle, _, _ = evidence oracle["people"]["Alena Nová"]["documents"].append("child.md") oracle["documents"]["child.md"] = { "categories": [], "tags": [], "author": "Iný autor", "source_url": "https://example.test/child", } assert claim_issues( {"person": "Alena Nová", "category": "bp2024", "document_author": "Iný autor"}, oracle ) def test_shared_comparison_years_cannot_contain_unbacked_extra_year(evidence): oracle, _, _ = evidence row = { "fact_type": "academic_year_comparison", "people": ["Alena Nová"], "shared_years": [2024, 1999], "same_academic_year": True, } assert not validate_returned_facts({"facts": [row]}, oracle)["all_facts_supported"] def test_comparison_checks_individual_years_as_well_as_shared_year(evidence): oracle, _, _ = evidence row = { "fact_type": "start_year_comparison", "people": ["Alena Nová"], "shared_start_year": 2020, "same_start_year": True, "start_years": {"Alena Nová": 1999}, } assert not validate_returned_facts({"facts": [row]}, oracle)["all_facts_supported"] row["start_years"]["Alena Nová"] = 2020 assert validate_returned_facts({"facts": [row]}, oracle)["all_facts_supported"] def test_freeze_includes_shared_modules_and_effective_publication_policy(): questions = Path("experiments/graphrag/graphrag_questions_v4.json") fp = content_fingerprints(questions) assert { "tools/retrieval/publication.py", "tools/retrieval/retrieval_contract.py", "tools/sync/index_state.py", } <= fp["implementation"].keys() assert "deployment/requirements.txt" in fp["implementation"] assert fp["retrieval_policy"]["published_only"] is True assert fp["retrieval_policy"]["academic_preparation_is_work"] is False def test_only_one_complete_json_fence_is_unwrapped(): answer = { "answer": "Správny text", "status": "ok", "claims": [], "source_urls": [], "count": None, } text = "Sprievodný text\n```json\n" + json.dumps(answer) + "\n```" parsed, normalized = parse_answer(text) assert parsed["answer"] == "Správny text" and normalized with pytest.raises(ValueError): parse_answer(text + "\n```json\n{}\n```") @pytest.mark.parametrize( "text", [ '{"answer":"zlá "citácia""}', '{"answer":"test","status":"ok","claims":[],"source_urls":[],"count":"2"}', ], ) def test_malformed_or_wrongly_typed_json_is_not_silently_repaired(text): with pytest.raises(ValueError): parse_answer(text)