"""Verify the running API and graph library against a frozen release.""" import argparse import json import os import platform from datetime import datetime, timezone from pathlib import Path from importlib.metadata import version as package_version from tools.rag.evaluation.rag_runner import request_json from tools.graphrag.graph_db import create_neo4j_driver from tools.graphrag.graph_search import search_graph from tools.graphrag.build_graphrag import graph_version from tools.graphrag.evaluation.reproducibility import ( graph_fingerprint, run_metadata, verify_freeze, sha256_file, ) from tools.sync.index_state import read_version, sqlite_version from tools.rag.common import DB_FILE def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", type=Path) parser.add_argument("--questions", type=Path, required=True) parser.add_argument("--audit", type=Path, required=True) parser.add_argument("--freeze-manifest", type=Path, required=True) parser.add_argument("--release-manifest", type=Path) args = parser.parse_args() from tools.runtime.report_files import report_output args.output = report_output(args.output, "deployment") base = Path("tools/graphrag/evaluation") release_path = args.release_manifest release_match = None if release_path is not None: release = json.loads(release_path.read_text()) for path, digest in release["runtime_files"].items(): assert sha256_file(Path(path)) == digest, f"Release code mismatch: {path}" assert sha256_file(args.freeze_manifest) == release["graph_freeze_sha256"] release_match = True freeze = json.loads(args.freeze_manifest.read_text()) health = request_json("http://localhost:8000/health") assert health["ready"] and all(v["ready"] for v in health["tools"].values()) with create_neo4j_driver() as driver, driver.session() as session: graph = graph_fingerprint(session) verify_freeze( freeze, run_metadata(args.questions), graph, audit=args.audit, **freeze["parameters"] ) current = read_version(Path("data")) assert ( current == sqlite_version(DB_FILE) == graph_version(session) == health["index_version"] ) documents = session.run( 'MATCH (d:Document {graph_scope:"zpwiki"}) RETURN d.path AS path, d.published AS published' ).data() headers = {"X-API-Key": os.environ["SEARCH_API_KEY"], "Content-Type": "application/json"} rag = request_json( "http://localhost:8000/rag", method="POST", headers=headers, payload={"query": "Aký je názov práce osoby Ján Holp a rok práce?", "limit": 5}, timeout=120, ) assert rag["status"] == "ok" and rag["sources"] and rag["index_version"] == current coverage = json.loads(rag["context"].splitlines()[1]) assert coverage == { k: rag[k] for k in ["complete", "truncated", "truncation_reasons", "index_version"] } cases = {} for id, question, expected in [ ("known_title", "Aký je názov práce osoby Ján Holp a rok práce?", "ok"), ( "generic_label_not_title", "Uveď názov práce osoby Dárius Lindvai v roku 2019 s kľúčovým slovom python.", "no_answer", ), ("unknown_person", "Akú prácu má osoba Jozef Neznámy Overovací?", "no_answer"), ("outside_century", "Aký je názov práce osoby Ján Holp v roku 1999?", "no_answer"), ("unsupported_year", "Aký je názov práce osoby Ján Holp v roku 10000?", "no_answer"), ("unknown_start", "Aký je rok začiatku štúdia osoby Tomáš Kuchárik?", "no_answer"), ]: result = search_graph(question) assert result["status"] == expected and result["index_version"] == current cases[id] = { "status": result["status"], "reason": result["reason"], "complete": result["complete"], } hidden = {row["path"] for row in documents if row["published"] is False} raw_documents = json.loads(Path("data/documents.json").read_text()) missing = sum(row["published"] is None for row in raw_documents) assert len(hidden) == 1 and missing == 16 assert {row["path"]: row["published"] for row in documents} == { row["path"]: row["published"] is not False for row in raw_documents } hidden_rag = request_json( "http://localhost:8000/rag", method="POST", headers=headers, payload={"query": "Tvorba nezodpovedateľných otázok", "limit": 5}, timeout=120, ) assert not hidden & {s["document_path"] for s in hidden_rag["sources"]} result = { "checked_at_utc": datetime.now(timezone.utc).isoformat(), "health": health, "index_version": current, "frozen_implementation_inputs_and_graph_match": True, "graph": graph, "graph_cases": cases, "rag_live_query": "ok", "release_code_match": release_match, "rag_context_metadata_matches_contract": True, "publication": { "documents": len(documents), "explicit_false": len(hidden), "missing_included": missing, "visible_by_policy": len(documents) - len(hidden), "hidden_document_absent_from_rag": True, }, "runtime": { "python": platform.python_version(), **{ p: package_version(p) for p in ["fastapi", "pydantic", "neo4j", "sentence-transformers", "torch"] }, }, "scope": "RAG HTTP API plus GraphRAG library/CLI; native OpenWebUI graph-tool routing is point 4.", } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") print(json.dumps({"ready": True, "version": current["version"], "graph_cases": cases})) if __name__ == "__main__": main()