"""Verify the running API and graph library against a frozen release.""" import argparse import json import os import platform from datetime import datetime, timezone from pathlib import Path from importlib.metadata import version as package_version from tools.rag.evaluation.rag_runner import request_json from tools.graphrag.graph_db import create_neo4j_driver from tools.graphrag.graph_search import search_graph from tools.graphrag.build_graphrag import graph_version from tools.graphrag.evaluation.reproducibility import graph_fingerprint, run_metadata, verify_freeze, sha256_file from tools.index_state import read_version, sqlite_version from tools.rag.common import DB_FILE def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--questions', type=Path, required=True) parser.add_argument('--audit', type=Path, required=True) parser.add_argument('--freeze-manifest', type=Path, required=True) parser.add_argument('--release-manifest', type=Path) args = parser.parse_args() if args.output.exists(): parser.error('Refusing to overwrite deployment verification.') base = Path('tools/graphrag/evaluation') release_path = args.release_manifest release_match = None if release_path is not None: release = json.loads(release_path.read_text()) for path, digest in release['runtime_files'].items(): assert sha256_file(Path(path)) == digest, f'Release code mismatch: {path}' assert sha256_file(args.freeze_manifest) == release['graph_freeze_sha256'] release_match = True freeze = json.loads(args.freeze_manifest.read_text()) health = request_json('http://localhost:8000/health') assert health['ready'] and all(v['ready'] for v in health['tools'].values()) with create_neo4j_driver() as driver, driver.session() as session: graph = graph_fingerprint(session) verify_freeze(freeze, run_metadata(args.questions), graph, audit=args.audit, **freeze['parameters']) current = read_version(Path('data')) assert current == sqlite_version(DB_FILE) == graph_version(session) == health['index_version'] documents = session.run('MATCH (d:Document {graph_scope:"zpwiki"}) RETURN d.path AS path, d.published AS published').data() headers = {'X-API-Key': os.environ['SEARCH_API_KEY'], 'Content-Type': 'application/json'} rag = request_json('http://localhost:8000/rag', method='POST', headers=headers, payload={'query': 'Aký je názov práce osoby Ján Holp a rok práce?', 'limit': 5}, timeout=120) assert rag['status'] == 'ok' and rag['sources'] and rag['index_version'] == current coverage = json.loads(rag['context'].splitlines()[1]) assert coverage == {k: rag[k] for k in ['complete', 'truncated', 'truncation_reasons', 'index_version']} cases = {} for id, question, expected in [ ('known_title', 'Aký je názov práce osoby Ján Holp a rok práce?', 'ok'), ('generic_label_not_title', 'Uveď názov práce osoby Dárius Lindvai v roku 2019 s kľúčovým slovom python.', 'no_answer'), ('unknown_person', 'Akú prácu má osoba Jozef Neznámy Overovací?', 'no_answer'), ('outside_century', 'Aký je názov práce osoby Ján Holp v roku 1999?', 'no_answer'), ('unsupported_year', 'Aký je názov práce osoby Ján Holp v roku 10000?', 'no_answer'), ('unknown_start', 'Aký je rok začiatku štúdia osoby Tomáš Kuchárik?', 'no_answer')]: result = search_graph(question) assert result['status'] == expected and result['index_version'] == current cases[id] = {'status': result['status'], 'reason': result['reason'], 'complete': result['complete']} hidden = {row['path'] for row in documents if row['published'] is False} raw_documents = json.loads(Path('data/documents.json').read_text()) missing = sum(row['published'] is None for row in raw_documents) assert len(hidden) == 1 and missing == 16 assert {row['path']: row['published'] for row in documents} == { row['path']: row['published'] is not False for row in raw_documents} hidden_rag = request_json('http://localhost:8000/rag', method='POST', headers=headers, payload={'query': 'Tvorba nezodpovedateľných otázok', 'limit': 5}, timeout=120) assert not hidden & {s['document_path'] for s in hidden_rag['sources']} result = {'checked_at_utc': datetime.now(timezone.utc).isoformat(), 'health': health, 'index_version': current, 'frozen_implementation_inputs_and_graph_match': True, 'graph': graph, 'graph_cases': cases, 'rag_live_query': 'ok', 'release_code_match': release_match, 'rag_context_metadata_matches_contract': True, 'publication': {'documents': len(documents), 'explicit_false': len(hidden), 'missing_included': missing, 'visible_by_policy': len(documents)-len(hidden), 'hidden_document_absent_from_rag': True}, 'runtime': {'python': platform.python_version(), **{p: package_version(p) for p in ['fastapi', 'pydantic', 'neo4j', 'sentence-transformers', 'torch']}}, 'scope': 'RAG HTTP API plus GraphRAG library/CLI; native OpenWebUI graph-tool routing is point 4.'} args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') print(json.dumps({'ready': True, 'version': current['version'], 'graph_cases': cases})) if __name__ == '__main__': main()