90 lines
5.5 KiB
Python
90 lines
5.5 KiB
Python
"""Verify the running API and graph library against a frozen release."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import platform
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from importlib.metadata import version as package_version
|
|
|
|
from tools.rag.evaluation.rag_runner import request_json
|
|
from tools.graphrag.graph_db import create_neo4j_driver
|
|
from tools.graphrag.graph_search import search_graph
|
|
from tools.graphrag.build_graphrag import graph_version
|
|
from tools.graphrag.evaluation.reproducibility import graph_fingerprint, run_metadata, verify_freeze, sha256_file
|
|
from tools.index_state import read_version, sqlite_version
|
|
from tools.rag.common import DB_FILE
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--questions', type=Path, required=True)
|
|
parser.add_argument('--audit', type=Path, required=True)
|
|
parser.add_argument('--freeze-manifest', type=Path, required=True)
|
|
parser.add_argument('--release-manifest', type=Path)
|
|
args = parser.parse_args()
|
|
if args.output.exists(): parser.error('Refusing to overwrite deployment verification.')
|
|
base = Path('tools/graphrag/evaluation')
|
|
release_path = args.release_manifest
|
|
release_match = None
|
|
if release_path is not None:
|
|
release = json.loads(release_path.read_text())
|
|
for path, digest in release['runtime_files'].items():
|
|
assert sha256_file(Path(path)) == digest, f'Release code mismatch: {path}'
|
|
assert sha256_file(args.freeze_manifest) == release['graph_freeze_sha256']
|
|
release_match = True
|
|
freeze = json.loads(args.freeze_manifest.read_text())
|
|
health = request_json('http://localhost:8000/health')
|
|
assert health['ready'] and all(v['ready'] for v in health['tools'].values())
|
|
with create_neo4j_driver() as driver, driver.session() as session:
|
|
graph = graph_fingerprint(session)
|
|
verify_freeze(freeze, run_metadata(args.questions), graph,
|
|
audit=args.audit, **freeze['parameters'])
|
|
current = read_version(Path('data'))
|
|
assert current == sqlite_version(DB_FILE) == graph_version(session) == health['index_version']
|
|
documents = session.run('MATCH (d:Document {graph_scope:"zpwiki"}) RETURN d.path AS path, d.published AS published').data()
|
|
headers = {'X-API-Key': os.environ['SEARCH_API_KEY'], 'Content-Type': 'application/json'}
|
|
rag = request_json('http://localhost:8000/rag', method='POST', headers=headers,
|
|
payload={'query': 'Aký je názov práce osoby Ján Holp a rok práce?', 'limit': 5}, timeout=120)
|
|
assert rag['status'] == 'ok' and rag['sources'] and rag['index_version'] == current
|
|
coverage = json.loads(rag['context'].splitlines()[1])
|
|
assert coverage == {k: rag[k] for k in ['complete', 'truncated', 'truncation_reasons', 'index_version']}
|
|
cases = {}
|
|
for id, question, expected in [
|
|
('known_title', 'Aký je názov práce osoby Ján Holp a rok práce?', 'ok'),
|
|
('generic_label_not_title', 'Uveď názov práce osoby Dárius Lindvai v roku 2019 s kľúčovým slovom python.', 'no_answer'),
|
|
('unknown_person', 'Akú prácu má osoba Jozef Neznámy Overovací?', 'no_answer'),
|
|
('outside_century', 'Aký je názov práce osoby Ján Holp v roku 1999?', 'no_answer'),
|
|
('unsupported_year', 'Aký je názov práce osoby Ján Holp v roku 10000?', 'no_answer'),
|
|
('unknown_start', 'Aký je rok začiatku štúdia osoby Tomáš Kuchárik?', 'no_answer')]:
|
|
result = search_graph(question)
|
|
assert result['status'] == expected and result['index_version'] == current
|
|
cases[id] = {'status': result['status'], 'reason': result['reason'], 'complete': result['complete']}
|
|
hidden = {row['path'] for row in documents if row['published'] is False}
|
|
raw_documents = json.loads(Path('data/documents.json').read_text())
|
|
missing = sum(row['published'] is None for row in raw_documents)
|
|
assert len(hidden) == 1 and missing == 16
|
|
assert {row['path']: row['published'] for row in documents} == {
|
|
row['path']: row['published'] is not False for row in raw_documents}
|
|
hidden_rag = request_json('http://localhost:8000/rag', method='POST', headers=headers,
|
|
payload={'query': 'Tvorba nezodpovedateľných otázok', 'limit': 5}, timeout=120)
|
|
assert not hidden & {s['document_path'] for s in hidden_rag['sources']}
|
|
result = {'checked_at_utc': datetime.now(timezone.utc).isoformat(), 'health': health,
|
|
'index_version': current, 'frozen_implementation_inputs_and_graph_match': True,
|
|
'graph': graph, 'graph_cases': cases, 'rag_live_query': 'ok',
|
|
'release_code_match': release_match,
|
|
'rag_context_metadata_matches_contract': True,
|
|
'publication': {'documents': len(documents), 'explicit_false': len(hidden),
|
|
'missing_included': missing, 'visible_by_policy': len(documents)-len(hidden),
|
|
'hidden_document_absent_from_rag': True},
|
|
'runtime': {'python': platform.python_version(), **{p: package_version(p) for p in
|
|
['fastapi', 'pydantic', 'neo4j', 'sentence-transformers', 'torch']}},
|
|
'scope': 'RAG HTTP API plus GraphRAG library/CLI; native OpenWebUI graph-tool routing is point 4.'}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
|
print(json.dumps({'ready': True, 'version': current['version'], 'graph_cases': cases}))
|
|
|
|
|
|
if __name__ == '__main__': main()
|