dp-zp-agent/tools/evaluation/check_deployment.py

"""Verify the running API and graph library against a frozen release."""

import argparse
import json
import os
import platform
from datetime import datetime, timezone
from pathlib import Path
from importlib.metadata import version as package_version

from tools.rag.evaluation.rag_runner import request_json
from tools.graphrag.graph_db import create_neo4j_driver
from tools.graphrag.graph_search import search_graph
from tools.graphrag.build_graphrag import graph_version
from tools.graphrag.evaluation.reproducibility import (
    graph_fingerprint,
    run_metadata,
    verify_freeze,
    sha256_file,
)
from tools.sync.index_state import read_version, sqlite_version
from tools.rag.common import DB_FILE


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output", type=Path)
    parser.add_argument("--questions", type=Path, required=True)
    parser.add_argument("--audit", type=Path, required=True)
    parser.add_argument("--freeze-manifest", type=Path, required=True)
    parser.add_argument("--release-manifest", type=Path)
    args = parser.parse_args()
    from tools.runtime.report_files import report_output

    args.output = report_output(args.output, "deployment")
    base = Path("tools/graphrag/evaluation")
    release_path = args.release_manifest
    release_match = None
    if release_path is not None:
        release = json.loads(release_path.read_text())
        for path, digest in release["runtime_files"].items():
            assert sha256_file(Path(path)) == digest, f"Release code mismatch: {path}"
        assert sha256_file(args.freeze_manifest) == release["graph_freeze_sha256"]
        release_match = True
    freeze = json.loads(args.freeze_manifest.read_text())
    health = request_json("http://localhost:8000/health")
    assert health["ready"] and all(v["ready"] for v in health["tools"].values())
    with create_neo4j_driver() as driver, driver.session() as session:
        graph = graph_fingerprint(session)
        verify_freeze(
            freeze, run_metadata(args.questions), graph, audit=args.audit, **freeze["parameters"]
        )
        current = read_version(Path("data"))
        assert (
            current == sqlite_version(DB_FILE) == graph_version(session) == health["index_version"]
        )
        documents = session.run(
            'MATCH (d:Document {graph_scope:"zpwiki"}) RETURN d.path AS path, d.published AS published'
        ).data()
    headers = {"X-API-Key": os.environ["SEARCH_API_KEY"], "Content-Type": "application/json"}
    rag = request_json(
        "http://localhost:8000/rag",
        method="POST",
        headers=headers,
        payload={"query": "Aký je názov práce osoby Ján Holp a rok práce?", "limit": 5},
        timeout=120,
    )
    assert rag["status"] == "ok" and rag["sources"] and rag["index_version"] == current
    coverage = json.loads(rag["context"].splitlines()[1])
    assert coverage == {
        k: rag[k] for k in ["complete", "truncated", "truncation_reasons", "index_version"]
    }
    cases = {}
    for id, question, expected in [
        ("known_title", "Aký je názov práce osoby Ján Holp a rok práce?", "ok"),
        (
            "generic_label_not_title",
            "Uveď názov práce osoby Dárius Lindvai v roku 2019 s kľúčovým slovom python.",
            "no_answer",
        ),
        ("unknown_person", "Akú prácu má osoba Jozef Neznámy Overovací?", "no_answer"),
        ("outside_century", "Aký je názov práce osoby Ján Holp v roku 1999?", "no_answer"),
        ("unsupported_year", "Aký je názov práce osoby Ján Holp v roku 10000?", "no_answer"),
        ("unknown_start", "Aký je rok začiatku štúdia osoby Tomáš Kuchárik?", "no_answer"),
    ]:
        result = search_graph(question)
        assert result["status"] == expected and result["index_version"] == current
        cases[id] = {
            "status": result["status"],
            "reason": result["reason"],
            "complete": result["complete"],
        }
    hidden = {row["path"] for row in documents if row["published"] is False}
    raw_documents = json.loads(Path("data/documents.json").read_text())
    missing = sum(row["published"] is None for row in raw_documents)
    assert len(hidden) == 1 and missing == 16
    assert {row["path"]: row["published"] for row in documents} == {
        row["path"]: row["published"] is not False for row in raw_documents
    }
    hidden_rag = request_json(
        "http://localhost:8000/rag",
        method="POST",
        headers=headers,
        payload={"query": "Tvorba nezodpovedateľných otázok", "limit": 5},
        timeout=120,
    )
    assert not hidden & {s["document_path"] for s in hidden_rag["sources"]}
    result = {
        "checked_at_utc": datetime.now(timezone.utc).isoformat(),
        "health": health,
        "index_version": current,
        "frozen_implementation_inputs_and_graph_match": True,
        "graph": graph,
        "graph_cases": cases,
        "rag_live_query": "ok",
        "release_code_match": release_match,
        "rag_context_metadata_matches_contract": True,
        "publication": {
            "documents": len(documents),
            "explicit_false": len(hidden),
            "missing_included": missing,
            "visible_by_policy": len(documents) - len(hidden),
            "hidden_document_absent_from_rag": True,
        },
        "runtime": {
            "python": platform.python_version(),
            **{
                p: package_version(p)
                for p in ["fastapi", "pydantic", "neo4j", "sentence-transformers", "torch"]
            },
        },
        "scope": "RAG HTTP API plus GraphRAG library/CLI; native OpenWebUI graph-tool routing is point 4.",
    }
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
    print(json.dumps({"ready": True, "version": current["version"], "graph_cases": cases}))


if __name__ == "__main__":
    main()