"""Audit the real checked RAG path on a predeclared, known regression sample.

Default mode only retrieves locally. `--mode deliver` explicitly invokes the
configured backend model. Expected answers are used here, never sent to /answer.
"""

from __future__ import annotations

import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
import hashlib
import json
import os
from pathlib import Path
import time
import urllib.request

from dotenv import load_dotenv

from tools.answers.context_policy import normalized, citation_check, allowed_urls
from tools.answers.rag_evidence import rag_evidence_context
from tools.answers.verified_rag import rag_answer_issues, render_rag_answer, quoted_reference_urls
from tools.graphrag.evaluation.source_oracle import DEFAULT_WIKI, load_source_oracle
from tools.rag.evaluation.rag_metrics import expected_phrase_matches
from tools.runtime.report_files import report_output, resolve_report_input

ROOT = Path(__file__).resolve().parents[2]
DATASET = ROOT / "experiments/rag/datasets/verified_rag_regression.json"
FIELDS = ["person", "work_type", "title", "year", "start_year"]


def digest(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def binding_matches(expected: dict, rows: list[dict]) -> bool:
    # Study start can be a separate fact about the same person; the academic
    # fields must all occur together in one actual academic row.
    academic = {k: expected[k] for k in FIELDS if k in expected and k != "start_year"}
    academic_ok = any(
        all(normalized(row.get(k)) == normalized(v) for k, v in academic.items()) for row in rows
    )
    start_ok = "start_year" not in expected or any(
        normalized(row.get("person")) == normalized(expected["person"])
        and row.get("start_year") == expected["start_year"]
        for row in rows
    )
    return academic_ok and start_ok


def source_audit(case: dict, oracle: dict) -> list[str]:
    issues = []
    for item in case["source_checks"]:
        if digest(DEFAULT_WIKI / item["document_path"]) != item["sha256"]:
            issues.append("changed_source_document")
    for binding in case["expected_bindings"]:
        person = oracle["people"].get(binding["person"], {})
        rows = [
            {"person": binding["person"], "work_type": a["academic_type"], **a}
            for a in person.get("academics", [])
            if a["document_path"] == binding["document_path"]
        ]
        rows.append({"person": binding["person"], "start_year": person.get("start_year")})
        if not binding_matches(binding, rows):
            issues.append("gold_binding_not_in_original_markdown")
        if "start_year" in binding and person.get("start_year_basis") != "explicit_text":
            issues.append("study_start_not_explicit")
    return issues


def evaluate_case(case: dict, payload: dict, delivered: dict | None = None) -> dict:
    context = rag_evidence_context(case["question"], payload)
    source_urls = allowed_urls(payload)
    missing = [b for b in case["expected_bindings"] if not binding_matches(b, context["facts"])]
    retrieval = {
        "expected_sources_found": bool(source_urls & set(case["expected_source_urls"])),
        "missing_bindings": missing,
        "expected_values_in_context": [
            v
            for v in case["expected_answer_contains"]
            if expected_phrase_matches(v, payload.get("context", ""))
        ],
    }
    row = {
        "id": case["id"],
        "category": case["category"],
        "question": case["question"],
        "should_answer": case["should_answer"],
        "retrieval": retrieval,
        "typed_expected": bool(case["expected_bindings"]),
        "index_version": payload.get("index_version"),
    }
    if delivered is None:
        return row
    issues = []
    accepted = delivered.get("accepted_for_delivery") is True
    status = delivered.get("status")
    attempts = delivered.get("attempts", [])
    parsed = attempts[-1].get("parsed_answer") if attempts else None
    if accepted and parsed:
        issues.extend(rag_answer_issues(parsed, context))
        if delivered.get("answer") != render_rag_answer(parsed, context):
            issues.append("delivered_text_differs_from_verified_renderer")
        if delivered.get("claims", []) != parsed.get("claims", []):
            issues.append("delivered_claims_differ_from_verified_selection")
    references = quoted_reference_urls(parsed or {}, context)
    if accepted and not citation_check(delivered, source_urls | references)["citations_ok"]:
        issues.append("delivered_unsupported_citation")
    claims = delivered.get("claims", [])
    if accepted:
        # Validate every delivered claim against the actual retrieved tuple.
        # Additional supported projects of the same person are not a wrong
        # thesis year. Required benchmark tuples are checked separately below.
        issues.extend(
            rag_answer_issues(
                {
                    "status": status,
                    "claims": claims,
                    "source_urls": delivered.get("source_urls", []),
                    "evidence_quotes": (parsed or {}).get("evidence_quotes", []),
                },
                context,
            )
        )
    if case["should_answer"]:
        if not accepted:
            issues.append("delivery_blocked")
        elif status != "ok":
            issues.append("false_no_answer" if status == "no_answer" else "nonpositive_answer")
        if not all(binding_matches(b, claims) for b in case["expected_bindings"]):
            issues.append("missing_expected_binding")
        # Exclude sources from factual scoring: a name/year in a URL cannot
        # supply the requested fact. Quotes must cover the expected subject.
        text = delivered.get("answer", "").split("\nZdroje:", 1)[0]
        missing_values = [
            v for v in case["expected_answer_contains"] if not expected_phrase_matches(v, text)
        ]
        if missing_values:
            issues.append("missing_expected_answer_value")
        quotes = (parsed or {}).get("evidence_quotes", [])
        if quotes:
            content = "\n".join(q["quote"] for q in quotes)
            names = {s.get("title") for s in context["source_blocks"]}
            # Names in source headers are attribution, not answers to a factual
            # question. Value coverage checks relevance beyond literal copying.
            relevant = [v for v in case["expected_answer_contains"] if v not in names]
            row["quote_expected_value_coverage"] = {
                v: expected_phrase_matches(v, content) for v in relevant
            }
    elif not accepted or status != "no_answer":
        issues.append("incorrect_negative_answer")
    row.update(
        passed=not issues,
        issues=sorted(set(issues)),
        delivered=delivered,
        first_attempt_accepted=bool(attempts and not attempts[0].get("issues")),
    )
    return row


def summarize(rows: list[dict]) -> dict:
    counted = [r for r in rows if "passed" in r]
    total = len(counted)
    passed = sum(r["passed"] for r in counted)
    return {
        "total": len(rows),
        "answers_evaluated": total,
        "passed": passed,
        "success_percent": round(100 * passed / total, 2) if total else None,
        "false_no_answers": sum("false_no_answer" in r["issues"] for r in counted),
        "blocked_deliveries": sum("delivery_blocked" in r["issues"] for r in counted),
        "first_attempt_accepted": sum(r.get("first_attempt_accepted", False) for r in counted),
        "failure_reasons": dict(Counter(i for r in counted for i in r["issues"])),
        "source_coverage": sum(r["retrieval"]["expected_sources_found"] for r in rows),
        "answerable_source_coverage": {
            "passed": sum(
                r["retrieval"]["expected_sources_found"] for r in rows if r["should_answer"]
            ),
            "total": sum(r["should_answer"] for r in rows),
        },
        "typed_binding_coverage": {
            "passed": sum(
                not r["retrieval"]["missing_bindings"] for r in rows if r.get("typed_expected")
            ),
            "total": sum(r.get("typed_expected", False) for r in rows),
        },
        "categories": {
            c: {
                "total": sum(r["category"] == c for r in rows),
                "passed": sum(r.get("passed", False) for r in counted if r["category"] == c),
            }
            for c in sorted({r["category"] for r in rows})
        },
    }


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--mode", choices=["retrieve", "deliver", "replay"], default="retrieve")
    parser.add_argument("--input", type=Path, help="Saved report to regrade without model calls")
    parser.add_argument("--dataset", type=Path, default=DATASET)
    parser.add_argument("--backend", default="http://127.0.0.1:8000")
    parser.add_argument("--output", type=Path)
    parser.add_argument(
        "--no-save", action="store_true", help="Print results only; create no reports or logs"
    )
    args = parser.parse_args()
    load_dotenv(ROOT / ".env", override=False)
    dataset = json.loads(args.dataset.read_text())
    assert (
        digest(ROOT / dataset["source_file"]) == dataset["source_sha256"]
    ), "Historical baseline changed"
    oracle = load_source_oracle()
    audit = {c["id"]: source_audit(c, oracle) for c in dataset["cases"]}
    assert not any(audit.values()), "Expected answer/source audit failed"
    if args.no_save and args.output:
        parser.error("--no-save cannot be combined with --output")
    output = None if args.no_save else report_output(args.output, "rag_verified")
    files = [
        Path(__file__),
        ROOT / "tools/answers/answer_delivery.py",
        ROOT / "app/routes.py",
        *[
            (ROOT / "tools/answers" / name)
            for name in ["verified_rag.py", "rag_evidence.py", "context_policy.py"]
        ],
        ROOT / "tools/rag/rag_utils.py",
        ROOT / "tools/rag/llm_context.py",
    ]
    metadata = {
        "method": dataset["method"],
        "mode": args.mode,
        "dataset_sha256": digest(args.dataset),
        "limitations": dataset.get("limitations", []),
        "source_audit": audit,
        "source_corpus": oracle["source_corpus"],
        "code_hashes": {str(p.relative_to(ROOT)): digest(p) for p in files},
        "model_calls": args.mode == "deliver",
        "backend": args.backend,
        "model": os.getenv("ANSWER_MODEL", "model120-fast") if args.mode == "deliver" else None,
    }
    if output:
        output.with_suffix(".declaration.json").write_text(
            json.dumps(metadata, ensure_ascii=False, indent=2) + "\n"
        )
    key = os.getenv("SEARCH_API_KEY", "")
    saved = None
    if args.mode == "replay":
        if not args.input:
            parser.error("--input is required for replay")
        saved = {
            r["id"]: r for r in json.loads(resolve_report_input(args.input).read_text())["results"]
        }

    def run(case):
        started = time.monotonic()
        if saved is not None:
            old = saved[case["id"]]
            return evaluate_case(case, old["retrieval_context"], old.get("delivered")) | {
                "retrieval_context": old["retrieval_context"]
            }
        endpoint = "/answer" if args.mode == "deliver" else "/rag"
        body = {"query": case["question"], "limit": 5}
        if args.mode == "deliver":
            body.update(engine="rag", include_diagnostics=True)
        req = urllib.request.Request(
            args.backend.rstrip("/") + endpoint,
            data=json.dumps(body).encode(),
            headers={"Content-Type": "application/json", "X-API-Key": key},
        )
        try:
            with urllib.request.urlopen(req, timeout=300) as response:
                data = json.load(response)
            payload = data.pop("retrieval_context") if args.mode == "deliver" else data
            result = evaluate_case(case, payload, data if args.mode == "deliver" else None)
            result.update(retrieval_context=payload, seconds=round(time.monotonic() - started, 2))
            return result
        except Exception as error:
            # Count operational failures in the denominator; never retry only
            # selected failed questions or expose raw HTTP/token-bearing errors.
            return {
                "id": case["id"],
                "category": case["category"],
                "question": case["question"],
                "should_answer": case["should_answer"],
                "passed": False,
                "issues": ["request_or_evaluation_error"],
                "typed_expected": bool(case["expected_bindings"]),
                "error_type": type(error).__name__,
                "retrieval": {
                    "expected_sources_found": False,
                    "missing_bindings": case["expected_bindings"],
                },
            }

    rows = []
    with ThreadPoolExecutor(max_workers=2) as pool:
        futures = {pool.submit(run, case): case for case in dataset["cases"]}
        for future in as_completed(futures):
            rows.append(future.result())
            result = {
                "configuration": metadata,
                "summary": summarize(rows),
                "results": sorted(rows, key=lambda r: r["id"]),
            }
            if output:
                output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
            print(
                f'{len(rows)}/{len(dataset["cases"])} {rows[-1]["id"]}: {rows[-1].get("issues", "retrieved")}',
                flush=True,
            )
    assert metadata["code_hashes"] == {
        name: digest(ROOT / name) for name in metadata["code_hashes"]
    }, "Code changed during measurement"
    print(json.dumps(result["summary"], ensure_ascii=False))
    if output:
        print("Report: " + str(output))
    if args.no_save:
        print(
            json.dumps(
                {
                    "failures": [
                        {
                            "id": r["id"],
                            "issues": r["issues"],
                            "attempt_issues": [
                                a.get("issues") for a in r.get("delivered", {}).get("attempts", [])
                            ],
                        }
                        for r in result["results"]
                        if not r["passed"]
                    ]
                },
                ensure_ascii=False,
            )
        )


if __name__ == "__main__":
    main()