from __future__ import annotations

import argparse
import json
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any


PROJECT_ROOT = Path(__file__).resolve().parents[1]

if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(
        0,
        str(PROJECT_ROOT),
    )


from evaluation.rag_answer_data import (
    DEFAULT_DELAY,
    apply_question_overrides,
    questions_fingerprint,
    result_prefix,
    select_questions,
    validate_answer_questions,
    validate_string_list,
)
from evaluation.rag_answer_state import (
    PARTIAL_SCHEMA_VERSION,
    build_base_result,
    build_error_result,
    build_partial_payload,
    build_run_configuration,
    determine_run_status,
    extract_resume_state,
    load_partial_payload,
    ordered_results,
    prepare_output_state,
    print_summary,
    save_csv_results,
    save_json_results,
    validate_resume_compatibility,
)
from evaluation.rag_metrics import (
    evaluate_answer,
    summarize_results,
)
from evaluation.rag_runner import (
    DEFAULT_BACKOFF_BASE,
    DEFAULT_BACKOFF_MAX,
    DEFAULT_MAX_ATTEMPTS,
    DEFAULT_MODEL,
    DEFAULT_TIMEOUT,
    LOCAL_OPENAPI_URL,
    build_rag_tool,
    load_env_value,
    request_json,
    run_question,
)
from evaluation.retrieval_runner import (
    load_questions,
)


EVALUATION_DIR = (
    PROJECT_ROOT
    / "evaluation"
)

JSON_FILES_DIR = (
    EVALUATION_DIR
    / "json_files"
)

QUESTIONS_PATH = (
    JSON_FILES_DIR
    / "questions.json"
)

RAG_ANSWER_OVERRIDES_PATH = (
    JSON_FILES_DIR
    / "rag_answer_overrides.json"
)

RESULTS_DIR = (
    EVALUATION_DIR
    / "results"
)


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description=(
            "Answer-level RAG evaluácia "
            "cez OpenWebUI, Model120 "
            "a ZP Agent."
        )
    )

    parser.add_argument(
        "--split",
        choices=(
            "dev",
            "test",
            "all",
        ),
        default="dev",
        help=(
            "Časť datasetu. "
            "Predvolené je dev."
        ),
    )

    parser.add_argument(
        "--limit",
        type=int,
        default=None,
        help=(
            "Maximálny počet otázok "
            "na vyhodnotenie."
        ),
    )

    parser.add_argument(
        "--model",
        default=DEFAULT_MODEL,
        help=(
            "OpenWebUI model ID. "
            f"Predvolené: {DEFAULT_MODEL}."
        ),
    )

    parser.add_argument(
        "--questions",
        type=Path,
        default=QUESTIONS_PATH,
        help=(
            "Cesta k questions.json."
        ),
    )

    parser.add_argument(
        "--overrides",
        type=Path,
        default=RAG_ANSWER_OVERRIDES_PATH,
        help=(
            "Cesta k answer-level "
            "override súboru."
        ),
    )

    parser.add_argument(
        "--results-dir",
        type=Path,
        default=RESULTS_DIR,
        help=(
            "Adresár pre výsledky."
        ),
    )

    parser.add_argument(
        "--timeout",
        type=int,
        default=DEFAULT_TIMEOUT,
        help=(
            "HTTP timeout v sekundách. "
            f"Predvolené: {DEFAULT_TIMEOUT}."
        ),
    )

    parser.add_argument(
        "--max-attempts",
        type=int,
        default=DEFAULT_MAX_ATTEMPTS,
        help=(
            "Maximálny počet HTTP pokusov "
            "pre retryable chyby. "
            f"Predvolené: "
            f"{DEFAULT_MAX_ATTEMPTS}."
        ),
    )

    parser.add_argument(
        "--backoff-base",
        type=float,
        default=DEFAULT_BACKOFF_BASE,
        help=(
            "Počiatočný exponential "
            "backoff v sekundách. "
            f"Predvolené: "
            f"{DEFAULT_BACKOFF_BASE}."
        ),
    )

    parser.add_argument(
        "--backoff-max",
        type=float,
        default=DEFAULT_BACKOFF_MAX,
        help=(
            "Maximálny exponential "
            "backoff v sekundách. "
            f"Predvolené: "
            f"{DEFAULT_BACKOFF_MAX}."
        ),
    )

    parser.add_argument(
        "--delay",
        type=float,
        default=DEFAULT_DELAY,
        help=(
            "Pauza medzi otázkami "
            "v sekundách. "
            f"Predvolené: "
            f"{DEFAULT_DELAY}."
        ),
    )

    parser.add_argument(
        "--question-id",
        action="append",
        default=[],
        help=(
            "Vyhodnoť iba konkrétne ID. "
            "Parameter možno použiť "
            "opakovane."
        ),
    )

    parser.add_argument(
        "--fail-fast",
        action="store_true",
        help=(
            "Pri prvej chybe "
            "ukonči evaluáciu."
        ),
    )

    output_group = (
        parser.add_mutually_exclusive_group()
    )

    output_group.add_argument(
        "--resume",
        action="store_true",
        help=(
            "Pokračuj z kompatibilného "
            ".partial.json. Úspešné otázky "
            "sa preskočia a chybové sa "
            "vyhodnotia znova."
        ),
    )

    output_group.add_argument(
        "--overwrite",
        action="store_true",
        help=(
            "Začni nový run a povoľ "
            "prepísanie existujúcich "
            "výsledkov."
        ),
    )

    args = parser.parse_args()

    if (
        args.limit is not None
        and args.limit <= 0
    ):
        parser.error(
            "--limit musí byť > 0"
        )

    if args.timeout <= 0:
        parser.error(
            "--timeout musí byť > 0"
        )

    if args.max_attempts <= 0:
        parser.error(
            "--max-attempts musí byť > 0"
        )

    if args.backoff_base < 0:
        parser.error(
            "--backoff-base "
            "nesmie byť záporné"
        )

    if args.backoff_max < 0:
        parser.error(
            "--backoff-max "
            "nesmie byť záporné"
        )

    if args.delay < 0:
        parser.error(
            "--delay nesmie byť záporné"
        )

    return args


def main() -> int:
    args = parse_args()

    try:
        all_questions = (
            load_questions(
                args.questions
            )
        )

        (
            all_questions,
            applied_override_ids,
        ) = apply_question_overrides(
            all_questions,
            args.overrides,
        )

        questions = select_questions(
            all_questions,
            split=args.split,
            question_ids=(
                args.question_id
            ),
            limit=args.limit,
        )

        if not questions:
            raise ValueError(
                "Žiadne otázky "
                "nezodpovedajú filtru."
            )

        validate_answer_questions(
            questions
        )

    except (
        FileNotFoundError,
        ValueError,
        json.JSONDecodeError,
    ) as exc:
        print(
            f"ERROR: {exc}",
            file=sys.stderr,
        )

        return 2

    selected_ids = {
        str(
            question["id"]
        )
        for question in questions
    }

    selected_override_ids = [
        question_id
        for question_id
        in applied_override_ids
        if question_id
        in selected_ids
    ]

    prefix = result_prefix(
        split=args.split,
        limit=args.limit,
        question_ids=(
            args.question_id
        ),
    )

    json_path = (
        args.results_dir
        / f"{prefix}.json"
    )

    csv_path = (
        args.results_dir
        / f"{prefix}.csv"
    )

    partial_path = (
        args.results_dir
        / f"{prefix}.partial.json"
    )

    try:
        prepare_output_state(
            json_path=json_path,
            csv_path=csv_path,
            partial_path=(
                partial_path
            ),
            resume=args.resume,
            overwrite=args.overwrite,
        )

    except (
        FileNotFoundError,
        FileExistsError,
    ) as exc:
        print(
            f"ERROR: {exc}",
            file=sys.stderr,
        )

        return 2

    try:
        openwebui_api_key = (
            load_env_value(
                "OPENWEBUI_API_KEY",
                allow_env_file=False,
            )
        )

        search_api_key = (
            load_env_value(
                "SEARCH_API_KEY"
            )
        )

    except RuntimeError as exc:
        print(
            f"ERROR: {exc}",
            file=sys.stderr,
        )

        return 2

    try:
        openapi = request_json(
            LOCAL_OPENAPI_URL,
            timeout=args.timeout,
            max_attempts=(
                args.max_attempts
            ),
            backoff_base=(
                args.backoff_base
            ),
            backoff_max=(
                args.backoff_max
            ),
        )

        (
            operation_id,
            rag_tool,
        ) = build_rag_tool(
            openapi
        )

    except RuntimeError as exc:
        print(
            f"ERROR: {exc}",
            file=sys.stderr,
        )

        return 2

    configuration = (
        build_run_configuration(
            args=args,
            questions=questions,
            selected_override_ids=(
                selected_override_ids
            ),
            operation_id=(
                operation_id
            ),
        )
    )

    results_by_id: dict[
        str,
        dict[str, Any],
    ] = {}

    previous_failed_ids: list[
        str
    ] = []

    if args.resume:
        try:
            partial_payload = (
                load_partial_payload(
                    partial_path
                )
            )

            validate_resume_compatibility(
                partial_payload[
                    "configuration"
                ],
                configuration,
            )

            (
                results_by_id,
                previous_failed_ids,
            ) = extract_resume_state(
                partial_payload,
                selected_question_ids=(
                    selected_ids
                ),
            )

        except (
            FileNotFoundError,
            ValueError,
            json.JSONDecodeError,
        ) as exc:
            print(
                f"ERROR: {exc}",
                file=sys.stderr,
            )

            return 2

    pending_questions = [
        question
        for question in questions
        if str(
            question["id"]
        )
        not in results_by_id
    ]

    print()
    print(
        "RAG answer evaluation"
    )
    print(
        "=" * 60
    )
    print(
        f"Dataset: "
        f"{args.questions}"
    )
    print(
        f"Overrides: "
        f"{args.overrides}"
    )
    print(
        "Applied overrides: "
        f"{len(selected_override_ids)}"
    )

    if selected_override_ids:
        print(
            "Override IDs: "
            + ", ".join(
                selected_override_ids
            )
        )

    print(
        f"Model: "
        f"{args.model}"
    )
    print(
        f"Tool: "
        f"{operation_id}"
    )
    print(
        f"Split: "
        f"{args.split}"
    )
    print(
        f"Questions: "
        f"{len(questions)}"
    )
    print(
        "Retry: "
        f"{args.max_attempts} "
        "pokusy"
    )
    print(
        "Backoff: "
        f"{args.backoff_base}s "
        "→ max "
        f"{args.backoff_max}s"
    )
    print(
        f"Delay: "
        f"{args.delay}s"
    )

    if args.resume:
        print(
            "Resume: áno"
        )
        print(
            "  úspešne obnovené: "
            f"{len(results_by_id)}"
        )
        print(
            "  predchádzajúce chyby "
            "na retry: "
            f"{len(previous_failed_ids)}"
        )
        print(
            "  zostáva spustiť: "
            f"{len(pending_questions)}"
        )

    else:
        print(
            "Resume: nie"
        )

    print(
        "=" * 60
    )
    print()

    total = len(
        questions
    )

    pending_total = len(
        pending_questions
    )

    processed_pending = 0
    stopped_early = False

    for index, question in enumerate(
        questions,
        start=1,
    ):
        question_id = str(
            question["id"]
        )

        question_text = str(
            question["question"]
        )

        if (
            question_id
            in results_by_id
        ):
            print(
                f"[{index:04d}/"
                f"{total:04d}] "
                f"{question_id}: "
                "SKIP (resume)"
            )

            continue

        print(
            f"[{index:04d}/"
            f"{total:04d}] "
            f"{question_id}: "
            f"{question_text}"
        )

        base_result = (
            build_base_result(
                question=question,
                question_id=(
                    question_id
                ),
                question_text=(
                    question_text
                ),
                selected_override_ids=(
                    selected_override_ids
                ),
                requested_model=(
                    args.model
                ),
            )
        )

        try:
            run_result = (
                run_question(
                    question_text,
                    model=args.model,
                    operation_id=(
                        operation_id
                    ),
                    rag_tool=rag_tool,
                    openwebui_api_key=(
                        openwebui_api_key
                    ),
                    search_api_key=(
                        search_api_key
                    ),
                    timeout=args.timeout,
                    max_attempts=(
                        args.max_attempts
                    ),
                    backoff_base=(
                        args.backoff_base
                    ),
                    backoff_max=(
                        args.backoff_max
                    ),
                )
            )

            scores = evaluate_answer(
                question,
                run_result[
                    "answer"
                ],
                tool_called=bool(
                    run_result[
                        "tool_called"
                    ]
                ),
            )

            result = {
                **base_result,
                **run_result,
                **scores,
                "error": None,
            }

            print(
                "    tool:",
                result[
                    "tool_called"
                ],
                "| answer:",
                f"{result['answer_contains_score']:.2f}",
                "| source:",
                f"{result['source_url_score']:.2f}",
                "| strict:",
                result[
                    "strict_pass"
                ],
                "| latency:",
                f"{result['total_latency_seconds']:.2f}s",
            )

        except Exception as exc:
            result = (
                build_error_result(
                    base_result,
                    exc,
                )
            )

            print(
                "    ERROR:",
                result[
                    "error"
                ],
            )

            if args.fail_fast:
                stopped_early = True

        results_by_id[
            question_id
        ] = result

        processed_pending += 1

        current_results = (
            ordered_results(
                questions,
                results_by_id,
            )
        )

        partial_payload = (
            build_partial_payload(
                status="partial",
                configuration=(
                    configuration
                ),
                results=(
                    current_results
                ),
                expected_total=(
                    total
                ),
            )
        )

        save_json_results(
            partial_path,
            partial_payload,
        )

        if stopped_early:
            break

        if (
            args.delay > 0
            and processed_pending
            < pending_total
        ):
            time.sleep(
                args.delay
            )

    results = ordered_results(
        questions,
        results_by_id,
    )

    status = determine_run_status(
        results,
        expected_total=(
            total
        ),
    )

    summary = (
        summarize_results(
            results
        )
    )

    payload = {
        "generated_at": (
            datetime.now(
                timezone.utc
            ).isoformat()
        ),
        "status": status,
        "configuration": (
            configuration
        ),
        "summary": summary,
        "results": results,
    }

    save_json_results(
        json_path,
        payload,
    )

    save_csv_results(
        csv_path,
        results,
    )

    if status == "complete":
        if partial_path.exists():
            partial_path.unlink()

    else:
        save_json_results(
            partial_path,
            build_partial_payload(
                status=status,
                configuration=(
                    configuration
                ),
                results=results,
                expected_total=(
                    total
                ),
            ),
        )

    print_summary(
        summary
    )

    print(
        "Výsledky:"
    )
    print(
        f"  JSON: "
        f"{json_path}"
    )
    print(
        f"  CSV:  "
        f"{csv_path}"
    )

    if partial_path.exists():
        print(
            f"  Partial: "
            f"{partial_path}"
        )

    print(
        f"  Status: "
        f"{status}"
    )

    if summary[
        "errors"
    ]:
        return 1

    return 0


if __name__ == "__main__":
    raise SystemExit(
        main()
    )