"""Declare a fresh query-group holdout using raw sources only, before retrieval.

All 1,042 previously exposed questions become DEV. No profile is new; this is
a new-combination holdout over a known corpus, not an unseen-person experiment.
"""

import copy
import json
import random
from collections import Counter

from .benchmark_v2 import (
    BASE,
    audit_dataset,
    make_case,
    validate_case,
    write_json,
    binding_source_evidence,
)
from .benchmark_v4 import build_pool, logical_group, refresh_titles
from .source_oracle import load_source_oracle, norm

QUOTAS = {
    "shared_keyword_pair": 10,
    "same_academic_year_pair": 10,
    "same_start_year_pair": 10,
    "person_title_year": 10,
    "person_keyword_title": 10,
    "document_author_category_multi_hop": 2,
    "person_multiple_categories": 7,
    "candidate_set_aggregation": 7,
    "conjunctive_disambiguation": 6,
    "negative_lookup": 8,
}
SEED = 20261026


def main():
    output = BASE / "graphrag_questions_v7.json"
    artifacts = BASE / "results/v7_preparation"
    if output.exists():
        raise RuntimeError("Refusing to overwrite declared questions.")
    oracle = load_source_oracle()
    old = json.loads((BASE / "graphrag_questions_v6.json").read_text())
    dev = []
    changes = []
    exposed = {logical_group(q) for q in old}
    for original in old:
        case = copy.deepcopy(original)
        case["split"] = "dev"
        names = [r["value"] for r in case["graph"]["start_entities"] if r["type"] == "person"]
        if any(
            b.get("start_year") is not None and oracle["people"][b["person"]]["start_year"] is None
            for b in case["graph"]["expected_bindings"]
        ):
            case = make_case(
                case["id"],
                "dev",
                case["category"],
                case["question"],
                names,
                oracle,
                [],
                no_answer=True,
                reason="The original Markdown does not explicitly document every requested study-start year.",
                negative_constraints={"missing_start_year": True},
            )
            changes.append(
                {
                    "id": case["id"],
                    "old_bindings": original["graph"]["expected_bindings"],
                    "new_status": "no_answer",
                    "reason": "Directory cohort is not a documented study-start year.",
                }
            )
        elif case["should_answer"]:
            case["source_evidence"] = [
                e
                for b in case["graph"]["expected_bindings"]
                for e in binding_source_evidence(b, oracle)
            ]
        case["graph"]["partition_group"] = logical_group(case)
        dev.append(case)
    # Fixed audit regressions; these are explicitly DEV, not extra easy TEST negatives.
    for year in [1999, 1899, 2100, 9999]:
        case = make_case(
            f"v7-dev-year-{year}",
            "dev",
            "negative_lookup",
            f"Aký je názov práce osoby Ján Holp v roku {year}?",
            ["Ján Holp"],
            oracle,
            [],
            no_answer=True,
            negative_constraints={"year": year},
        )
        case["graph"]["partition_group"] = logical_group(case)
        dev.append(case)
    exposed.update(logical_group(q) for q in dev)
    texts = {norm(q["question"]) for q in dev}
    rng = random.Random(SEED)
    holdout = []
    capacity = {}
    for category, pool in build_pool(oracle).items():
        unique = {}
        for candidate in pool:
            case = refresh_titles(candidate, oracle)
            group = logical_group(case)
            if group not in exposed and norm(case["question"]) not in texts:
                unique[group] = case
        capacity[category] = len(unique)
        candidates = [unique[key] for key in sorted(unique)]
        rng.shuffle(candidates)
        quota = QUOTAS[category]
        if len(candidates) < quota:
            raise RuntimeError(f"Insufficient source capacity: {category}")
        for case in candidates[:quota]:
            case["id"] = f"fresh7-{len(holdout)+1:03d}"
            case["split"] = "test"
            case["graph"]["partition_group"] = logical_group(case)
            holdout.append(case)
    questions = dev + holdout
    for case in questions:
        issues = validate_case(case, oracle)
        if issues:
            raise RuntimeError(f'{case["id"]}: {issues}')
    write_json(output, questions)
    audit = audit_dataset(questions, oracle, output)
    audit["partition_policy"] = {
        "method": "disjoint logical query groups",
        "shared_source_profiles": True,
        "seed": SEED,
        "previously_exposed_groups": sorted(exposed),
        "limitations": [
            "Known corpus, persons and template families; only the query/fact combinations are fresh.",
            "Two remaining document-author/category groups and seven category-list groups constrain category coverage.",
            "AI source annotation, not human annotation. No tuning on this TEST.",
        ],
    }
    write_json(artifacts / "source_audit.json", audit)
    write_json(
        artifacts / "source_corrections.json",
        {
            "questions": changes,
            "policy": "Only explicit study-start text is a fact. Cohort remains separate metadata; missing means unknown.",
        },
    )
    write_json(artifacts / "regression_ids.json", [q["id"] for q in old if q["split"] == "test"])
    ids = set(json.loads((BASE / "results/v6_preparation/generation_dev_ids.json").read_text()))
    ids.update(f"v7-dev-year-{year}" for year in [1999, 1899, 2100, 9999])
    write_json(artifacts / "generation_dev_ids.json", sorted(ids))
    write_json(
        artifacts / "protocol.json",
        {
            "seed": SEED,
            "dev_count": len(dev),
            "fresh_test_count": len(holdout),
            "test_categories": dict(Counter(q["category"] for q in holdout)),
            "test_positives": sum(q["should_answer"] for q in holdout),
            "test_negatives": sum(not q["should_answer"] for q in holdout),
            "available_groups": capacity,
            "old_gold_corrections": len(changes),
            "regression_count": 320,
            "llm_dev_count": len(ids),
            "target": 0.92,
            "policy": "Freeze after complete DEV; run fresh TEST once, retain every failure. Regression remains separate.",
            "annotation": "Independent raw-source oracle plus AI source review; not human review.",
        },
    )
    print(
        json.dumps(
            {
                "dev": len(dev),
                "test": len(holdout),
                "source_corrections": len(changes),
                "quotas": QUOTAS,
            }
        )
    )


if __name__ == "__main__":
    main()