"""Audit existing outputs without rebuilding or modifying either graph."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import unicodedata
import xml.etree.ElementTree as ET
from collections import Counter, defaultdict
from pathlib import Path


def records(path):
    """Stream a top-level JSON array with bounded memory and strict decoding."""
    decoder=json.JSONDecoder()
    with path.open(encoding="utf-8-sig") as f:
        buffer=""
        started=False
        while True:
            block=f.read(1024*1024)
            buffer+=block
            while True:
                buffer=buffer.lstrip()
                if not started:
                    if not buffer:
                        break
                    if not buffer.startswith("["):
                        raise ValueError("Expected JSON array")
                    buffer=buffer[1:]
                    started=True
                    continue
                if buffer.startswith(","):
                    buffer=buffer[1:]
                    continue
                if buffer.startswith("]"):
                    if buffer[1:].strip() or f.read().strip():
                        raise ValueError("Trailing JSON data")
                    return
                try:
                    value,end=decoder.raw_decode(buffer)
                except json.JSONDecodeError:
                    break
                yield value
                buffer=buffer[end:]
            if not block:
                raise ValueError("Truncated JSON array")


def norm(text):
    text=unicodedata.normalize("NFKD",text).lower()
    return re.sub(r"[^a-z0-9]+"," ","".join(c for c in text if not unicodedata.combining(c))).strip()


def main():
    p=argparse.ArgumentParser(description=__doc__)
    p.add_argument("--root",type=Path,required=True)
    p.add_argument("--out",type=Path,default=Path("outputs/evaluation_audit"))
    args=p.parse_args()
    root=args.root
    ns="{http://graphml.graphdrawing.org/xmlns}"
    graph=root/"lightrag/rag_storage_qwen36/graph_chunk_entity_relation.graphml"
    keys={}; labels=defaultdict(list); types=Counter(); edges=0; nodes=0
    for event,element in ET.iterparse(graph,events=("end",)):
        if element.tag==ns+"key":
            keys[element.attrib["id"]]=element.attrib.get("attr.name","")
        elif element.tag==ns+"node":
            nodes+=1
            label=element.attrib["id"]
            values={keys.get(d.attrib.get("key")):d.text for d in element.findall(ns+"data")}
            types[values.get("entity_type","unknown")]+=1
            labels[norm(label)].append(label)
            element.clear()
        elif element.tag==ns+"edge":
            edges+=1; element.clear()
    duplicates=[v for v in labels.values() if len(v)>1]
    groups=defaultdict(lambda:{"count":0,"signatures":set(),"forms":set()})
    counts=Counter()
    clinical_keys=("contraindications_text","interactions_text","warnings_text","side_effects_text")
    main_data=root/"data_adc_databaza/adc_scrape_2026_05_04/adc_products_structured.json"
    for record in records(main_data):
        counts["records"]+=1
        hints=record.get("graph_hints") or {}
        texts=[str(hints.get(k) or "").strip() for k in clinical_keys]
        if not any(texts):
            continue
        counts["clinical_records"]+=1
        subs=hints.get("active_substances") or []
        if isinstance(subs,str): subs=[subs]
        substance_key="|".join(sorted({s.strip().lower() for s in subs if s.strip()}))
        signature=hashlib.sha256(json.dumps(texts,ensure_ascii=False).encode()).hexdigest()
        key="substance:"+substance_key if substance_key else "text:"+signature
        g=groups[key]; g["count"]+=1; g["signatures"].add(signature); g["forms"].add(str(hints.get("dosage_form") or ""))
    varied=[(k,v) for k,v in groups.items() if v["count"]>1 and len(v["signatures"])>1]
    status=json.loads((root/"lightrag/rag_storage_qwen36/kv_store_doc_status.json").read_text(encoding="utf-8"))
    report={"dataset":dict(counts),"deduplication":{"groups":len(groups),
        "removed_records_if_one_per_group":counts["clinical_records"]-len(groups),
        "groups_with_nonidentical_clinical_texts":len(varied),
        "records_in_nonidentical_groups":sum(v["count"] for _,v in varied),
        "examples":[{"key":k,"records":v["count"],"different_texts":len(v["signatures"]),"forms":sorted(v["forms"])} for k,v in varied[:15]],
        "limitation":"Text differences are not evidence of clinically material differences; manual review is required."},
        "semantic_graph":{"nodes":nodes,"edges":edges,"types":dict(types),"normalized_duplicate_groups":len(duplicates),
            "nodes_in_duplicate_groups":sum(map(len,duplicates)),"excess_nodes_after_normalization":sum(len(v)-1 for v in duplicates)},
        "document_status":dict(Counter(r.get("status") for r in status.values()))}
    args.out.mkdir(parents=True,exist_ok=True)
    (args.out/"audit.json").write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding="utf-8")
    print(json.dumps({k:v for k,v in report.items() if k!="deduplication"},ensure_ascii=False,indent=2))
    print("Deduplication:",json.dumps({k:v for k,v in report["deduplication"].items() if k!="examples"},ensure_ascii=False))


if __name__=="__main__": main()