111 lines
5.2 KiB
Python
111 lines
5.2 KiB
Python
"""Audit existing outputs without rebuilding or modifying either graph."""
|
|
from __future__ import annotations
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
import xml.etree.ElementTree as ET
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
|
|
|
|
def records(path):
|
|
"""Stream a top-level JSON array with bounded memory and strict decoding."""
|
|
decoder=json.JSONDecoder()
|
|
with path.open(encoding="utf-8-sig") as f:
|
|
buffer=""
|
|
started=False
|
|
while True:
|
|
block=f.read(1024*1024)
|
|
buffer+=block
|
|
while True:
|
|
buffer=buffer.lstrip()
|
|
if not started:
|
|
if not buffer:
|
|
break
|
|
if not buffer.startswith("["):
|
|
raise ValueError("Expected JSON array")
|
|
buffer=buffer[1:]
|
|
started=True
|
|
continue
|
|
if buffer.startswith(","):
|
|
buffer=buffer[1:]
|
|
continue
|
|
if buffer.startswith("]"):
|
|
if buffer[1:].strip() or f.read().strip():
|
|
raise ValueError("Trailing JSON data")
|
|
return
|
|
try:
|
|
value,end=decoder.raw_decode(buffer)
|
|
except json.JSONDecodeError:
|
|
break
|
|
yield value
|
|
buffer=buffer[end:]
|
|
if not block:
|
|
raise ValueError("Truncated JSON array")
|
|
|
|
|
|
def norm(text):
|
|
text=unicodedata.normalize("NFKD",text).lower()
|
|
return re.sub(r"[^a-z0-9]+"," ","".join(c for c in text if not unicodedata.combining(c))).strip()
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--root",type=Path,required=True)
|
|
p.add_argument("--out",type=Path,default=Path("outputs/evaluation_audit"))
|
|
args=p.parse_args()
|
|
root=args.root
|
|
ns="{http://graphml.graphdrawing.org/xmlns}"
|
|
graph=root/"lightrag/rag_storage_qwen36/graph_chunk_entity_relation.graphml"
|
|
keys={}; labels=defaultdict(list); types=Counter(); edges=0; nodes=0
|
|
for event,element in ET.iterparse(graph,events=("end",)):
|
|
if element.tag==ns+"key":
|
|
keys[element.attrib["id"]]=element.attrib.get("attr.name","")
|
|
elif element.tag==ns+"node":
|
|
nodes+=1
|
|
label=element.attrib["id"]
|
|
values={keys.get(d.attrib.get("key")):d.text for d in element.findall(ns+"data")}
|
|
types[values.get("entity_type","unknown")]+=1
|
|
labels[norm(label)].append(label)
|
|
element.clear()
|
|
elif element.tag==ns+"edge":
|
|
edges+=1; element.clear()
|
|
duplicates=[v for v in labels.values() if len(v)>1]
|
|
groups=defaultdict(lambda:{"count":0,"signatures":set(),"forms":set()})
|
|
counts=Counter()
|
|
clinical_keys=("contraindications_text","interactions_text","warnings_text","side_effects_text")
|
|
main_data=root/"data_adc_databaza/adc_scrape_2026_05_04/adc_products_structured.json"
|
|
for record in records(main_data):
|
|
counts["records"]+=1
|
|
hints=record.get("graph_hints") or {}
|
|
texts=[str(hints.get(k) or "").strip() for k in clinical_keys]
|
|
if not any(texts):
|
|
continue
|
|
counts["clinical_records"]+=1
|
|
subs=hints.get("active_substances") or []
|
|
if isinstance(subs,str): subs=[subs]
|
|
substance_key="|".join(sorted({s.strip().lower() for s in subs if s.strip()}))
|
|
signature=hashlib.sha256(json.dumps(texts,ensure_ascii=False).encode()).hexdigest()
|
|
key="substance:"+substance_key if substance_key else "text:"+signature
|
|
g=groups[key]; g["count"]+=1; g["signatures"].add(signature); g["forms"].add(str(hints.get("dosage_form") or ""))
|
|
varied=[(k,v) for k,v in groups.items() if v["count"]>1 and len(v["signatures"])>1]
|
|
status=json.loads((root/"lightrag/rag_storage_qwen36/kv_store_doc_status.json").read_text(encoding="utf-8"))
|
|
report={"dataset":dict(counts),"deduplication":{"groups":len(groups),
|
|
"removed_records_if_one_per_group":counts["clinical_records"]-len(groups),
|
|
"groups_with_nonidentical_clinical_texts":len(varied),
|
|
"records_in_nonidentical_groups":sum(v["count"] for _,v in varied),
|
|
"examples":[{"key":k,"records":v["count"],"different_texts":len(v["signatures"]),"forms":sorted(v["forms"])} for k,v in varied[:15]],
|
|
"limitation":"Text differences are not evidence of clinically material differences; manual review is required."},
|
|
"semantic_graph":{"nodes":nodes,"edges":edges,"types":dict(types),"normalized_duplicate_groups":len(duplicates),
|
|
"nodes_in_duplicate_groups":sum(map(len,duplicates)),"excess_nodes_after_normalization":sum(len(v)-1 for v in duplicates)},
|
|
"document_status":dict(Counter(r.get("status") for r in status.values()))}
|
|
args.out.mkdir(parents=True,exist_ok=True)
|
|
(args.out/"audit.json").write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding="utf-8")
|
|
print(json.dumps({k:v for k,v in report.items() if k!="deduplication"},ensure_ascii=False,indent=2))
|
|
print("Deduplication:",json.dumps({k:v for k,v in report["deduplication"].items() if k!="examples"},ensure_ascii=False))
|
|
|
|
|
|
if __name__=="__main__": main()
|