"""Run paired LightRAG experiments; preserve answers and actual response evidence. No dependency on RAGAS is required. Source hit is a retrieval proxy, not clinical correctness. Failed requests remain in the ledger and are never scored as zero. """ from __future__ import annotations import argparse import hashlib import json import os import platform import sys import time import urllib.request from collections import defaultdict from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) from pharmacy_agent import QUERY_SETTINGS, build_aliases, prepare_query def request(base: str, path: str, payload=None, timeout=180): data = None if payload is None else json.dumps(payload, ensure_ascii=False).encode() headers = {"Content-Type": "application/json"} if os.getenv("LIGHTRAG_API_KEY"): headers["X-API-Key"] = os.environ["LIGHTRAG_API_KEY"] req = urllib.request.Request(base.rstrip("/") + path, data=data, headers=headers) with urllib.request.urlopen(req, timeout=timeout) as response: return json.load(response) def source_key(value): return str(value).rstrip("/").replace("/pil/", "/detail/").replace("/spc/", "/detail/") def metrics(case, response, mode): refs = response.get("references") or [] retrieved = {source_key(r.get("file_path", "")) for r in refs} gold = {source_key(s) for s in case.get("sources", [])} return { "source_hit": int(bool(retrieved & gold)) if gold and mode != "bypass" else None, "reference_count": len(refs), "evidence_chunks": sum(len(r.get("content") or []) for r in refs), } def summarize(rows): groups = defaultdict(list) for row in rows: groups[row["mode"]].append(row) result = {} for mode, items in groups.items(): ok = [r for r in items if r["status"] == "ok"] hits = [r["metrics"]["source_hit"] for r in ok if r["metrics"]["source_hit"] is not None] times = sorted(r["seconds"] for r in ok) result[mode] = { "attempted": len(items), "successful": len(ok), "failed": len(items)-len(ok), "source_hit_n": len(hits), "source_hit": sum(hits)/len(hits) if hits else None, "latency_median_seconds": (times[(len(times)-1)//2]+times[len(times)//2])/2 if times else None, } return result def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--cases", type=Path, default=ROOT/"evaluation/scenarios.json") p.add_argument("--index", type=Path, default=ROOT/"outputs/demo/product_index.json") p.add_argument("--out", type=Path, default=ROOT/"outputs/evaluation") p.add_argument("--url", default="http://localhost:9621") p.add_argument("--modes", nargs="+", choices=["bypass","naive","local","hybrid","mix"], default=["bypass","naive","local","hybrid","mix"]) p.add_argument("--limit", type=int) p.add_argument("--timeout", type=int, default=180) p.add_argument("--resume", action="store_true") args = p.parse_args() raw = args.cases.read_bytes() cases = json.loads(raw) if args.limit is not None: cases = cases[:args.limit] if not cases or len({c["id"] for c in cases}) != len(cases): raise SystemExit("Empty dataset or duplicate case IDs") products = json.loads(args.index.read_text(encoding="utf-8")) aliases, prefixes, _ = build_aliases(products) health = request(args.url, "/health", timeout=10) if request(args.url, "/documents/pipeline_status", timeout=10).get("busy"): raise SystemExit("Indexing is busy; use a stable index for evaluation") args.out.mkdir(parents=True, exist_ok=True) ledger = args.out/"answers.jsonl" manifest_path = args.out/"manifest.json" settings = QUERY_SETTINGS.copy() identity = {"dataset_sha256":hashlib.sha256(raw).hexdigest(),"case_ids":[c["id"] for c in cases], "index_sha256":hashlib.sha256(args.index.read_bytes()).hexdigest(), "modes":args.modes,"settings":settings,"configuration":health.get("configuration")} rows = [] if args.resume and ledger.exists(): old = json.loads(manifest_path.read_text(encoding="utf-8")) if old["identity"] != identity: raise SystemExit("Resume refused: data, index, server configuration or settings changed") rows = [json.loads(s) for s in ledger.read_text(encoding="utf-8").splitlines() if s.strip()] elif ledger.exists(): raise SystemExit("Output already exists. Use --resume or a new --out directory") manifest_path.write_text(json.dumps({"identity":identity,"started_utc":datetime.now(timezone.utc).isoformat(), "python":platform.python_version(),"note":"Pilot scenarios are source-derived and not independently clinically validated. Timings may include server cache hits."}, ensure_ascii=False, indent=2),encoding="utf-8") completed = {(r["case_id"],r["mode"]) for r in rows if r["status"] == "ok"} for i, case in enumerate(cases): prepared = prepare_query(case["question"], products, aliases, prefixes) # Rotate order to reduce systematic warm-cache/order effects. modes = args.modes[i % len(args.modes):] + args.modes[:i % len(args.modes)] for mode in modes: if (case["id"],mode) in completed: continue payload = {**settings,"query":prepared["query"],"mode":mode,"include_references":True, "include_chunk_content":True,"user_prompt":prepared["user_prompt"]} row = {"case_id":case["id"],"category":case["category"],"mode":mode,"question":case["question"],"payload":payload} start = time.perf_counter() try: response = request(args.url, "/query", payload, timeout=args.timeout) if not response.get("response", "").strip(): raise ValueError("Empty answer") row.update(status="ok",response=response,metrics=metrics(case,response,mode)) except Exception as exc: row.update(status="error",error_type=type(exc).__name__) row["seconds"] = round(time.perf_counter()-start,3) rows.append(row) with ledger.open("a",encoding="utf-8") as f: f.write(json.dumps(row,ensure_ascii=False)+"\n") print(f"{case['id']} {mode}: {row['status']} ({row['seconds']} s)",flush=True) # Last attempt per pair wins; retries must not inflate the denominator. latest = {(r["case_id"],r["mode"]):r for r in rows} summary = summarize(list(latest.values())) (args.out/"summary.json").write_text(json.dumps(summary,indent=2),encoding="utf-8") print(json.dumps(summary,indent=2)) if __name__ == "__main__": main()