139 lines
6.7 KiB
Python
139 lines
6.7 KiB
Python
"""Run paired LightRAG experiments; preserve answers and actual response evidence.
|
|
|
|
No dependency on RAGAS is required. Source hit is a retrieval proxy, not clinical
|
|
correctness. Failed requests remain in the ledger and are never scored as zero.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
from collections import defaultdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT))
|
|
from pharmacy_agent import QUERY_SETTINGS, build_aliases, prepare_query
|
|
|
|
|
|
def request(base: str, path: str, payload=None, timeout=180):
|
|
data = None if payload is None else json.dumps(payload, ensure_ascii=False).encode()
|
|
headers = {"Content-Type": "application/json"}
|
|
if os.getenv("LIGHTRAG_API_KEY"):
|
|
headers["X-API-Key"] = os.environ["LIGHTRAG_API_KEY"]
|
|
req = urllib.request.Request(base.rstrip("/") + path, data=data, headers=headers)
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def source_key(value):
|
|
return str(value).rstrip("/").replace("/pil/", "/detail/").replace("/spc/", "/detail/")
|
|
|
|
|
|
def metrics(case, response, mode):
|
|
refs = response.get("references") or []
|
|
retrieved = {source_key(r.get("file_path", "")) for r in refs}
|
|
gold = {source_key(s) for s in case.get("sources", [])}
|
|
return {
|
|
"source_hit": int(bool(retrieved & gold)) if gold and mode != "bypass" else None,
|
|
"reference_count": len(refs),
|
|
"evidence_chunks": sum(len(r.get("content") or []) for r in refs),
|
|
}
|
|
|
|
|
|
def summarize(rows):
|
|
groups = defaultdict(list)
|
|
for row in rows:
|
|
groups[row["mode"]].append(row)
|
|
result = {}
|
|
for mode, items in groups.items():
|
|
ok = [r for r in items if r["status"] == "ok"]
|
|
hits = [r["metrics"]["source_hit"] for r in ok if r["metrics"]["source_hit"] is not None]
|
|
times = sorted(r["seconds"] for r in ok)
|
|
result[mode] = {
|
|
"attempted": len(items), "successful": len(ok), "failed": len(items)-len(ok),
|
|
"source_hit_n": len(hits), "source_hit": sum(hits)/len(hits) if hits else None,
|
|
"latency_median_seconds": (times[(len(times)-1)//2]+times[len(times)//2])/2 if times else None,
|
|
}
|
|
return result
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--cases", type=Path, default=ROOT/"evaluation/scenarios.json")
|
|
p.add_argument("--index", type=Path, default=ROOT/"outputs/demo/product_index.json")
|
|
p.add_argument("--out", type=Path, default=ROOT/"outputs/evaluation")
|
|
p.add_argument("--url", default="http://localhost:9621")
|
|
p.add_argument("--modes", nargs="+", choices=["bypass","naive","local","hybrid","mix"], default=["bypass","naive","local","hybrid","mix"])
|
|
p.add_argument("--limit", type=int)
|
|
p.add_argument("--timeout", type=int, default=180)
|
|
p.add_argument("--resume", action="store_true")
|
|
args = p.parse_args()
|
|
raw = args.cases.read_bytes()
|
|
cases = json.loads(raw)
|
|
if args.limit is not None:
|
|
cases = cases[:args.limit]
|
|
if not cases or len({c["id"] for c in cases}) != len(cases):
|
|
raise SystemExit("Empty dataset or duplicate case IDs")
|
|
products = json.loads(args.index.read_text(encoding="utf-8"))
|
|
aliases, prefixes, _ = build_aliases(products)
|
|
health = request(args.url, "/health", timeout=10)
|
|
if request(args.url, "/documents/pipeline_status", timeout=10).get("busy"):
|
|
raise SystemExit("Indexing is busy; use a stable index for evaluation")
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
ledger = args.out/"answers.jsonl"
|
|
manifest_path = args.out/"manifest.json"
|
|
settings = QUERY_SETTINGS.copy()
|
|
identity = {"dataset_sha256":hashlib.sha256(raw).hexdigest(),"case_ids":[c["id"] for c in cases],
|
|
"index_sha256":hashlib.sha256(args.index.read_bytes()).hexdigest(),
|
|
"modes":args.modes,"settings":settings,"configuration":health.get("configuration")}
|
|
rows = []
|
|
if args.resume and ledger.exists():
|
|
old = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
if old["identity"] != identity:
|
|
raise SystemExit("Resume refused: data, index, server configuration or settings changed")
|
|
rows = [json.loads(s) for s in ledger.read_text(encoding="utf-8").splitlines() if s.strip()]
|
|
elif ledger.exists():
|
|
raise SystemExit("Output already exists. Use --resume or a new --out directory")
|
|
manifest_path.write_text(json.dumps({"identity":identity,"started_utc":datetime.now(timezone.utc).isoformat(),
|
|
"python":platform.python_version(),"note":"Pilot scenarios are source-derived and not independently clinically validated. Timings may include server cache hits."}, ensure_ascii=False, indent=2),encoding="utf-8")
|
|
completed = {(r["case_id"],r["mode"]) for r in rows if r["status"] == "ok"}
|
|
for i, case in enumerate(cases):
|
|
prepared = prepare_query(case["question"], products, aliases, prefixes)
|
|
# Rotate order to reduce systematic warm-cache/order effects.
|
|
modes = args.modes[i % len(args.modes):] + args.modes[:i % len(args.modes)]
|
|
for mode in modes:
|
|
if (case["id"],mode) in completed:
|
|
continue
|
|
payload = {**settings,"query":prepared["query"],"mode":mode,"include_references":True,
|
|
"include_chunk_content":True,"user_prompt":prepared["user_prompt"]}
|
|
row = {"case_id":case["id"],"category":case["category"],"mode":mode,"question":case["question"],"payload":payload}
|
|
start = time.perf_counter()
|
|
try:
|
|
response = request(args.url, "/query", payload, timeout=args.timeout)
|
|
if not response.get("response", "").strip():
|
|
raise ValueError("Empty answer")
|
|
row.update(status="ok",response=response,metrics=metrics(case,response,mode))
|
|
except Exception as exc:
|
|
row.update(status="error",error_type=type(exc).__name__)
|
|
row["seconds"] = round(time.perf_counter()-start,3)
|
|
rows.append(row)
|
|
with ledger.open("a",encoding="utf-8") as f:
|
|
f.write(json.dumps(row,ensure_ascii=False)+"\n")
|
|
print(f"{case['id']} {mode}: {row['status']} ({row['seconds']} s)",flush=True)
|
|
# Last attempt per pair wins; retries must not inflate the denominator.
|
|
latest = {(r["case_id"],r["mode"]):r for r in rows}
|
|
summary = summarize(list(latest.values()))
|
|
(args.out/"summary.json").write_text(json.dumps(summary,indent=2),encoding="utf-8")
|
|
print(json.dumps(summary,indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|