97 lines
3.2 KiB
Python
97 lines
3.2 KiB
Python
"""Five preselected historical RAG smoke cases; not a new accuracy estimate."""
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from tools.rag.evaluation.rag_runner import (
|
|
build_rag_tool,
|
|
request_json,
|
|
run_question,
|
|
LOCAL_OPENAPI_URL,
|
|
DEFAULT_MODEL,
|
|
)
|
|
from tools.rag.evaluation.rag_metrics import evaluate_answer
|
|
from tools.graphrag.evaluation.reproducibility import sha256_file
|
|
from tools.evaluation.rag_delivery import finalize_rag_answer
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output", type=Path)
|
|
args = parser.parse_args()
|
|
from tools.runtime.report_files import report_output
|
|
|
|
args.output = report_output(args.output, "rag_runtime")
|
|
source = Path("experiments/rag/results/test_classic_rag_frozen/rag_answers_test.json")
|
|
historic = json.loads(source.read_text())["results"]
|
|
selected = ["q0053", "q0060", "q0608", "q1147", "q0888"]
|
|
cases = [next(row for row in historic if row["id"] == id) for id in selected]
|
|
operation, tool = build_rag_tool(request_json(LOCAL_OPENAPI_URL))
|
|
metadata = {
|
|
"method": __doc__,
|
|
"selected_ids": selected,
|
|
"model": DEFAULT_MODEL,
|
|
"source_file": str(source),
|
|
"source_sha256": sha256_file(source),
|
|
"generation_files": {
|
|
str(p): sha256_file(p)
|
|
for p in [
|
|
Path(__file__),
|
|
Path("tools/rag/evaluation/rag_runner.py"),
|
|
Path("tools/rag/evaluation/rag_metrics.py"),
|
|
Path("tools/answers/context_policy.py"),
|
|
Path("tools/evaluation/rag_delivery.py"),
|
|
]
|
|
},
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.with_suffix(".declaration.json").write_text(
|
|
json.dumps(metadata, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
|
|
def run(case):
|
|
out = run_question(
|
|
case["question"],
|
|
model=DEFAULT_MODEL,
|
|
operation_id=operation,
|
|
rag_tool=tool,
|
|
openwebui_api_key=os.environ["OPENWEBUI_API_KEY"],
|
|
search_api_key=os.environ["SEARCH_API_KEY"],
|
|
timeout=120,
|
|
max_attempts=2,
|
|
)
|
|
out = finalize_rag_answer(out)
|
|
score = evaluate_answer(
|
|
case,
|
|
out["answer"],
|
|
tool_called=out["tool_called"],
|
|
allowed_source_urls=out["rag_source_urls"],
|
|
)
|
|
return {
|
|
"id": case["id"],
|
|
"question": case["question"],
|
|
**out,
|
|
**score,
|
|
"strict_pass": bool(score["strict_pass"] and out["accepted_for_delivery"]),
|
|
}
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
|
rows = list(executor.map(run, cases))
|
|
assert metadata["generation_files"] == {
|
|
p: sha256_file(Path(p)) for p in metadata["generation_files"]
|
|
}
|
|
result = {
|
|
"configuration": metadata,
|
|
"results": rows,
|
|
"summary": {"questions": len(rows), "strict_pass": sum(row["strict_pass"] for row in rows)},
|
|
}
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(result["summary"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|