90 lines
3.5 KiB
Python
90 lines
3.5 KiB
Python
"""Render a truthful coverage notice without altering the model's facts.
|
|
|
|
No gold or source oracle is accepted. Unsupported citations and unqualified
|
|
exhaustive statements remain rejected. Keep the exact original model answer.
|
|
"""
|
|
|
|
import copy
|
|
|
|
|
|
def finalize_rag_answer(result: dict) -> dict:
|
|
out = copy.deepcopy(result)
|
|
if out.get("display_notice_added"):
|
|
return out
|
|
issues = out.get("context_issues", [])
|
|
out["model_answer"] = out["answer"]
|
|
out["first_pass_accepted_for_delivery"] = out.get("accepted_for_delivery")
|
|
out["first_pass_context_issues"] = list(issues)
|
|
out["display_notice_added"] = False
|
|
# Only a missing notice can be rendered automatically. Adding a caveat to
|
|
# a contradictory exhaustive statement would not repair that statement.
|
|
if issues == ["missing_incompleteness_notice"] and out.get("citations_ok") is True:
|
|
out["answer"] = "Výpis je neúplný.\n\n" + out["model_answer"]
|
|
out["context_issues"] = []
|
|
out["accepted_for_delivery"] = True
|
|
out["display_notice_added"] = True
|
|
return out
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from tools.graphrag.evaluation.reproducibility import sha256_file
|
|
from tools.rag.evaluation.rag_metrics import evaluate_answer
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Replay saved RAG outputs through the delivery layer; no model calls."
|
|
)
|
|
parser.add_argument("--input", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path)
|
|
args = parser.parse_args()
|
|
from tools.runtime.report_files import report_output, resolve_report_input
|
|
|
|
args.output = report_output(args.output, "rag_replay")
|
|
args.input = resolve_report_input(args.input)
|
|
original = json.loads(args.input.read_text())
|
|
source = Path(original["configuration"]["source_file"])
|
|
assert sha256_file(source) == original["configuration"]["source_sha256"]
|
|
gold = {case["id"]: case for case in json.loads(source.read_text())["results"]}
|
|
files = [
|
|
Path(__file__),
|
|
Path("tools/rag/evaluation/evaluate_rag_answers.py"),
|
|
Path("tools/rag/evaluation/rag_metrics.py"),
|
|
Path("tools/rag/evaluation/rag_runner.py"),
|
|
Path("tools/answers/context_policy.py"),
|
|
]
|
|
rows = []
|
|
for before in original["results"]:
|
|
after = finalize_rag_answer(before)
|
|
score = evaluate_answer(
|
|
gold[after["id"]],
|
|
after["answer"],
|
|
tool_called=after["tool_called"],
|
|
allowed_source_urls=after["rag_source_urls"],
|
|
)
|
|
after.update(score)
|
|
after["strict_pass"] = bool(score["strict_pass"] and after["accepted_for_delivery"])
|
|
rows.append(after)
|
|
result = {
|
|
"configuration": {
|
|
"method": "Deterministic delivery replay on the same saved model responses, not new generation.",
|
|
"input": str(args.input),
|
|
"input_sha256": sha256_file(args.input),
|
|
"delivery_files": {str(p): sha256_file(p) for p in files},
|
|
},
|
|
"results": rows,
|
|
"summary": {
|
|
"questions": len(rows),
|
|
"strict_pass": sum(r["strict_pass"] for r in rows),
|
|
"display_notice_added": sum(r["display_notice_added"] for r in rows),
|
|
},
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(result["summary"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|