342 lines
14 KiB
Python
342 lines
14 KiB
Python
"""Audit the real checked RAG path on a predeclared, known regression sample.
|
|
|
|
Default mode only retrieves locally. `--mode deliver` explicitly invokes the
|
|
configured backend model. Expected answers are used here, never sent to /answer.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.request
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
from tools.answers.context_policy import normalized, citation_check, allowed_urls
|
|
from tools.answers.rag_evidence import rag_evidence_context
|
|
from tools.answers.verified_rag import rag_answer_issues, render_rag_answer, quoted_reference_urls
|
|
from tools.graphrag.evaluation.source_oracle import DEFAULT_WIKI, load_source_oracle
|
|
from tools.rag.evaluation.rag_metrics import expected_phrase_matches
|
|
from tools.runtime.report_files import report_output, resolve_report_input
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
DATASET = ROOT / "experiments/rag/datasets/verified_rag_regression.json"
|
|
FIELDS = ["person", "work_type", "title", "year", "start_year"]
|
|
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def binding_matches(expected: dict, rows: list[dict]) -> bool:
|
|
# Study start can be a separate fact about the same person; the academic
|
|
# fields must all occur together in one actual academic row.
|
|
academic = {k: expected[k] for k in FIELDS if k in expected and k != "start_year"}
|
|
academic_ok = any(
|
|
all(normalized(row.get(k)) == normalized(v) for k, v in academic.items()) for row in rows
|
|
)
|
|
start_ok = "start_year" not in expected or any(
|
|
normalized(row.get("person")) == normalized(expected["person"])
|
|
and row.get("start_year") == expected["start_year"]
|
|
for row in rows
|
|
)
|
|
return academic_ok and start_ok
|
|
|
|
|
|
def source_audit(case: dict, oracle: dict) -> list[str]:
|
|
issues = []
|
|
for item in case["source_checks"]:
|
|
if digest(DEFAULT_WIKI / item["document_path"]) != item["sha256"]:
|
|
issues.append("changed_source_document")
|
|
for binding in case["expected_bindings"]:
|
|
person = oracle["people"].get(binding["person"], {})
|
|
rows = [
|
|
{"person": binding["person"], "work_type": a["academic_type"], **a}
|
|
for a in person.get("academics", [])
|
|
if a["document_path"] == binding["document_path"]
|
|
]
|
|
rows.append({"person": binding["person"], "start_year": person.get("start_year")})
|
|
if not binding_matches(binding, rows):
|
|
issues.append("gold_binding_not_in_original_markdown")
|
|
if "start_year" in binding and person.get("start_year_basis") != "explicit_text":
|
|
issues.append("study_start_not_explicit")
|
|
return issues
|
|
|
|
|
|
def evaluate_case(case: dict, payload: dict, delivered: dict | None = None) -> dict:
|
|
context = rag_evidence_context(case["question"], payload)
|
|
source_urls = allowed_urls(payload)
|
|
missing = [b for b in case["expected_bindings"] if not binding_matches(b, context["facts"])]
|
|
retrieval = {
|
|
"expected_sources_found": bool(source_urls & set(case["expected_source_urls"])),
|
|
"missing_bindings": missing,
|
|
"expected_values_in_context": [
|
|
v
|
|
for v in case["expected_answer_contains"]
|
|
if expected_phrase_matches(v, payload.get("context", ""))
|
|
],
|
|
}
|
|
row = {
|
|
"id": case["id"],
|
|
"category": case["category"],
|
|
"question": case["question"],
|
|
"should_answer": case["should_answer"],
|
|
"retrieval": retrieval,
|
|
"typed_expected": bool(case["expected_bindings"]),
|
|
"index_version": payload.get("index_version"),
|
|
}
|
|
if delivered is None:
|
|
return row
|
|
issues = []
|
|
accepted = delivered.get("accepted_for_delivery") is True
|
|
status = delivered.get("status")
|
|
attempts = delivered.get("attempts", [])
|
|
parsed = attempts[-1].get("parsed_answer") if attempts else None
|
|
if accepted and parsed:
|
|
issues.extend(rag_answer_issues(parsed, context))
|
|
if delivered.get("answer") != render_rag_answer(parsed, context):
|
|
issues.append("delivered_text_differs_from_verified_renderer")
|
|
if delivered.get("claims", []) != parsed.get("claims", []):
|
|
issues.append("delivered_claims_differ_from_verified_selection")
|
|
references = quoted_reference_urls(parsed or {}, context)
|
|
if accepted and not citation_check(delivered, source_urls | references)["citations_ok"]:
|
|
issues.append("delivered_unsupported_citation")
|
|
claims = delivered.get("claims", [])
|
|
if accepted:
|
|
# Validate every delivered claim against the actual retrieved tuple.
|
|
# Additional supported projects of the same person are not a wrong
|
|
# thesis year. Required benchmark tuples are checked separately below.
|
|
issues.extend(
|
|
rag_answer_issues(
|
|
{
|
|
"status": status,
|
|
"claims": claims,
|
|
"source_urls": delivered.get("source_urls", []),
|
|
"evidence_quotes": (parsed or {}).get("evidence_quotes", []),
|
|
},
|
|
context,
|
|
)
|
|
)
|
|
if case["should_answer"]:
|
|
if not accepted:
|
|
issues.append("delivery_blocked")
|
|
elif status != "ok":
|
|
issues.append("false_no_answer" if status == "no_answer" else "nonpositive_answer")
|
|
if not all(binding_matches(b, claims) for b in case["expected_bindings"]):
|
|
issues.append("missing_expected_binding")
|
|
# Exclude sources from factual scoring: a name/year in a URL cannot
|
|
# supply the requested fact. Quotes must cover the expected subject.
|
|
text = delivered.get("answer", "").split("\nZdroje:", 1)[0]
|
|
missing_values = [
|
|
v for v in case["expected_answer_contains"] if not expected_phrase_matches(v, text)
|
|
]
|
|
if missing_values:
|
|
issues.append("missing_expected_answer_value")
|
|
quotes = (parsed or {}).get("evidence_quotes", [])
|
|
if quotes:
|
|
content = "\n".join(q["quote"] for q in quotes)
|
|
names = {s.get("title") for s in context["source_blocks"]}
|
|
# Names in source headers are attribution, not answers to a factual
|
|
# question. Value coverage checks relevance beyond literal copying.
|
|
relevant = [v for v in case["expected_answer_contains"] if v not in names]
|
|
row["quote_expected_value_coverage"] = {
|
|
v: expected_phrase_matches(v, content) for v in relevant
|
|
}
|
|
elif not accepted or status != "no_answer":
|
|
issues.append("incorrect_negative_answer")
|
|
row.update(
|
|
passed=not issues,
|
|
issues=sorted(set(issues)),
|
|
delivered=delivered,
|
|
first_attempt_accepted=bool(attempts and not attempts[0].get("issues")),
|
|
)
|
|
return row
|
|
|
|
|
|
def summarize(rows: list[dict]) -> dict:
|
|
counted = [r for r in rows if "passed" in r]
|
|
total = len(counted)
|
|
passed = sum(r["passed"] for r in counted)
|
|
return {
|
|
"total": len(rows),
|
|
"answers_evaluated": total,
|
|
"passed": passed,
|
|
"success_percent": round(100 * passed / total, 2) if total else None,
|
|
"false_no_answers": sum("false_no_answer" in r["issues"] for r in counted),
|
|
"blocked_deliveries": sum("delivery_blocked" in r["issues"] for r in counted),
|
|
"first_attempt_accepted": sum(r.get("first_attempt_accepted", False) for r in counted),
|
|
"failure_reasons": dict(Counter(i for r in counted for i in r["issues"])),
|
|
"source_coverage": sum(r["retrieval"]["expected_sources_found"] for r in rows),
|
|
"answerable_source_coverage": {
|
|
"passed": sum(
|
|
r["retrieval"]["expected_sources_found"] for r in rows if r["should_answer"]
|
|
),
|
|
"total": sum(r["should_answer"] for r in rows),
|
|
},
|
|
"typed_binding_coverage": {
|
|
"passed": sum(
|
|
not r["retrieval"]["missing_bindings"] for r in rows if r.get("typed_expected")
|
|
),
|
|
"total": sum(r.get("typed_expected", False) for r in rows),
|
|
},
|
|
"categories": {
|
|
c: {
|
|
"total": sum(r["category"] == c for r in rows),
|
|
"passed": sum(r.get("passed", False) for r in counted if r["category"] == c),
|
|
}
|
|
for c in sorted({r["category"] for r in rows})
|
|
},
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--mode", choices=["retrieve", "deliver", "replay"], default="retrieve")
|
|
parser.add_argument("--input", type=Path, help="Saved report to regrade without model calls")
|
|
parser.add_argument("--dataset", type=Path, default=DATASET)
|
|
parser.add_argument("--backend", default="http://127.0.0.1:8000")
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument(
|
|
"--no-save", action="store_true", help="Print results only; create no reports or logs"
|
|
)
|
|
args = parser.parse_args()
|
|
load_dotenv(ROOT / ".env", override=False)
|
|
dataset = json.loads(args.dataset.read_text())
|
|
assert (
|
|
digest(ROOT / dataset["source_file"]) == dataset["source_sha256"]
|
|
), "Historical baseline changed"
|
|
oracle = load_source_oracle()
|
|
audit = {c["id"]: source_audit(c, oracle) for c in dataset["cases"]}
|
|
assert not any(audit.values()), "Expected answer/source audit failed"
|
|
if args.no_save and args.output:
|
|
parser.error("--no-save cannot be combined with --output")
|
|
output = None if args.no_save else report_output(args.output, "rag_verified")
|
|
files = [
|
|
Path(__file__),
|
|
ROOT / "tools/answers/answer_delivery.py",
|
|
ROOT / "app/routes.py",
|
|
*[
|
|
(ROOT / "tools/answers" / name)
|
|
for name in ["verified_rag.py", "rag_evidence.py", "context_policy.py"]
|
|
],
|
|
ROOT / "tools/rag/rag_utils.py",
|
|
ROOT / "tools/rag/llm_context.py",
|
|
]
|
|
metadata = {
|
|
"method": dataset["method"],
|
|
"mode": args.mode,
|
|
"dataset_sha256": digest(args.dataset),
|
|
"limitations": dataset.get("limitations", []),
|
|
"source_audit": audit,
|
|
"source_corpus": oracle["source_corpus"],
|
|
"code_hashes": {str(p.relative_to(ROOT)): digest(p) for p in files},
|
|
"model_calls": args.mode == "deliver",
|
|
"backend": args.backend,
|
|
"model": os.getenv("ANSWER_MODEL", "model120-fast") if args.mode == "deliver" else None,
|
|
}
|
|
if output:
|
|
output.with_suffix(".declaration.json").write_text(
|
|
json.dumps(metadata, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
key = os.getenv("SEARCH_API_KEY", "")
|
|
saved = None
|
|
if args.mode == "replay":
|
|
if not args.input:
|
|
parser.error("--input is required for replay")
|
|
saved = {
|
|
r["id"]: r for r in json.loads(resolve_report_input(args.input).read_text())["results"]
|
|
}
|
|
|
|
def run(case):
|
|
started = time.monotonic()
|
|
if saved is not None:
|
|
old = saved[case["id"]]
|
|
return evaluate_case(case, old["retrieval_context"], old.get("delivered")) | {
|
|
"retrieval_context": old["retrieval_context"]
|
|
}
|
|
endpoint = "/answer" if args.mode == "deliver" else "/rag"
|
|
body = {"query": case["question"], "limit": 5}
|
|
if args.mode == "deliver":
|
|
body.update(engine="rag", include_diagnostics=True)
|
|
req = urllib.request.Request(
|
|
args.backend.rstrip("/") + endpoint,
|
|
data=json.dumps(body).encode(),
|
|
headers={"Content-Type": "application/json", "X-API-Key": key},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=300) as response:
|
|
data = json.load(response)
|
|
payload = data.pop("retrieval_context") if args.mode == "deliver" else data
|
|
result = evaluate_case(case, payload, data if args.mode == "deliver" else None)
|
|
result.update(retrieval_context=payload, seconds=round(time.monotonic() - started, 2))
|
|
return result
|
|
except Exception as error:
|
|
# Count operational failures in the denominator; never retry only
|
|
# selected failed questions or expose raw HTTP/token-bearing errors.
|
|
return {
|
|
"id": case["id"],
|
|
"category": case["category"],
|
|
"question": case["question"],
|
|
"should_answer": case["should_answer"],
|
|
"passed": False,
|
|
"issues": ["request_or_evaluation_error"],
|
|
"typed_expected": bool(case["expected_bindings"]),
|
|
"error_type": type(error).__name__,
|
|
"retrieval": {
|
|
"expected_sources_found": False,
|
|
"missing_bindings": case["expected_bindings"],
|
|
},
|
|
}
|
|
|
|
rows = []
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = {pool.submit(run, case): case for case in dataset["cases"]}
|
|
for future in as_completed(futures):
|
|
rows.append(future.result())
|
|
result = {
|
|
"configuration": metadata,
|
|
"summary": summarize(rows),
|
|
"results": sorted(rows, key=lambda r: r["id"]),
|
|
}
|
|
if output:
|
|
output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print(
|
|
f'{len(rows)}/{len(dataset["cases"])} {rows[-1]["id"]}: {rows[-1].get("issues", "retrieved")}',
|
|
flush=True,
|
|
)
|
|
assert metadata["code_hashes"] == {
|
|
name: digest(ROOT / name) for name in metadata["code_hashes"]
|
|
}, "Code changed during measurement"
|
|
print(json.dumps(result["summary"], ensure_ascii=False))
|
|
if output:
|
|
print("Report: " + str(output))
|
|
if args.no_save:
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"failures": [
|
|
{
|
|
"id": r["id"],
|
|
"issues": r["issues"],
|
|
"attempt_issues": [
|
|
a.get("issues") for a in r.get("delivered", {}).get("attempts", [])
|
|
],
|
|
}
|
|
for r in result["results"]
|
|
if not r["passed"]
|
|
]
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|