92 lines
3.5 KiB
Python
92 lines
3.5 KiB
Python
"""Recheck archived citations against archived tool sources, without generation.
|
|
|
|
This changes only citation eligibility; it does not replace historical gold or
|
|
claim a new full semantic evaluation. Missing archived context is unverifiable.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from tools.graphrag.evaluation.reproducibility import sha256_file
|
|
from tools.answers.context_policy import citation_check, allowed_urls
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--input", type=Path, action="append", required=True)
|
|
parser.add_argument("--output", type=Path)
|
|
args = parser.parse_args()
|
|
from tools.runtime.report_files import report_output, resolve_report_input
|
|
|
|
args.output = report_output(args.output, "citations")
|
|
args.input = [resolve_report_input(path) for path in args.input]
|
|
reports = []
|
|
for path in args.input:
|
|
data = json.loads(path.read_text())
|
|
rows = []
|
|
for old in data.get("questions", data.get("results", [])):
|
|
answer = old.get("answer") or ""
|
|
if isinstance(answer, str):
|
|
answer = {"answer": answer, "source_urls": []}
|
|
context = old.get("graph_context")
|
|
legacy_sources = (
|
|
{row["source_url"] for row in context.get("documents", []) if row.get("source_url")}
|
|
if isinstance(context, dict) and "documents" in context
|
|
else None
|
|
)
|
|
urls = (
|
|
allowed_urls(context)
|
|
if isinstance(context, dict) and "sources" in context
|
|
else (
|
|
legacy_sources
|
|
if legacy_sources is not None
|
|
else (
|
|
set(old["rag_source_urls"])
|
|
if isinstance(old.get("rag_source_urls"), list)
|
|
else None
|
|
)
|
|
)
|
|
)
|
|
check = (
|
|
citation_check(answer, urls)
|
|
if urls is not None
|
|
else {"citations_ok": None, "unsupported_urls": []}
|
|
)
|
|
strict = bool(old.get("strict", old.get("strict_pass", False)))
|
|
rows.append(
|
|
{
|
|
"id": old["id"],
|
|
"previous_strict": strict,
|
|
**check,
|
|
"citation_adjusted_strict": (
|
|
(strict and check["citations_ok"]) if urls is not None else None
|
|
),
|
|
}
|
|
)
|
|
reports.append(
|
|
{
|
|
"file": str(path),
|
|
"sha256": sha256_file(path),
|
|
"rows": rows,
|
|
"summary": {
|
|
"questions": len(rows),
|
|
"checked": sum(r["citations_ok"] is not None for r in rows),
|
|
"invalid_citations": sum(r["citations_ok"] is False for r in rows),
|
|
"previous_strict": sum(r["previous_strict"] for r in rows),
|
|
"strict_after_citation_check": sum(
|
|
r["citation_adjusted_strict"] is True for r in rows
|
|
),
|
|
},
|
|
}
|
|
)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(
|
|
json.dumps({"method": __doc__, "files": reports}, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(json.dumps([{"file": r["file"], **r["summary"]} for r in reports], ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|