"""Recheck archived citations against archived tool sources, without generation. This changes only citation eligibility; it does not replace historical gold or claim a new full semantic evaluation. Missing archived context is unverifiable. """ import argparse import json from pathlib import Path from tools.graphrag.evaluation.reproducibility import sha256_file from tools.answers.context_policy import citation_check, allowed_urls def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input", type=Path, action="append", required=True) parser.add_argument("--output", type=Path) args = parser.parse_args() from tools.runtime.report_files import report_output, resolve_report_input args.output = report_output(args.output, "citations") args.input = [resolve_report_input(path) for path in args.input] reports = [] for path in args.input: data = json.loads(path.read_text()) rows = [] for old in data.get("questions", data.get("results", [])): answer = old.get("answer") or "" if isinstance(answer, str): answer = {"answer": answer, "source_urls": []} context = old.get("graph_context") legacy_sources = ( {row["source_url"] for row in context.get("documents", []) if row.get("source_url")} if isinstance(context, dict) and "documents" in context else None ) urls = ( allowed_urls(context) if isinstance(context, dict) and "sources" in context else ( legacy_sources if legacy_sources is not None else ( set(old["rag_source_urls"]) if isinstance(old.get("rag_source_urls"), list) else None ) ) ) check = ( citation_check(answer, urls) if urls is not None else {"citations_ok": None, "unsupported_urls": []} ) strict = bool(old.get("strict", old.get("strict_pass", False))) rows.append( { "id": old["id"], "previous_strict": strict, **check, "citation_adjusted_strict": ( (strict and check["citations_ok"]) if urls is not None else None ), } ) reports.append( { "file": str(path), "sha256": sha256_file(path), "rows": rows, "summary": { "questions": len(rows), "checked": sum(r["citations_ok"] is not None for r in rows), "invalid_citations": sum(r["citations_ok"] is False for r in rows), "previous_strict": sum(r["previous_strict"] for r in rows), "strict_after_citation_check": sum( r["citation_adjusted_strict"] is True for r in rows ), }, } ) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( json.dumps({"method": __doc__, "files": reports}, ensure_ascii=False, indent=2) + "\n" ) print(json.dumps([{"file": r["file"], **r["summary"]} for r in reports], ensure_ascii=False)) if __name__ == "__main__": main()