167 lines
7.3 KiB
Python
167 lines
7.3 KiB
Python
"""Check source consistency without changing questions or consulting search output."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from ..graphrag import build_graph_payload
|
|
from .graphrag_evaluator import normalize_text
|
|
from .reproducibility import content_fingerprints
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
|
|
|
|
def audit_questions(questions: list[dict], payload: dict) -> dict[str, Any]:
|
|
people = {normalize_text(row["name"]): row for row in payload["people"]}
|
|
documents = {row["path"]: row for row in payload["documents"]}
|
|
person_docs: dict[str, set[str]] = defaultdict(set)
|
|
for row in payload["person_documents"]:
|
|
person_docs[row["person_id"]].add(row["document_path"])
|
|
doc_keywords: dict[str, set[str]] = defaultdict(set)
|
|
for row in payload["document_keywords"]:
|
|
doc_keywords[row["document_path"]].add(row["keyword_id"])
|
|
keywords = {normalize_text(row["name"]): row["id"] for row in payload["keywords"]}
|
|
keywords.update({normalize_text(row["id"]): row["id"] for row in payload["keywords"]})
|
|
academic: dict[str, list[dict]] = defaultdict(list)
|
|
for row in payload["theses"] + payload["works"]:
|
|
academic[row["source_document"]].append(row)
|
|
rows = []
|
|
for question in questions:
|
|
graph = question["graph"]
|
|
issues = []
|
|
starts = [
|
|
people.get(normalize_text(item["value"]))
|
|
for item in graph.get("start_entities", [])
|
|
if item["type"] == "person"
|
|
]
|
|
if any(person is None for person in starts):
|
|
issues.append("unknown_start_person")
|
|
starts = [person for person in starts if person]
|
|
expected = graph.get("expected_entities", [])
|
|
targets = [
|
|
people.get(normalize_text(item["value"]))
|
|
for item in expected
|
|
if item["type"] == "person"
|
|
]
|
|
targets = [person for person in targets if person] or starts
|
|
required_people = starts if question["category"] == "shared_keyword_pair" else targets
|
|
for entity in expected:
|
|
if entity["type"] == "keyword":
|
|
key = keywords.get(normalize_text(entity["value"]))
|
|
if key is None:
|
|
issues.append("keyword_not_in_frontmatter_catalog")
|
|
elif any(
|
|
not any(key in doc_keywords[path] for path in person_docs[person["id"]])
|
|
for person in required_people
|
|
):
|
|
issues.append("keyword_not_linked_to_expected_person")
|
|
elif entity["type"] in {"thesis_title", "work_title"}:
|
|
records = (
|
|
payload["theses"] if entity["type"] == "thesis_title" else payload["works"]
|
|
)
|
|
paths = {person["source_document"] for person in starts}
|
|
if not any(
|
|
row["source_document"] in paths
|
|
and normalize_text(row.get("title")) == normalize_text(entity["value"])
|
|
for row in records
|
|
):
|
|
issues.append("title_not_supported_by_source")
|
|
category = question["category"]
|
|
if category == "person_multiple_categories" and starts:
|
|
actual = {
|
|
normalize_text(cat)
|
|
for path in person_docs[starts[0]["id"]]
|
|
for cat in documents[path]["categories"]
|
|
}
|
|
expected_set = {normalize_text(value) for value in graph.get("expected_entity_set", [])}
|
|
if graph.get("entity_set_mode") == "exact" and actual != expected_set:
|
|
issues.append("exact_categories_omit_or_add_source_categories")
|
|
years = {int(value) for value in re.findall(r"\b20\d{2}\b", question["question"])}
|
|
if category == "same_academic_year_pair":
|
|
expected_years = {int(item["value"]) for item in expected if item["type"] == "year"}
|
|
if any(
|
|
not expected_years <= {r["year"] for r in academic[p["source_document"]]}
|
|
for p in starts
|
|
):
|
|
issues.append("shared_year_not_supported_by_source")
|
|
if category == "person_title_year":
|
|
expected_years = {int(item["value"]) for item in expected if item["type"] == "year"}
|
|
titles = {
|
|
normalize_text(item["value"])
|
|
for item in expected
|
|
if item["type"] in {"thesis_title", "work_title"}
|
|
}
|
|
if starts and not any(
|
|
normalize_text(r.get("title")) in titles and r["year"] in expected_years
|
|
for p in starts
|
|
for r in academic[p["source_document"]]
|
|
):
|
|
issues.append("title_year_pair_not_supported_by_source")
|
|
if category == "candidate_set_aggregation":
|
|
actual = {
|
|
normalize_text(p["name"])
|
|
for p in starts
|
|
if any(r["year"] in years for r in academic[p["source_document"]])
|
|
}
|
|
expected_set = {normalize_text(value) for value in graph.get("expected_entity_set", [])}
|
|
if actual != expected_set or len(actual) != graph.get("expected_count"):
|
|
issues.append("candidate_set_or_count_not_supported_by_source")
|
|
for path in question.get("expected_documents", []):
|
|
if path not in documents:
|
|
issues.append("unknown_expected_document")
|
|
rows.append(
|
|
{
|
|
"id": question["id"],
|
|
"split": question["split"],
|
|
"category": category,
|
|
"source_consistent": not issues,
|
|
"issues": sorted(set(issues)),
|
|
}
|
|
)
|
|
return {
|
|
"method": "Static source checks; no retrieval output, question rewriting or alias invention.",
|
|
"questions": rows,
|
|
"by_split": {
|
|
split: {
|
|
"questions": len(selected),
|
|
"source_consistent": sum(row["source_consistent"] for row in selected),
|
|
"issues": dict(Counter(issue for row in selected for issue in row["issues"])),
|
|
}
|
|
for split in ["dev", "test"]
|
|
for selected in [[row for row in rows if row["split"] == split]]
|
|
},
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--questions",
|
|
type=Path,
|
|
default=Path(__file__).resolve().parents[3]
|
|
/ "experiments/graphrag/graphrag_questions.json",
|
|
)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
questions = json.loads(args.questions.read_text())
|
|
payload = build_graph_payload(
|
|
json.loads((ROOT / "data/documents.json").read_text()),
|
|
json.loads((ROOT / "data/chunks.json").read_text()),
|
|
)
|
|
audit = audit_questions(questions, payload)
|
|
audit["dataset_sha256"] = hashlib.sha256(args.questions.read_bytes()).hexdigest()
|
|
audit["inputs"] = content_fingerprints(args.questions)["inputs"]
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(audit, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(audit["by_split"], ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|