148 lines
5.9 KiB
Python
148 lines
5.9 KiB
Python
"""Check every returned factual claim against the independent raw-source oracle.
|
|
|
|
Gold bindings define required coverage; this check also rejects false extras.
|
|
It does not use graph construction, query predictions or expected gold values.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from functools import lru_cache
|
|
from .source_oracle import load_source_oracle, norm
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def default_oracle():
|
|
return load_source_oracle()
|
|
|
|
|
|
def claim_issues(claim: dict, oracle: dict, *, published_only: bool = True) -> list[str]:
|
|
names = {norm(name): name for name in oracle["people"]}
|
|
name = names.get(norm(claim.get("person")))
|
|
if name is None:
|
|
return ["unknown_person"]
|
|
person = oracle["people"][name]
|
|
issues = []
|
|
if claim.get("start_year") is not None and claim["start_year"] != person.get("start_year"):
|
|
issues.append("unsupported_start_year")
|
|
path = claim.get("document_path")
|
|
documents = [p for p in person.get("documents", []) if path is None or p == path]
|
|
if path and not documents:
|
|
issues.append("document_belongs_to_other_person")
|
|
documents = [
|
|
p for p in documents if not published_only or oracle["documents"][p].get("published", True)
|
|
]
|
|
if path and not documents:
|
|
issues.append("missing_or_unpublished_source")
|
|
if claim.get("source_url") and not any(
|
|
oracle["documents"][p]["source_url"] == claim["source_url"] for p in documents
|
|
):
|
|
issues.append("incorrect_source_url")
|
|
metadata = {
|
|
k: claim[k] for k in ["keyword", "category", "document_author"] if claim.get(k) is not None
|
|
}
|
|
|
|
def supported(p, k, value):
|
|
d = oracle["documents"][p]
|
|
values = (
|
|
d["tags"] if k == "keyword" else d["categories"] if k == "category" else [d["author"]]
|
|
)
|
|
return norm(value) in {norm(v) for v in values}
|
|
|
|
if metadata and not any(
|
|
all(supported(p, k, v) for k, v in metadata.items()) for p in documents
|
|
):
|
|
issues.append("unsupported_metadata_on_document")
|
|
academic = {
|
|
k: claim[k]
|
|
for k in ["title", "year", "academic_type", "academic_kind"]
|
|
if claim.get(k) is not None
|
|
}
|
|
if academic and not any(
|
|
all(norm(record.get(k)) == norm(v) for k, v in academic.items())
|
|
and (not path or record["document_path"] == path)
|
|
for record in person.get("academics", [])
|
|
):
|
|
issues.append("unsupported_academic_record")
|
|
return issues
|
|
|
|
|
|
def validate_returned_facts(result: dict, oracle: dict) -> dict:
|
|
failures = []
|
|
published_only = result.get("constraints", {}).get("published_only", True)
|
|
for index, row in enumerate(result.get("facts", [])):
|
|
kind = row.get("fact_type") if isinstance(row, dict) else None
|
|
claims = []
|
|
issues = []
|
|
if kind in {"person_thesis", "person_work"}:
|
|
claims = [
|
|
{
|
|
**row,
|
|
"academic_kind": "Thesis" if kind == "person_thesis" else "Work",
|
|
"academic_type": (
|
|
row.get("thesis_type") if kind == "person_thesis" else row.get("work_type")
|
|
),
|
|
}
|
|
]
|
|
elif kind in {
|
|
"person_keyword",
|
|
"person_category",
|
|
"person_document_author",
|
|
"person_start_year",
|
|
"person_academic_year",
|
|
}:
|
|
claims = [row]
|
|
elif kind in {"shared_keyword", "shared_category", "shared_document_author"}:
|
|
field = {
|
|
"shared_keyword": "keyword",
|
|
"shared_category": "category",
|
|
"shared_document_author": "document_author",
|
|
}[kind]
|
|
claims = [
|
|
{
|
|
"person": row.get(side + "_person"),
|
|
field: row.get(field),
|
|
"document_path": row.get(side + "_document_path"),
|
|
"source_url": row.get(side + "_source_url"),
|
|
}
|
|
for side in ["left", "right"]
|
|
]
|
|
elif kind in {"academic_year_comparison", "start_year_comparison"}:
|
|
names = {norm(n): n for n in oracle["people"]}
|
|
people = [names.get(norm(n)) for n in row.get("people", [])]
|
|
if not people or any(n is None for n in people):
|
|
issues.append("invalid_comparison_people")
|
|
elif kind == "academic_year_comparison":
|
|
sets = [
|
|
{
|
|
r["year"]
|
|
for r in oracle["people"][n]["academics"]
|
|
if r.get("year") is not None
|
|
}
|
|
for n in people
|
|
]
|
|
years = sorted(set.intersection(*sets))
|
|
if row.get("shared_years") != years or row.get("same_academic_year") != bool(years):
|
|
issues.append("incorrect_shared_academic_years")
|
|
else:
|
|
years = {oracle["people"][n].get("start_year") for n in people}
|
|
same = len(years) == 1 and None not in years
|
|
expected = {norm(n): oracle["people"][n].get("start_year") for n in people}
|
|
actual = {norm(n): year for n, year in row.get("start_years", {}).items()}
|
|
if actual != expected:
|
|
issues.append("incorrect_individual_start_years")
|
|
if row.get("same_start_year") != same or row.get("shared_start_year") != (
|
|
next(iter(years)) if same else None
|
|
):
|
|
issues.append("incorrect_shared_start_year")
|
|
else:
|
|
issues.append("unsupported_fact_type")
|
|
for claim in claims:
|
|
issues.extend(claim_issues(claim, oracle, published_only=published_only))
|
|
if issues:
|
|
failures.append({"fact_index": index, "fact_type": kind, "issues": sorted(set(issues))})
|
|
return {
|
|
"source_checked": True,
|
|
"all_facts_supported": not failures,
|
|
"unsupported_facts": failures,
|
|
}
|