"""Check every returned factual claim against the independent raw-source oracle. Gold bindings define required coverage; this check also rejects false extras. It does not use graph construction, query predictions or expected gold values. """ from __future__ import annotations from functools import lru_cache from .source_oracle import load_source_oracle, norm @lru_cache(maxsize=1) def default_oracle(): return load_source_oracle() def claim_issues(claim: dict, oracle: dict, *, published_only: bool = True) -> list[str]: names = {norm(name): name for name in oracle["people"]} name = names.get(norm(claim.get("person"))) if name is None: return ["unknown_person"] person = oracle["people"][name] issues = [] if claim.get("start_year") is not None and claim["start_year"] != person.get("start_year"): issues.append("unsupported_start_year") path = claim.get("document_path") documents = [p for p in person.get("documents", []) if path is None or p == path] if path and not documents: issues.append("document_belongs_to_other_person") documents = [ p for p in documents if not published_only or oracle["documents"][p].get("published", True) ] if path and not documents: issues.append("missing_or_unpublished_source") if claim.get("source_url") and not any( oracle["documents"][p]["source_url"] == claim["source_url"] for p in documents ): issues.append("incorrect_source_url") metadata = { k: claim[k] for k in ["keyword", "category", "document_author"] if claim.get(k) is not None } def supported(p, k, value): d = oracle["documents"][p] values = ( d["tags"] if k == "keyword" else d["categories"] if k == "category" else [d["author"]] ) return norm(value) in {norm(v) for v in values} if metadata and not any( all(supported(p, k, v) for k, v in metadata.items()) for p in documents ): issues.append("unsupported_metadata_on_document") academic = { k: claim[k] for k in ["title", "year", "academic_type", "academic_kind"] if claim.get(k) is not None } if academic and not any( all(norm(record.get(k)) == norm(v) for k, v in academic.items()) and (not path or record["document_path"] == path) for record in person.get("academics", []) ): issues.append("unsupported_academic_record") return issues def validate_returned_facts(result: dict, oracle: dict) -> dict: failures = [] published_only = result.get("constraints", {}).get("published_only", True) for index, row in enumerate(result.get("facts", [])): kind = row.get("fact_type") if isinstance(row, dict) else None claims = [] issues = [] if kind in {"person_thesis", "person_work"}: claims = [ { **row, "academic_kind": "Thesis" if kind == "person_thesis" else "Work", "academic_type": ( row.get("thesis_type") if kind == "person_thesis" else row.get("work_type") ), } ] elif kind in { "person_keyword", "person_category", "person_document_author", "person_start_year", "person_academic_year", }: claims = [row] elif kind in {"shared_keyword", "shared_category", "shared_document_author"}: field = { "shared_keyword": "keyword", "shared_category": "category", "shared_document_author": "document_author", }[kind] claims = [ { "person": row.get(side + "_person"), field: row.get(field), "document_path": row.get(side + "_document_path"), "source_url": row.get(side + "_source_url"), } for side in ["left", "right"] ] elif kind in {"academic_year_comparison", "start_year_comparison"}: names = {norm(n): n for n in oracle["people"]} people = [names.get(norm(n)) for n in row.get("people", [])] if not people or any(n is None for n in people): issues.append("invalid_comparison_people") elif kind == "academic_year_comparison": sets = [ { r["year"] for r in oracle["people"][n]["academics"] if r.get("year") is not None } for n in people ] years = sorted(set.intersection(*sets)) if row.get("shared_years") != years or row.get("same_academic_year") != bool(years): issues.append("incorrect_shared_academic_years") else: years = {oracle["people"][n].get("start_year") for n in people} same = len(years) == 1 and None not in years expected = {norm(n): oracle["people"][n].get("start_year") for n in people} actual = {norm(n): year for n, year in row.get("start_years", {}).items()} if actual != expected: issues.append("incorrect_individual_start_years") if row.get("same_start_year") != same or row.get("shared_start_year") != ( next(iter(years)) if same else None ): issues.append("incorrect_shared_start_year") else: issues.append("unsupported_fact_type") for claim in claims: issues.extend(claim_issues(claim, oracle, published_only=published_only)) if issues: failures.append({"fact_index": index, "fact_type": kind, "issues": sorted(set(issues))}) return { "source_checked": True, "all_facts_supported": not failures, "unsupported_facts": failures, }