94 lines
5.1 KiB
Python
94 lines
5.1 KiB
Python
"""Check every returned factual claim against the independent raw-source oracle.
|
|
|
|
Gold bindings define required coverage; this check also rejects false extras.
|
|
It does not use graph construction, query predictions or expected gold values.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from functools import lru_cache
|
|
from .source_oracle import load_source_oracle, norm
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def default_oracle():
|
|
return load_source_oracle()
|
|
|
|
|
|
def claim_issues(claim: dict, oracle: dict, *, published_only: bool = True) -> list[str]:
|
|
names = {norm(name): name for name in oracle['people']}
|
|
name = names.get(norm(claim.get('person')))
|
|
if name is None:
|
|
return ['unknown_person']
|
|
person = oracle['people'][name]
|
|
issues = []
|
|
if claim.get('start_year') is not None and claim['start_year'] != person.get('start_year'):
|
|
issues.append('unsupported_start_year')
|
|
path = claim.get('document_path')
|
|
documents = [p for p in person.get('documents', []) if path is None or p == path]
|
|
if path and not documents:
|
|
issues.append('document_belongs_to_other_person')
|
|
documents = [p for p in documents if not published_only or oracle['documents'][p].get('published', True)]
|
|
if path and not documents:
|
|
issues.append('missing_or_unpublished_source')
|
|
if claim.get('source_url') and not any(oracle['documents'][p]['source_url'] == claim['source_url'] for p in documents):
|
|
issues.append('incorrect_source_url')
|
|
metadata = {k: claim[k] for k in ['keyword','category','document_author'] if claim.get(k) is not None}
|
|
def supported(p, k, value):
|
|
d=oracle['documents'][p]
|
|
values = d['tags'] if k=='keyword' else d['categories'] if k=='category' else [d['author']]
|
|
return norm(value) in {norm(v) for v in values}
|
|
if metadata and not any(all(supported(p,k,v) for k,v in metadata.items()) for p in documents):
|
|
issues.append('unsupported_metadata_on_document')
|
|
academic = {k: claim[k] for k in ['title','year','academic_type','academic_kind'] if claim.get(k) is not None}
|
|
if academic and not any(
|
|
all(norm(record.get(k)) == norm(v) for k,v in academic.items())
|
|
and (not path or record['document_path']==path)
|
|
for record in person.get('academics', [])
|
|
):
|
|
issues.append('unsupported_academic_record')
|
|
return issues
|
|
|
|
|
|
def validate_returned_facts(result: dict, oracle: dict) -> dict:
|
|
failures = []
|
|
published_only = result.get('constraints', {}).get('published_only', True)
|
|
for index, row in enumerate(result.get('facts', [])):
|
|
kind=row.get('fact_type') if isinstance(row, dict) else None
|
|
claims=[]; issues=[]
|
|
if kind in {'person_thesis','person_work'}:
|
|
claims=[{**row,'academic_kind':'Thesis' if kind=='person_thesis' else 'Work',
|
|
'academic_type':row.get('thesis_type') if kind=='person_thesis' else row.get('work_type')}]
|
|
elif kind in {'person_keyword','person_category','person_document_author','person_start_year','person_academic_year'}:
|
|
claims=[row]
|
|
elif kind in {'shared_keyword','shared_category','shared_document_author'}:
|
|
field={'shared_keyword':'keyword','shared_category':'category','shared_document_author':'document_author'}[kind]
|
|
claims=[{'person':row.get(side+'_person'),field:row.get(field),
|
|
'document_path':row.get(side+'_document_path'),'source_url':row.get(side+'_source_url')}
|
|
for side in ['left','right']]
|
|
elif kind in {'academic_year_comparison','start_year_comparison'}:
|
|
names={norm(n):n for n in oracle['people']}
|
|
people=[names.get(norm(n)) for n in row.get('people', [])]
|
|
if not people or any(n is None for n in people):
|
|
issues.append('invalid_comparison_people')
|
|
elif kind=='academic_year_comparison':
|
|
sets=[{r['year'] for r in oracle['people'][n]['academics'] if r.get('year') is not None} for n in people]
|
|
years=sorted(set.intersection(*sets))
|
|
if row.get('shared_years')!=years or row.get('same_academic_year') != bool(years):
|
|
issues.append('incorrect_shared_academic_years')
|
|
else:
|
|
years={oracle['people'][n].get('start_year') for n in people}
|
|
same=len(years)==1 and None not in years
|
|
expected={norm(n):oracle['people'][n].get('start_year') for n in people}
|
|
actual={norm(n):year for n,year in row.get('start_years',{}).items()}
|
|
if actual != expected:
|
|
issues.append('incorrect_individual_start_years')
|
|
if row.get('same_start_year') != same or row.get('shared_start_year') != (next(iter(years)) if same else None):
|
|
issues.append('incorrect_shared_start_year')
|
|
else:
|
|
issues.append('unsupported_fact_type')
|
|
for claim in claims:
|
|
issues.extend(claim_issues(claim,oracle,published_only=published_only))
|
|
if issues:
|
|
failures.append({'fact_index':index,'fact_type':kind,'issues':sorted(set(issues))})
|
|
return {'source_checked':True,'all_facts_supported':not failures,'unsupported_facts':failures}
|