371 lines
21 KiB
Python
371 lines
21 KiB
Python
"""Connected, typed evidence checks shared by GraphRAG evaluation versions.
|
|
|
|
Facts are structured records emitted by fixed Cypher queries. They are expanded
|
|
into the very paths those query contracts establish; query text, resolved aliases
|
|
and context-only documents never create edges. Ordered paths must have compatible
|
|
endpoint types. Bindings are satisfied by one connected witness, not a bag of
|
|
independently occurring values.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import itertools
|
|
import re
|
|
import unicodedata
|
|
from collections import defaultdict
|
|
from typing import Any
|
|
|
|
|
|
RELATION_ENDPOINTS = {
|
|
'AUTHORED': ('Person', 'Thesis'),
|
|
'HAS_WORK': ('Person', 'Work'),
|
|
'HAS_DOCUMENT': ('Person', 'Document'),
|
|
'EVIDENCED_BY': ('Academic', 'Document'),
|
|
'HAS_KEYWORD': ('Document', 'Keyword'),
|
|
'IN_CATEGORY': ('Document', 'Category'),
|
|
'AUTHORED_DOCUMENT': ('DocumentAuthor', 'Document'),
|
|
'DESCRIBES': ('Document', 'Topic'),
|
|
}
|
|
|
|
|
|
def canonical(value: Any) -> str:
|
|
raw = unicodedata.normalize('NFKD', str(value if value is not None else ''))
|
|
return re.sub(r'\s+', ' ', re.sub(r'[^a-z0-9]+', ' ', ''.join(
|
|
character for character in raw.casefold() if not unicodedata.combining(character)))).strip()
|
|
|
|
|
|
def kind_matches(actual: str, expected: str) -> bool:
|
|
return actual in {'Thesis', 'Work'} if expected in {'Academic', 'Thesis|Work'} else actual in expected.split('|')
|
|
|
|
|
|
class EvidenceGraph:
|
|
def __init__(self, result: dict[str, Any]):
|
|
self.nodes: dict[tuple[str, str], dict] = {}
|
|
self.conflicts: set[tuple[str, str]] = set()
|
|
self.adjacency: dict[tuple[str, str], set[tuple[str, tuple[str, str]]]] = defaultdict(set)
|
|
for path in result.get('paths', []):
|
|
if not isinstance(path, dict):
|
|
continue
|
|
nodes, relationships = path.get('nodes', []), path.get('relationships', [])
|
|
if not isinstance(nodes, list) or len(relationships) != max(0, len(nodes) - 1):
|
|
continue
|
|
keys = [self.node(row.get('kind', ''), row) for row in nodes if isinstance(row, dict)]
|
|
if len(keys) != len(nodes):
|
|
continue
|
|
for left, relation, right in zip(keys, relationships, keys[1:]):
|
|
self.edge(left, relation, right)
|
|
for fact in result.get('facts', []):
|
|
if isinstance(fact, dict):
|
|
self.add_fact(fact)
|
|
|
|
def node(self, kind: str, properties: dict) -> tuple[str, str]:
|
|
properties = {key: value for key, value in properties.items() if value is not None}
|
|
if kind == 'Document':
|
|
identity = properties.get('path') or properties.get('document_path')
|
|
else:
|
|
identity = properties.get('element_id') or properties.get('id')
|
|
if not identity:
|
|
if kind in {'Thesis', 'Work'}:
|
|
identity = repr((properties.get('person'), properties.get('title'),
|
|
properties.get('year'), properties.get('document_path')))
|
|
else:
|
|
identity = canonical(properties.get('name'))
|
|
key = (kind, str(identity))
|
|
current = self.nodes.setdefault(key, {'kind': kind})
|
|
for field in {'name', 'title', 'year', 'start_year', 'thesis_type', 'work_type', 'path', 'source_url'}:
|
|
if field in current and field in properties and canonical(current[field]) != canonical(properties[field]):
|
|
self.conflicts.add(key)
|
|
for field, value in properties.items():
|
|
current.setdefault(field, value)
|
|
return key
|
|
|
|
def edge(self, left: tuple[str, str], relation: str, right: tuple[str, str]) -> None:
|
|
expected = RELATION_ENDPOINTS.get(relation)
|
|
if expected is None:
|
|
return
|
|
if kind_matches(left[0], expected[0]) and kind_matches(right[0], expected[1]):
|
|
pass
|
|
elif kind_matches(right[0], expected[0]) and kind_matches(left[0], expected[1]):
|
|
left, right = right, left
|
|
else:
|
|
return
|
|
self.adjacency[left].add((relation, right))
|
|
self.adjacency[right].add((relation, left))
|
|
|
|
def neighbors(self, key: tuple[str, str], relation: str, kind: str) -> list[tuple[str, str]]:
|
|
return [target for edge, target in self.adjacency.get(key, ())
|
|
if key not in self.conflicts and target not in self.conflicts
|
|
and edge in relation.split('|') and kind_matches(target[0], kind)]
|
|
|
|
def add_fact(self, fact: dict) -> None:
|
|
kind = fact.get('fact_type')
|
|
if kind in {'shared_keyword', 'shared_category', 'shared_document_author'}:
|
|
singular = {'shared_keyword': 'person_keyword', 'shared_category': 'person_category',
|
|
'shared_document_author': 'person_document_author'}[kind]
|
|
for side in ('left', 'right'):
|
|
self.add_fact({**fact, 'fact_type': singular, 'person': fact.get(side + '_person'),
|
|
'person_element_id': fact.get(side + '_person_element_id'),
|
|
'document_path': fact.get(side + '_document_path'),
|
|
'source_url': fact.get(side + '_source_url')})
|
|
return
|
|
if kind not in {'person_thesis', 'person_work', 'person_start_year', 'person_keyword',
|
|
'person_category', 'person_document_author'} or not fact.get('person'):
|
|
return
|
|
person = self.node('Person', {'name': fact['person'], 'element_id': fact.get('person_element_id'),
|
|
'start_year': fact.get('start_year') if kind == 'person_start_year' else None})
|
|
path = fact.get('document_path')
|
|
document = self.node('Document', {'path': path, 'source_url': fact.get('source_url')}) if path else None
|
|
if kind in {'person_thesis', 'person_work'}:
|
|
academic_kind = 'Thesis' if kind == 'person_thesis' else 'Work'
|
|
prefix = 'thesis' if academic_kind == 'Thesis' else 'work'
|
|
academic = self.node(academic_kind, {
|
|
'element_id': fact.get(prefix + '_element_id'), 'id': fact.get(prefix + '_id'),
|
|
'person': fact['person'], 'document_path': path, 'title': fact.get('title'),
|
|
'year': fact.get('year'), prefix + '_type': fact.get(prefix + '_type')})
|
|
self.edge(person, 'AUTHORED' if academic_kind == 'Thesis' else 'HAS_WORK', academic)
|
|
if document:
|
|
self.edge(academic, 'EVIDENCED_BY', document)
|
|
elif document:
|
|
self.edge(person, 'HAS_DOCUMENT', document)
|
|
if kind == 'person_keyword' and fact.get('keyword'):
|
|
target = self.node('Keyword', {'name': fact['keyword'], 'id': fact.get('keyword_id'),
|
|
'element_id': fact.get('keyword_element_id')})
|
|
self.edge(document, 'HAS_KEYWORD', target)
|
|
elif kind == 'person_category' and fact.get('category'):
|
|
target = self.node('Category', {'name': fact['category'],
|
|
'element_id': fact.get('category_element_id')})
|
|
self.edge(document, 'IN_CATEGORY', target)
|
|
elif kind == 'person_document_author' and fact.get('document_author'):
|
|
target = self.node('DocumentAuthor', {'name': fact['document_author'],
|
|
'id': fact.get('document_author_id'), 'element_id': fact.get('document_author_element_id')})
|
|
self.edge(target, 'AUTHORED_DOCUMENT', document)
|
|
if fact.get('category'):
|
|
category = self.node('Category', {'name': fact['category'],
|
|
'element_id': fact.get('category_element_id')})
|
|
self.edge(document, 'IN_CATEGORY', category)
|
|
|
|
def metadata_choices(self, person: tuple[str, str], binding: dict, field: str,
|
|
relation: str, kind: str) -> list[tuple[Any, Any]]:
|
|
if field not in binding:
|
|
return [(None, None)]
|
|
return [(document, node)
|
|
for document in self.neighbors(person, 'HAS_DOCUMENT', 'Document')
|
|
for node in self.neighbors(document, relation, kind)
|
|
if canonical(self.nodes[node].get('name')) == canonical(binding[field])]
|
|
|
|
def witnesses(self, binding: dict, *, require_provenance: bool) -> list[dict]:
|
|
people = [key for key, row in self.nodes.items() if key[0] == 'Person' and key not in self.conflicts
|
|
and ('person' not in binding or canonical(row.get('name')) == canonical(binding['person']))]
|
|
academic_required = any(field in binding for field in ['academic_kind', 'academic_type', 'title', 'year'])
|
|
witnesses = []
|
|
for person in people:
|
|
if 'start_year' in binding and canonical(self.nodes[person].get('start_year')) != canonical(binding['start_year']):
|
|
continue
|
|
academics = []
|
|
if academic_required:
|
|
for relation, kind in [('AUTHORED', 'Thesis'), ('HAS_WORK', 'Work')]:
|
|
for academic in self.neighbors(person, relation, kind):
|
|
row = self.nodes[academic]
|
|
if 'academic_kind' in binding and not kind_matches(kind, binding['academic_kind']):
|
|
continue
|
|
if any(canonical(row.get(field)) != canonical(binding[field])
|
|
for field in ['title', 'year'] if field in binding):
|
|
continue
|
|
type_field = 'thesis_type' if kind == 'Thesis' else 'work_type'
|
|
if 'academic_type' in binding and canonical(row.get(type_field)) != canonical(binding['academic_type']):
|
|
continue
|
|
docs = self.neighbors(academic, 'EVIDENCED_BY', 'Document')
|
|
academics.extend((academic, doc) for doc in docs)
|
|
if not docs and not require_provenance:
|
|
academics.append((academic, None))
|
|
else:
|
|
academics = [(None, None)]
|
|
choices = [self.metadata_choices(person, binding, *arguments) for arguments in [
|
|
('keyword', 'HAS_KEYWORD', 'Keyword'), ('category', 'IN_CATEGORY', 'Category'),
|
|
('document_author', 'AUTHORED_DOCUMENT', 'DocumentAuthor')]]
|
|
person_docs = self.neighbors(person, 'HAS_DOCUMENT', 'Document') if (
|
|
require_provenance and ('start_year' in binding or not academic_required
|
|
and not any(field in binding for field in ['keyword', 'category', 'document_author']))
|
|
) else [None]
|
|
for (academic, academic_document), keyword, category, author, person_document in itertools.product(academics, *choices, person_docs):
|
|
docs = [doc for doc in [academic_document, keyword[0], category[0], author[0], person_document] if doc]
|
|
if binding.get('same_document') and len(set(docs)) > 1:
|
|
continue
|
|
if category[0] and author[0] and category[0] != author[0]:
|
|
continue
|
|
if 'document_path' in binding and any(
|
|
self.nodes[doc].get('path') != binding['document_path'] for doc in docs
|
|
):
|
|
continue
|
|
witnesses.append({'person': person, 'academic': academic, 'academic_document': academic_document,
|
|
'person_document': person_document,
|
|
'keyword': keyword[1], 'keyword_document': keyword[0],
|
|
'category': category[1], 'category_document': category[0],
|
|
'document_author': author[1], 'author_document': author[0]})
|
|
return witnesses
|
|
|
|
def pattern_path(self, witness: dict, pattern: dict) -> list[dict] | None:
|
|
kinds, relations = pattern.get('node_kinds', []), pattern.get('relationships', [])
|
|
if not kinds or len(relations) != len(kinds) - 1 or not kind_matches('Person', kinds[0]):
|
|
return None
|
|
target_roles = {'Keyword': 'keyword', 'Category': 'category', 'DocumentAuthor': 'document_author',
|
|
'Thesis': 'academic', 'Work': 'academic', 'Academic': 'academic', 'Thesis|Work': 'academic'}
|
|
allowed = {key for key in witness.values() if key is not None}
|
|
required_target = witness.get(target_roles.get(kinds[-1], ''))
|
|
|
|
def visit(key, index, path):
|
|
if index == len(relations):
|
|
return path if required_target is None or key == required_target else None
|
|
for next_key in self.neighbors(key, relations[index], kinds[index + 1]):
|
|
if next_key not in allowed:
|
|
continue
|
|
result = visit(next_key, index + 1, path + [next_key])
|
|
if result:
|
|
return result
|
|
return None
|
|
|
|
found = visit(witness['person'], 0, [witness['person']])
|
|
return [self.nodes[key] for key in found] if found else None
|
|
|
|
|
|
def implicit_patterns(binding: dict, index: int) -> list[dict]:
|
|
patterns = []
|
|
if 'start_year' in binding or not any(field in binding for field in [
|
|
'academic_kind', 'academic_type', 'title', 'year', 'keyword', 'category', 'document_author'
|
|
]):
|
|
patterns.append({'binding_index': index, 'node_kinds': ['Person', 'Document'],
|
|
'relationships': ['HAS_DOCUMENT']})
|
|
if any(field in binding for field in ['academic_kind', 'academic_type', 'title', 'year']):
|
|
kind = binding.get('academic_kind', 'Academic')
|
|
relationships = ['AUTHORED'] if kind == 'Thesis' else ['HAS_WORK'] if kind == 'Work' else ['AUTHORED|HAS_WORK']
|
|
patterns.append({'binding_index': index, 'node_kinds': ['Person', kind, 'Document'],
|
|
'relationships': relationships + ['EVIDENCED_BY']})
|
|
for field, kind, relation in [('keyword', 'Keyword', 'HAS_KEYWORD'),
|
|
('category', 'Category', 'IN_CATEGORY'),
|
|
('document_author', 'DocumentAuthor', 'AUTHORED_DOCUMENT')]:
|
|
if field in binding:
|
|
patterns.append({'binding_index': index, 'node_kinds': ['Person', 'Document', kind],
|
|
'relationships': ['HAS_DOCUMENT', relation]})
|
|
return patterns
|
|
|
|
|
|
def legacy_bindings(question: dict) -> list[dict]:
|
|
"""Best-effort semantic conversion of schema2 annotations, never result data."""
|
|
graph = question.get('graph', {})
|
|
entities = defaultdict(list)
|
|
for row in graph.get('expected_entities', []):
|
|
entities[row.get('type')].append(row.get('value'))
|
|
people = entities['person'] or [row['value'] for row in graph.get('start_entities', []) if row['type'] == 'person']
|
|
people = people or [None]
|
|
years = entities['year']
|
|
properties = graph.get('expected_properties', [])
|
|
start_year = any(prop == 'Person.start_year' for prop in properties)
|
|
if not years and any('.year' in prop for prop in properties) and not start_year:
|
|
years = re.findall(r'\b20\d{2}\b', question.get('question', ''))
|
|
bindings = []
|
|
for person in people:
|
|
common = {'person': person} if person else {}
|
|
if len(years) == 1:
|
|
common['start_year' if start_year else 'year'] = years[0]
|
|
for field, entity_type in [('keyword', 'keyword'), ('category', 'category'), ('document_author', 'document_author')]:
|
|
if len(entities[entity_type]) == 1:
|
|
common[field] = entities[entity_type][0]
|
|
titles = [('Thesis', value) for value in entities['thesis_title']] + [('Work', value) for value in entities['work_title']]
|
|
if titles:
|
|
bindings.extend({**common, 'academic_kind': kind, 'title': title} for kind, title in titles)
|
|
elif len(entities['category']) > 1:
|
|
bindings.extend({**common, 'category': value} for value in entities['category'])
|
|
elif any(key != 'person' for key in common):
|
|
bindings.append(common)
|
|
return bindings
|
|
|
|
|
|
def positive_answer_facts(result: dict) -> list[dict]:
|
|
facts = []
|
|
for row in result.get('facts', []):
|
|
if not isinstance(row, dict):
|
|
facts.append(row)
|
|
continue
|
|
kind = row.get('fact_type', '')
|
|
if kind == 'academic_year_comparison' and row.get('same_academic_year') is False and not row.get('shared_years'):
|
|
continue
|
|
if kind == 'start_year_comparison' and row.get('same_start_year') is False and row.get('shared_start_year') is None:
|
|
continue
|
|
facts.append(row)
|
|
return facts
|
|
|
|
|
|
def evaluate_contract(question: dict, result: dict) -> dict:
|
|
graph = question.get('graph', {})
|
|
schema = graph.get('schema_version', 2)
|
|
negative = bool(graph.get('expected_no_answer') or question.get('should_answer') is False)
|
|
expected_status = graph.get('expected_status')
|
|
if negative:
|
|
status = result.get('status')
|
|
no_positive = not positive_answer_facts(result) and not result.get('paths')
|
|
aggregation = result.get('aggregation') or {}
|
|
if schema >= 3 and expected_status is None:
|
|
expected_status = 'no_answer'
|
|
status_ok = status == expected_status if expected_status else status in {'no_answer', 'ambiguous'}
|
|
reason = result.get('reason')
|
|
ok = bool(status_ok and isinstance(reason, str) and reason.strip()
|
|
and no_positive and not aggregation.get('count'))
|
|
return {'answerability_ok': ok, 'bindings_ok': True, 'binding_score': 1.0,
|
|
'paths_ok': True, 'path_score': 1.0, 'binding_results': [], 'path_results': [],
|
|
'expected_no_answer': True, 'actual_status': status}
|
|
status_ok = (result.get('status') == 'ok' if schema >= 3
|
|
else result.get('status') not in {'no_answer', 'ambiguous', 'error'})
|
|
bindings = graph.get('expected_bindings')
|
|
if bindings is None:
|
|
bindings = legacy_bindings(question)
|
|
empty_count_answer = (graph.get('expected_count') == 0 and graph.get('expected_entity_set') == []
|
|
and (result.get('aggregation') or {}).get('count') == 0
|
|
and (result.get('aggregation') or {}).get('complete') is True)
|
|
if schema >= 3 and not bindings and not empty_count_answer:
|
|
status_ok = False
|
|
evidence = EvidenceGraph(result)
|
|
explicit_patterns = graph.get('expected_path_patterns', [])
|
|
source_choices = graph.get('expected_evidence_documents', [])
|
|
binding_results, path_results = [], []
|
|
for index, binding in enumerate(bindings):
|
|
candidates = evidence.witnesses(binding, require_provenance=schema >= 3)
|
|
patterns = [p for p in explicit_patterns if p.get('binding_index') == index]
|
|
if not patterns and schema >= 3:
|
|
patterns = implicit_patterns(binding, index)
|
|
winner = None
|
|
for candidate in candidates:
|
|
checks = [evidence.pattern_path(candidate, pattern) for pattern in patterns]
|
|
source_ok = True
|
|
for choice in source_choices:
|
|
if choice.get('binding_index') != index:
|
|
continue
|
|
document = evidence.nodes.get(candidate.get(choice.get('role')), {})
|
|
if not any(document.get('path') == option.get('document_path')
|
|
and document.get('source_url') == option.get('source_url')
|
|
for option in choice.get('any_of', [])):
|
|
source_ok = False
|
|
break
|
|
if source_ok and all(check is not None for check in checks):
|
|
winner = checks
|
|
break
|
|
binding_results.append({'binding': binding, 'matched': bool(candidates) and winner is not None})
|
|
for pattern_index, pattern in enumerate(patterns):
|
|
path_results.append({'pattern': pattern, 'matched': winner is not None,
|
|
'witness': winner[pattern_index] if winner is not None else []})
|
|
bindings_ok = all(row['matched'] for row in binding_results)
|
|
paths_ok = all(row['matched'] for row in path_results)
|
|
# A malformed pattern referencing a nonexistent binding must not vanish.
|
|
if any(not isinstance(p.get('binding_index'), int) or p['binding_index'] not in range(len(bindings))
|
|
for p in explicit_patterns):
|
|
paths_ok = False
|
|
if any(not isinstance(p.get('binding_index'), int) or p['binding_index'] not in range(len(bindings))
|
|
or p.get('role') not in {'academic_document', 'keyword_document', 'category_document',
|
|
'author_document', 'person_document'} or not p.get('any_of')
|
|
for p in source_choices):
|
|
bindings_ok = False
|
|
return {'answerability_ok': status_ok, 'bindings_ok': bindings_ok,
|
|
'binding_score': sum(row['matched'] for row in binding_results) / len(binding_results) if binding_results else 1.0,
|
|
'paths_ok': paths_ok,
|
|
'path_score': sum(row['matched'] for row in path_results) / len(path_results) if path_results else float(paths_ok),
|
|
'binding_results': binding_results, 'path_results': path_results,
|
|
'expected_no_answer': False, 'actual_status': result.get('status', 'ok')}
|