1472 lines
32 KiB
Python
1472 lines
32 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import re
|
|
import statistics
|
|
import unicodedata
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .evidence_contract import evaluate_contract
|
|
from .source_validation import default_oracle, validate_returned_facts
|
|
|
|
|
|
VALID_SPLITS = (
|
|
"dev",
|
|
"test",
|
|
)
|
|
|
|
FACT_RELATIONS = {
|
|
"person_thesis": {
|
|
"AUTHORED",
|
|
"EVIDENCED_BY",
|
|
},
|
|
"person_work": {
|
|
"HAS_WORK",
|
|
"EVIDENCED_BY",
|
|
},
|
|
"person_keyword": {
|
|
"HAS_DOCUMENT",
|
|
"HAS_KEYWORD",
|
|
},
|
|
"shared_keyword": {
|
|
"HAS_DOCUMENT",
|
|
"HAS_KEYWORD",
|
|
},
|
|
"person_category": {
|
|
"HAS_DOCUMENT",
|
|
"IN_CATEGORY",
|
|
},
|
|
"shared_category": {
|
|
"HAS_DOCUMENT",
|
|
"IN_CATEGORY",
|
|
},
|
|
"person_document_author": {
|
|
"HAS_DOCUMENT",
|
|
"AUTHORED_DOCUMENT",
|
|
},
|
|
"shared_document_author": {
|
|
"HAS_DOCUMENT",
|
|
"AUTHORED_DOCUMENT",
|
|
},
|
|
}
|
|
|
|
|
|
def normalize_text(
|
|
value: Any,
|
|
) -> str:
|
|
decomposed = unicodedata.normalize(
|
|
"NFKD",
|
|
str(value or ""),
|
|
)
|
|
|
|
without_marks = "".join(
|
|
character
|
|
for character in decomposed
|
|
if not unicodedata.combining(
|
|
character
|
|
)
|
|
)
|
|
|
|
normalized = re.sub(
|
|
r"[^a-zA-Z0-9]+",
|
|
" ",
|
|
without_marks.casefold(),
|
|
)
|
|
|
|
return re.sub(
|
|
r"\s+",
|
|
" ",
|
|
normalized,
|
|
).strip()
|
|
|
|
|
|
def load_questions(
|
|
path: Path,
|
|
) -> list[dict[str, Any]]:
|
|
if not path.exists():
|
|
raise FileNotFoundError(
|
|
f"Benchmark neexistuje: {path}"
|
|
)
|
|
|
|
data = json.loads(
|
|
path.read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
|
|
if not isinstance(
|
|
data,
|
|
list,
|
|
):
|
|
raise ValueError(
|
|
"GraphRAG benchmark musí byť JSON pole."
|
|
)
|
|
|
|
questions: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
seen_ids: set[str] = set()
|
|
|
|
for index, item in enumerate(
|
|
data,
|
|
start=1,
|
|
):
|
|
if not isinstance(
|
|
item,
|
|
dict,
|
|
):
|
|
raise ValueError(
|
|
f"Položka {index} nie je JSON objekt."
|
|
)
|
|
|
|
question_id = item.get(
|
|
"id"
|
|
)
|
|
|
|
question = item.get(
|
|
"question"
|
|
)
|
|
|
|
split = item.get(
|
|
"split"
|
|
)
|
|
|
|
graph = item.get(
|
|
"graph"
|
|
)
|
|
|
|
if (
|
|
not isinstance(
|
|
question_id,
|
|
str,
|
|
)
|
|
or not question_id.strip()
|
|
):
|
|
raise ValueError(
|
|
f"Položka {index} nemá platné id."
|
|
)
|
|
|
|
if question_id in seen_ids:
|
|
raise ValueError(
|
|
f"Duplicitné id: {question_id}"
|
|
)
|
|
|
|
seen_ids.add(
|
|
question_id
|
|
)
|
|
|
|
if (
|
|
not isinstance(
|
|
question,
|
|
str,
|
|
)
|
|
or not question.strip()
|
|
):
|
|
raise ValueError(
|
|
f"{question_id}: chýba question."
|
|
)
|
|
|
|
if split not in VALID_SPLITS:
|
|
raise ValueError(
|
|
f"{question_id}: neplatný split "
|
|
f"{split!r}."
|
|
)
|
|
|
|
if not isinstance(
|
|
graph,
|
|
dict,
|
|
):
|
|
raise ValueError(
|
|
f"{question_id}: chýba graph."
|
|
)
|
|
|
|
if graph.get("schema_version") not in {2, 3}:
|
|
raise ValueError(
|
|
f"{question_id}: benchmark nie je "
|
|
"schema_version=2 alebo 3."
|
|
)
|
|
|
|
if graph.get('schema_version') == 3:
|
|
bindings = graph.get('expected_bindings')
|
|
if not isinstance(bindings, list) or any(not isinstance(row, dict) for row in bindings):
|
|
raise ValueError(f'{question_id}: schema3 requires expected_bindings as a list of objects.')
|
|
negative = graph.get('expected_no_answer') or item.get('should_answer') is False
|
|
if negative and bindings:
|
|
raise ValueError(f'{question_id}: no-answer cases cannot require positive bindings.')
|
|
if not negative and not bindings and graph.get('expected_count') != 0:
|
|
raise ValueError(f'{question_id}: a positive case needs bindings or an explicit zero count.')
|
|
patterns = graph.get('expected_path_patterns', [])
|
|
if not isinstance(patterns, list) or any(not isinstance(row, dict) for row in patterns):
|
|
raise ValueError(f'{question_id}: expected_path_patterns must be a list of objects.')
|
|
for pattern in patterns:
|
|
index = pattern.get('binding_index')
|
|
nodes, relations = pattern.get('node_kinds'), pattern.get('relationships')
|
|
if (not isinstance(index, int) or index not in range(len(bindings))
|
|
or not isinstance(nodes, list) or not nodes
|
|
or not isinstance(relations, list) or len(relations) != len(nodes) - 1
|
|
or any(not isinstance(value, str) for value in nodes + relations)):
|
|
raise ValueError(f'{question_id}: invalid connected path pattern.')
|
|
|
|
questions.append(
|
|
item
|
|
)
|
|
|
|
return questions
|
|
|
|
|
|
def filter_questions(
|
|
questions: list[
|
|
dict[str, Any]
|
|
],
|
|
*,
|
|
split: str,
|
|
ids: set[str] | None = None,
|
|
max_questions: int | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
selected = [
|
|
item
|
|
for item in questions
|
|
if (
|
|
split == "all"
|
|
or item.get("split")
|
|
== split
|
|
)
|
|
]
|
|
|
|
if ids:
|
|
selected = [
|
|
item
|
|
for item in selected
|
|
if str(
|
|
item.get("id")
|
|
)
|
|
in ids
|
|
]
|
|
|
|
if max_questions is not None:
|
|
selected = selected[
|
|
:max_questions
|
|
]
|
|
|
|
return selected
|
|
|
|
|
|
def evidence_payload(
|
|
result: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"paths": result.get(
|
|
"paths",
|
|
[],
|
|
),
|
|
"facts": result.get(
|
|
"facts",
|
|
[],
|
|
),
|
|
"aggregation": result.get("aggregation"),
|
|
}
|
|
|
|
|
|
def evidence_text(
|
|
result: dict[str, Any],
|
|
) -> str:
|
|
payload = evidence_payload(
|
|
result
|
|
)
|
|
|
|
rendered = json.dumps(
|
|
payload,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
default=str,
|
|
)
|
|
|
|
return normalize_text(
|
|
rendered
|
|
)
|
|
|
|
|
|
def value_is_present(
|
|
value: Any,
|
|
result: dict[str, Any],
|
|
) -> bool:
|
|
expected = normalize_text(
|
|
value
|
|
)
|
|
|
|
if not expected:
|
|
return False
|
|
|
|
actual = evidence_text(
|
|
result
|
|
)
|
|
|
|
return f" {expected} " in f" {actual} "
|
|
|
|
|
|
def ratio(
|
|
matched: int,
|
|
expected: int,
|
|
) -> float:
|
|
if expected <= 0:
|
|
return 1.0
|
|
|
|
return matched / expected
|
|
|
|
|
|
def entity_value_is_present(entity_type: str, value: str, result: dict[str, Any],
|
|
question: dict[str, Any] | None = None) -> bool:
|
|
"""Score typed answer evidence, never aliases copied from the question."""
|
|
rows = list(result.get("facts", []))
|
|
rows.extend(node for path in result.get("paths", []) for node in path.get("nodes", []))
|
|
fields = {
|
|
"person": ("person", "left_person", "right_person", "people"),
|
|
"keyword": ("keyword",),
|
|
"category": ("category",),
|
|
"document_author": ("document_author",),
|
|
"year": ("year",),
|
|
"start_year": ("start_year", "shared_start_year"),
|
|
"thesis_title": ("title",),
|
|
"work_title": ("title",),
|
|
}.get(entity_type, ())
|
|
node_kinds = {"person": "Person", "keyword": "Keyword", "category": "Category",
|
|
"document_author": "DocumentAuthor"}
|
|
expected = normalize_text(value)
|
|
start_year_context = bool(question and question.get('graph', {}).get('expected_properties')
|
|
and all(p == 'Person.start_year' for p in question['graph']['expected_properties']))
|
|
if entity_type == 'year' and start_year_context:
|
|
fields = ('start_year', 'shared_start_year')
|
|
for row in rows:
|
|
if entity_type == 'year' and not start_year_context and not (
|
|
row.get('kind') in {'Thesis', 'Work'}
|
|
or row.get('fact_type') in {'person_thesis', 'person_work'}
|
|
or row.get('thesis_type') is not None or row.get('work_type') is not None
|
|
):
|
|
continue
|
|
if entity_type == "thesis_title" and not (row.get("kind") == "Thesis" or row.get("thesis_type")):
|
|
continue
|
|
if entity_type == "work_title" and not (row.get("kind") == "Work" or row.get("work_type")):
|
|
continue
|
|
candidates = [row.get(field) for field in fields]
|
|
if row.get("kind") == node_kinds.get(entity_type) and row.get("kind") is not None:
|
|
candidates.append(row.get("name"))
|
|
for candidate in candidates:
|
|
values = candidate if isinstance(candidate, list) else [candidate]
|
|
if any(normalize_text(item) == expected for item in values if item is not None):
|
|
return True
|
|
return False
|
|
|
|
|
|
def expected_entity_values(
|
|
question: dict[str, Any],
|
|
) -> list[
|
|
tuple[str, str]
|
|
]:
|
|
graph = question.get(
|
|
"graph",
|
|
{}
|
|
)
|
|
|
|
result: list[
|
|
tuple[str, str]
|
|
] = []
|
|
|
|
for entity in graph.get(
|
|
"expected_entities",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
entity,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
entity_type = str(
|
|
entity.get(
|
|
"type"
|
|
)
|
|
or ""
|
|
)
|
|
|
|
value = entity.get(
|
|
"value"
|
|
)
|
|
|
|
if (
|
|
entity_type
|
|
and value is not None
|
|
):
|
|
result.append(
|
|
(
|
|
entity_type,
|
|
str(value),
|
|
)
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
def collect_actual_relationships(
|
|
result: dict[str, Any],
|
|
) -> set[str]:
|
|
relationships: set[str] = set()
|
|
|
|
for path in result.get(
|
|
"paths",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
path,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
for relationship in path.get(
|
|
"relationships",
|
|
[],
|
|
):
|
|
if relationship:
|
|
relationships.add(
|
|
str(relationship)
|
|
)
|
|
|
|
for fact in result.get(
|
|
"facts",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
fact,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
fact_type = str(
|
|
fact.get(
|
|
"fact_type"
|
|
)
|
|
or ""
|
|
)
|
|
|
|
relationships.update(
|
|
FACT_RELATIONS.get(
|
|
fact_type,
|
|
set(),
|
|
)
|
|
)
|
|
|
|
return relationships
|
|
|
|
|
|
def property_present(
|
|
property_name: str,
|
|
result: dict[str, Any],
|
|
) -> bool:
|
|
if "|" in property_name:
|
|
return any(
|
|
property_present(
|
|
part,
|
|
result,
|
|
)
|
|
for part in property_name.split(
|
|
"|"
|
|
)
|
|
)
|
|
|
|
property_name = property_name.strip()
|
|
|
|
facts = result.get(
|
|
"facts",
|
|
[],
|
|
)
|
|
|
|
path_nodes = []
|
|
|
|
for path in result.get(
|
|
"paths",
|
|
[],
|
|
):
|
|
if isinstance(
|
|
path,
|
|
dict,
|
|
):
|
|
path_nodes.extend(
|
|
path.get(
|
|
"nodes",
|
|
[],
|
|
)
|
|
)
|
|
|
|
rows = [
|
|
row
|
|
for row in (
|
|
list(facts)
|
|
+ list(path_nodes)
|
|
)
|
|
if isinstance(
|
|
row,
|
|
dict,
|
|
)
|
|
]
|
|
|
|
if property_name == "Person.start_year":
|
|
return any(
|
|
row.get(
|
|
"start_year"
|
|
)
|
|
is not None
|
|
for row in rows
|
|
)
|
|
|
|
if property_name == "Thesis.title":
|
|
return any(
|
|
(
|
|
row.get("kind")
|
|
== "Thesis"
|
|
or row.get("thesis_type")
|
|
is not None
|
|
or row.get("fact_type")
|
|
== "person_thesis"
|
|
)
|
|
and bool(
|
|
row.get(
|
|
"title"
|
|
)
|
|
)
|
|
for row in rows
|
|
)
|
|
|
|
if property_name == "Thesis.year":
|
|
return any(
|
|
(
|
|
row.get("kind")
|
|
== "Thesis"
|
|
or row.get("thesis_type")
|
|
is not None
|
|
or row.get("fact_type")
|
|
== "person_thesis"
|
|
)
|
|
and row.get(
|
|
"year"
|
|
)
|
|
is not None
|
|
for row in rows
|
|
)
|
|
|
|
if property_name == "Work.title":
|
|
return any(
|
|
(
|
|
row.get("kind")
|
|
== "Work"
|
|
or row.get("work_type")
|
|
is not None
|
|
or row.get("fact_type")
|
|
== "person_work"
|
|
)
|
|
and bool(
|
|
row.get(
|
|
"title"
|
|
)
|
|
)
|
|
for row in rows
|
|
)
|
|
|
|
if property_name == "Work.year":
|
|
return any(
|
|
(
|
|
row.get("kind")
|
|
== "Work"
|
|
or row.get("work_type")
|
|
is not None
|
|
or row.get("fact_type")
|
|
== "person_work"
|
|
)
|
|
and row.get(
|
|
"year"
|
|
)
|
|
is not None
|
|
for row in rows
|
|
)
|
|
|
|
return False
|
|
|
|
|
|
def collect_document_paths(
|
|
result: dict[str, Any],
|
|
) -> set[str]:
|
|
paths: set[str] = set()
|
|
|
|
for document in result.get(
|
|
"documents",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
document,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
path = document.get(
|
|
"document_path"
|
|
)
|
|
|
|
if path:
|
|
paths.add(
|
|
str(path)
|
|
)
|
|
|
|
return paths
|
|
|
|
|
|
def collect_source_urls(
|
|
result: dict[str, Any],
|
|
) -> set[str]:
|
|
urls: set[str] = set()
|
|
|
|
def add(
|
|
value: Any,
|
|
) -> None:
|
|
if value:
|
|
urls.add(
|
|
str(value)
|
|
)
|
|
|
|
for document in result.get(
|
|
"documents",
|
|
[],
|
|
):
|
|
if isinstance(
|
|
document,
|
|
dict,
|
|
):
|
|
add(
|
|
document.get(
|
|
"source_url"
|
|
)
|
|
)
|
|
|
|
for fact in result.get(
|
|
"facts",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
fact,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
for key, value in fact.items():
|
|
if key.endswith(
|
|
"source_url"
|
|
):
|
|
add(
|
|
value
|
|
)
|
|
|
|
for path in result.get(
|
|
"paths",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
path,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
for node in path.get(
|
|
"nodes",
|
|
[],
|
|
):
|
|
if isinstance(
|
|
node,
|
|
dict,
|
|
):
|
|
add(
|
|
node.get(
|
|
"source_url"
|
|
)
|
|
)
|
|
|
|
return urls
|
|
|
|
|
|
def normalized_set(
|
|
values: list[Any] | set[Any],
|
|
) -> set[str]:
|
|
return {
|
|
normalize_text(
|
|
value
|
|
)
|
|
for value in values
|
|
if normalize_text(
|
|
value
|
|
)
|
|
}
|
|
|
|
|
|
def actual_entity_set(
|
|
question: dict[str, Any],
|
|
result: dict[str, Any],
|
|
) -> set[str]:
|
|
category = str(
|
|
question.get(
|
|
"category"
|
|
)
|
|
or ""
|
|
)
|
|
|
|
values: set[str] = set()
|
|
|
|
field = question.get('graph', {}).get('entity_set_field')
|
|
if category == "person_multiple_categories" or field == 'category':
|
|
for fact in result.get(
|
|
"facts",
|
|
[],
|
|
):
|
|
if (
|
|
isinstance(
|
|
fact,
|
|
dict,
|
|
)
|
|
and fact.get(
|
|
"fact_type"
|
|
)
|
|
== "person_category"
|
|
and fact.get(
|
|
"category"
|
|
)
|
|
):
|
|
values.add(
|
|
str(
|
|
fact[
|
|
"category"
|
|
]
|
|
)
|
|
)
|
|
|
|
return values
|
|
|
|
if category == "candidate_set_aggregation" or field == 'person':
|
|
for fact in result.get(
|
|
"facts",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
fact,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
if fact.get(
|
|
"fact_type"
|
|
) in {
|
|
"person_thesis",
|
|
"person_work",
|
|
"person_academic_year",
|
|
}:
|
|
person = fact.get(
|
|
"person"
|
|
)
|
|
|
|
if person:
|
|
values.add(
|
|
str(person)
|
|
)
|
|
|
|
return values
|
|
|
|
if field == 'keyword':
|
|
return {str(fact['keyword']) for fact in result.get('facts', [])
|
|
if isinstance(fact, dict) and fact.get('fact_type') in {'person_keyword', 'shared_keyword'}
|
|
and fact.get('keyword')}
|
|
|
|
return values
|
|
|
|
|
|
def entity_set_metrics(
|
|
question: dict[str, Any],
|
|
result: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
graph = question.get(
|
|
"graph",
|
|
{}
|
|
)
|
|
|
|
expected = graph.get(
|
|
"expected_entity_set"
|
|
)
|
|
|
|
if not isinstance(
|
|
expected,
|
|
list,
|
|
):
|
|
return {
|
|
"entity_set_expected": [],
|
|
"entity_set_actual": [],
|
|
"entity_set_mode": None,
|
|
"entity_set_ok": True,
|
|
}
|
|
|
|
actual = actual_entity_set(
|
|
question,
|
|
result,
|
|
)
|
|
|
|
expected_normalized = (
|
|
normalized_set(
|
|
expected
|
|
)
|
|
)
|
|
|
|
actual_normalized = (
|
|
normalized_set(
|
|
actual
|
|
)
|
|
)
|
|
|
|
mode = str(
|
|
graph.get(
|
|
"entity_set_mode"
|
|
)
|
|
or "contains"
|
|
)
|
|
|
|
if mode == "exact":
|
|
ok = (
|
|
actual_normalized
|
|
== expected_normalized
|
|
)
|
|
else:
|
|
ok = (
|
|
expected_normalized
|
|
<= actual_normalized
|
|
)
|
|
|
|
return {
|
|
"entity_set_expected": (
|
|
sorted(
|
|
expected_normalized
|
|
)
|
|
),
|
|
"entity_set_actual": (
|
|
sorted(
|
|
actual_normalized
|
|
)
|
|
),
|
|
"entity_set_mode": mode,
|
|
"entity_set_ok": ok,
|
|
}
|
|
|
|
|
|
def evaluate_result(
|
|
question: dict[str, Any],
|
|
result: dict[str, Any],
|
|
*,
|
|
latency_seconds: float,
|
|
error: str | None = None,
|
|
source_oracle: dict | None = None,
|
|
) -> dict[str, Any]:
|
|
graph = question.get(
|
|
"graph",
|
|
{}
|
|
)
|
|
|
|
expected_values = [
|
|
str(value)
|
|
for value in question.get(
|
|
"expected_answer_contains",
|
|
[],
|
|
)
|
|
]
|
|
|
|
matched_values = [
|
|
value
|
|
for value in expected_values
|
|
if value_is_present(
|
|
value,
|
|
result,
|
|
)
|
|
]
|
|
|
|
entities = expected_entity_values(
|
|
question
|
|
)
|
|
|
|
matched_entities = [
|
|
{
|
|
"type": entity_type,
|
|
"value": value,
|
|
}
|
|
for entity_type, value
|
|
in entities
|
|
if entity_value_is_present(
|
|
entity_type,
|
|
value,
|
|
result,
|
|
question,
|
|
)
|
|
]
|
|
|
|
expected_relations = [
|
|
str(value)
|
|
for value in graph.get(
|
|
"expected_relations",
|
|
[],
|
|
)
|
|
]
|
|
|
|
actual_relations = (
|
|
collect_actual_relationships(
|
|
result
|
|
)
|
|
)
|
|
|
|
matched_relations = [
|
|
relationship
|
|
for relationship
|
|
in expected_relations
|
|
if relationship
|
|
in actual_relations
|
|
]
|
|
|
|
expected_properties = [
|
|
str(value)
|
|
for value in graph.get(
|
|
"expected_properties",
|
|
[],
|
|
)
|
|
]
|
|
|
|
matched_properties = [
|
|
value
|
|
for value
|
|
in expected_properties
|
|
if property_present(
|
|
value,
|
|
result,
|
|
)
|
|
]
|
|
|
|
relation_choices = graph.get(
|
|
"expected_relation_choices",
|
|
[],
|
|
)
|
|
|
|
relation_choice_results = []
|
|
|
|
if isinstance(
|
|
relation_choices,
|
|
list,
|
|
):
|
|
for choice in relation_choices:
|
|
values = (
|
|
choice
|
|
if isinstance(
|
|
choice,
|
|
list,
|
|
)
|
|
else [choice]
|
|
)
|
|
|
|
clean = [
|
|
str(value)
|
|
for value in values
|
|
if value
|
|
]
|
|
|
|
relation_choice_results.append(
|
|
{
|
|
"choices": clean,
|
|
"matched": any(
|
|
value
|
|
in actual_relations
|
|
for value in clean
|
|
),
|
|
}
|
|
)
|
|
|
|
expected_documents = {
|
|
str(value)
|
|
for value in question.get(
|
|
"expected_documents",
|
|
[],
|
|
)
|
|
}
|
|
|
|
actual_documents = (
|
|
collect_document_paths(
|
|
result
|
|
)
|
|
)
|
|
|
|
matched_documents = (
|
|
expected_documents
|
|
& actual_documents
|
|
)
|
|
|
|
expected_sources = {
|
|
str(value)
|
|
for value in question.get(
|
|
"expected_source_urls",
|
|
[],
|
|
)
|
|
}
|
|
|
|
actual_sources = (
|
|
collect_source_urls(
|
|
result
|
|
)
|
|
)
|
|
|
|
matched_sources = (
|
|
expected_sources
|
|
& actual_sources
|
|
)
|
|
|
|
set_metrics = entity_set_metrics(
|
|
question,
|
|
result,
|
|
)
|
|
|
|
value_score = ratio(
|
|
len(matched_values),
|
|
len(expected_values),
|
|
)
|
|
|
|
entity_score = ratio(
|
|
len(matched_entities),
|
|
len(entities),
|
|
)
|
|
|
|
relation_score = ratio(
|
|
len(matched_relations),
|
|
len(expected_relations),
|
|
)
|
|
|
|
property_score = ratio(
|
|
len(matched_properties),
|
|
len(expected_properties),
|
|
)
|
|
|
|
relation_choice_score = ratio(
|
|
sum(
|
|
1
|
|
for item
|
|
in relation_choice_results
|
|
if item["matched"]
|
|
),
|
|
len(
|
|
relation_choice_results
|
|
),
|
|
)
|
|
|
|
document_recall = ratio(
|
|
len(matched_documents),
|
|
len(expected_documents),
|
|
)
|
|
|
|
source_recall = ratio(
|
|
len(matched_sources),
|
|
len(expected_sources),
|
|
)
|
|
|
|
expected_count = graph.get("expected_count")
|
|
aggregation = result.get("aggregation") or {}
|
|
count_ok = True
|
|
if expected_count is not None:
|
|
actual_count = aggregation.get('count')
|
|
actual_values = normalized_set(actual_entity_set(question, result))
|
|
declared_values = aggregation.get('people')
|
|
count_ok = bool(
|
|
aggregation.get('complete') is True
|
|
and isinstance(actual_count, int) and not isinstance(actual_count, bool)
|
|
and actual_count == expected_count
|
|
and len(actual_values) == actual_count
|
|
and isinstance(declared_values, list)
|
|
and normalized_set(declared_values) == actual_values
|
|
and len(declared_values) == actual_count
|
|
)
|
|
complete = not result.get("truncated", False) and result.get('complete', True) is True
|
|
# Real source-grounded benchmarks always run the independent check. Small
|
|
# synthetic unit fixtures without source_evidence can inject their own oracle.
|
|
if source_oracle is None and question.get('source_evidence'):
|
|
source_oracle = default_oracle()
|
|
source_check = (validate_returned_facts(result, source_oracle) if source_oracle is not None else
|
|
{'source_checked':False,'all_facts_supported':None,'unsupported_facts':[]})
|
|
contract = evaluate_contract(question, result)
|
|
strict = all(
|
|
(
|
|
error is None,
|
|
source_check['all_facts_supported'] is not False,
|
|
contract['answerability_ok'],
|
|
contract['bindings_ok'],
|
|
contract['paths_ok'],
|
|
complete,
|
|
count_ok,
|
|
value_score == 1.0,
|
|
entity_score == 1.0,
|
|
relation_score == 1.0,
|
|
property_score == 1.0,
|
|
relation_choice_score
|
|
== 1.0,
|
|
document_recall == 1.0,
|
|
source_recall == 1.0,
|
|
bool(
|
|
set_metrics[
|
|
"entity_set_ok"
|
|
]
|
|
),
|
|
)
|
|
)
|
|
|
|
return {
|
|
**contract,
|
|
**source_check,
|
|
"count_ok": count_ok,
|
|
"expected_count": expected_count,
|
|
"actual_count": aggregation.get("count"),
|
|
"complete": complete,
|
|
"id": question.get(
|
|
"id"
|
|
),
|
|
"split": question.get(
|
|
"split"
|
|
),
|
|
"category": question.get(
|
|
"category"
|
|
),
|
|
"difficulty": question.get(
|
|
"difficulty"
|
|
),
|
|
"question": question.get(
|
|
"question"
|
|
),
|
|
"latency_seconds": round(
|
|
latency_seconds,
|
|
6,
|
|
),
|
|
"error": error,
|
|
"strict": strict,
|
|
"value_score": round(
|
|
value_score,
|
|
6,
|
|
),
|
|
"entity_score": round(
|
|
entity_score,
|
|
6,
|
|
),
|
|
"relation_score": round(
|
|
relation_score,
|
|
6,
|
|
),
|
|
"property_score": round(
|
|
property_score,
|
|
6,
|
|
),
|
|
"relation_choice_score": round(
|
|
relation_choice_score,
|
|
6,
|
|
),
|
|
"document_recall": round(
|
|
document_recall,
|
|
6,
|
|
),
|
|
"source_recall": round(
|
|
source_recall,
|
|
6,
|
|
),
|
|
"expected_values": (
|
|
expected_values
|
|
),
|
|
"matched_values": (
|
|
matched_values
|
|
),
|
|
"expected_entities": [
|
|
{
|
|
"type": entity_type,
|
|
"value": value,
|
|
}
|
|
for entity_type, value
|
|
in entities
|
|
],
|
|
"matched_entities": (
|
|
matched_entities
|
|
),
|
|
"expected_relations": (
|
|
expected_relations
|
|
),
|
|
"actual_relations": sorted(
|
|
actual_relations
|
|
),
|
|
"matched_relations": (
|
|
matched_relations
|
|
),
|
|
"expected_properties": (
|
|
expected_properties
|
|
),
|
|
"matched_properties": (
|
|
matched_properties
|
|
),
|
|
"relation_choices": (
|
|
relation_choice_results
|
|
),
|
|
"expected_documents": sorted(
|
|
expected_documents
|
|
),
|
|
"actual_documents": sorted(
|
|
actual_documents
|
|
),
|
|
"matched_documents": sorted(
|
|
matched_documents
|
|
),
|
|
"expected_source_urls": sorted(
|
|
expected_sources
|
|
),
|
|
"actual_source_urls": sorted(
|
|
actual_sources
|
|
),
|
|
"matched_source_urls": sorted(
|
|
matched_sources
|
|
),
|
|
**set_metrics,
|
|
"strategies": list(
|
|
result.get(
|
|
"strategies",
|
|
[],
|
|
)
|
|
),
|
|
"fact_types": sorted(
|
|
{
|
|
str(
|
|
fact.get(
|
|
"fact_type"
|
|
)
|
|
)
|
|
for fact in result.get(
|
|
"facts",
|
|
[],
|
|
)
|
|
if (
|
|
isinstance(
|
|
fact,
|
|
dict,
|
|
)
|
|
and fact.get(
|
|
"fact_type"
|
|
)
|
|
)
|
|
}
|
|
),
|
|
}
|
|
|
|
|
|
def percentile(
|
|
values: list[float],
|
|
fraction: float,
|
|
) -> float:
|
|
if not values:
|
|
return 0.0
|
|
|
|
ordered = sorted(
|
|
values
|
|
)
|
|
|
|
if len(ordered) == 1:
|
|
return ordered[0]
|
|
|
|
position = (
|
|
(len(ordered) - 1)
|
|
* fraction
|
|
)
|
|
|
|
lower = math.floor(
|
|
position
|
|
)
|
|
|
|
upper = math.ceil(
|
|
position
|
|
)
|
|
|
|
if lower == upper:
|
|
return ordered[
|
|
lower
|
|
]
|
|
|
|
weight = (
|
|
position - lower
|
|
)
|
|
|
|
return (
|
|
ordered[lower]
|
|
* (1.0 - weight)
|
|
+ ordered[upper]
|
|
* weight
|
|
)
|
|
|
|
|
|
def aggregate_rows(
|
|
rows: list[
|
|
dict[str, Any]
|
|
],
|
|
) -> dict[str, Any]:
|
|
if not rows:
|
|
return {
|
|
"questions": 0,
|
|
"strict_count": 0,
|
|
"strict_accuracy": 0.0,
|
|
"error_count": 0,
|
|
"value_score": 0.0,
|
|
"entity_score": 0.0,
|
|
"relation_score": 0.0,
|
|
"property_score": 0.0,
|
|
"relation_choice_score": 0.0,
|
|
"document_recall": 0.0,
|
|
"source_recall": 0.0,
|
|
"binding_score": 0.0,
|
|
"path_score": 0.0,
|
|
"answerability_accuracy": 0.0,
|
|
"latency_mean_seconds": 0.0,
|
|
"latency_median_seconds": 0.0,
|
|
"latency_p95_seconds": 0.0,
|
|
}
|
|
|
|
latencies = [
|
|
float(
|
|
row[
|
|
"latency_seconds"
|
|
]
|
|
)
|
|
for row in rows
|
|
]
|
|
|
|
def mean(
|
|
key: str,
|
|
) -> float:
|
|
return round(
|
|
statistics.fmean(
|
|
float(
|
|
row[key]
|
|
)
|
|
for row in rows
|
|
),
|
|
6,
|
|
)
|
|
|
|
strict_count = sum(
|
|
bool(
|
|
row["strict"]
|
|
)
|
|
for row in rows
|
|
)
|
|
|
|
return {
|
|
"questions": len(
|
|
rows
|
|
),
|
|
'source_checked_count':sum(row.get('source_checked') is True for row in rows),
|
|
'all_facts_supported_count':sum(row.get('all_facts_supported') is True for row in rows),
|
|
"strict_count": (
|
|
strict_count
|
|
),
|
|
"strict_accuracy": round(
|
|
strict_count
|
|
/ len(rows),
|
|
6,
|
|
),
|
|
"error_count": sum(
|
|
bool(
|
|
row.get(
|
|
"error"
|
|
)
|
|
)
|
|
for row in rows
|
|
),
|
|
"value_score": mean(
|
|
"value_score"
|
|
),
|
|
"entity_score": mean(
|
|
"entity_score"
|
|
),
|
|
"relation_score": mean(
|
|
"relation_score"
|
|
),
|
|
"property_score": mean(
|
|
"property_score"
|
|
),
|
|
"relation_choice_score": (
|
|
mean(
|
|
"relation_choice_score"
|
|
)
|
|
),
|
|
"document_recall": mean(
|
|
"document_recall"
|
|
),
|
|
"source_recall": mean(
|
|
"source_recall"
|
|
),
|
|
"binding_score": round(statistics.fmean(float(row.get('binding_score', 1.0)) for row in rows), 6),
|
|
"path_score": round(statistics.fmean(float(row.get('path_score', 1.0)) for row in rows), 6),
|
|
"answerability_accuracy": round(statistics.fmean(bool(row.get('answerability_ok', True)) for row in rows), 6),
|
|
"latency_mean_seconds": round(
|
|
statistics.fmean(
|
|
latencies
|
|
),
|
|
6,
|
|
),
|
|
"latency_median_seconds": round(
|
|
statistics.median(
|
|
latencies
|
|
),
|
|
6,
|
|
),
|
|
"latency_p95_seconds": round(
|
|
percentile(
|
|
latencies,
|
|
0.95,
|
|
),
|
|
6,
|
|
),
|
|
}
|
|
|
|
|
|
def aggregate_by_field(
|
|
rows: list[
|
|
dict[str, Any]
|
|
],
|
|
field: str,
|
|
) -> dict[
|
|
str,
|
|
dict[str, Any],
|
|
]:
|
|
grouped: dict[
|
|
str,
|
|
list[dict[str, Any]],
|
|
] = defaultdict(list)
|
|
|
|
for row in rows:
|
|
key = str(
|
|
row.get(
|
|
field
|
|
)
|
|
or "<missing>"
|
|
)
|
|
|
|
grouped[
|
|
key
|
|
].append(
|
|
row
|
|
)
|
|
|
|
return {
|
|
key: aggregate_rows(
|
|
group_rows
|
|
)
|
|
for key, group_rows
|
|
in sorted(
|
|
grouped.items()
|
|
)
|
|
}
|