681 lines
16 KiB
Python
681 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from ..graph_db import (
|
|
create_neo4j_driver,
|
|
get_neo4j_settings,
|
|
)
|
|
from ..graph_search import (
|
|
DEFAULT_LIMIT,
|
|
DEFAULT_MAX_ENTITIES,
|
|
search_graph_with_session,
|
|
)
|
|
from .graphrag_evaluator import (
|
|
aggregate_by_field,
|
|
aggregate_rows,
|
|
evaluate_result,
|
|
filter_questions,
|
|
load_questions,
|
|
)
|
|
from .reproducibility import graph_fingerprint, run_metadata, sha256_file, verify_freeze
|
|
from .source_oracle import load_source_oracle
|
|
|
|
|
|
EVALUATION_DIR = Path(
|
|
__file__
|
|
).resolve().parent
|
|
|
|
QUESTIONS_PATH = (
|
|
EVALUATION_DIR
|
|
/ "graphrag_questions_v4.json"
|
|
)
|
|
|
|
RESULTS_DIR = (
|
|
EVALUATION_DIR
|
|
/ "results"
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description=(
|
|
"Vyhodnotenie GraphRAG vyhľadávania "
|
|
"proti graphrag_questions.json."
|
|
)
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--questions",
|
|
type=Path,
|
|
default=QUESTIONS_PATH,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
type=Path,
|
|
default=RESULTS_DIR / 'development_v4',
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--split",
|
|
choices=(
|
|
"dev",
|
|
"test",
|
|
"all",
|
|
),
|
|
default="dev",
|
|
help=(
|
|
"Predvolene sa vyhodnocuje iba DEV. "
|
|
"TEST používaj až po zmrazení GraphRAG."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=DEFAULT_LIMIT,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-entities",
|
|
type=int,
|
|
default=DEFAULT_MAX_ENTITIES,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--ids",
|
|
type=str,
|
|
default="",
|
|
help=(
|
|
"Voliteľný zoznam ID oddelených čiarkou."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-questions",
|
|
type=int,
|
|
default=None,
|
|
help=(
|
|
"Voliteľne vyhodnotí iba prvých N "
|
|
"otázok z vybraného splitu."
|
|
),
|
|
)
|
|
|
|
parser.add_argument('--audit', type=Path, help='Predeclared source-consistency audit.')
|
|
parser.add_argument('--freeze-manifest', type=Path,
|
|
help='Required for TEST/all; verifies code, data, graph and parameters.')
|
|
parser.add_argument('--overwrite', action='store_true',
|
|
help='Replace a previous DEV output; frozen TEST outputs are immutable.')
|
|
args = parser.parse_args()
|
|
if args.split in {'test', 'all'} and args.freeze_manifest is None:
|
|
parser.error('TEST/all requires --freeze-manifest after complete DEV evaluation.')
|
|
if args.split in {'test', 'all'} and (args.ids or args.max_questions):
|
|
parser.error('Frozen holdout evaluation must cover the complete selected split.')
|
|
if args.split in {'test', 'all'} and args.overwrite:
|
|
parser.error('Use a fresh output directory to reproduce a frozen holdout.')
|
|
|
|
if args.limit < 1:
|
|
parser.error(
|
|
"--limit musí byť aspoň 1."
|
|
)
|
|
|
|
if args.max_entities < 1:
|
|
parser.error(
|
|
"--max-entities musí byť aspoň 1."
|
|
)
|
|
|
|
if (
|
|
args.max_questions
|
|
is not None
|
|
and args.max_questions < 1
|
|
):
|
|
parser.error(
|
|
"--max-questions musí byť aspoň 1."
|
|
)
|
|
|
|
return args
|
|
|
|
|
|
def save_json(
|
|
path: Path,
|
|
payload: dict[str, Any],
|
|
) -> None:
|
|
path.parent.mkdir(
|
|
parents=True,
|
|
exist_ok=True,
|
|
)
|
|
|
|
path.write_text(
|
|
json.dumps(
|
|
payload,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def save_csv(
|
|
path: Path,
|
|
rows: list[
|
|
dict[str, Any]
|
|
],
|
|
) -> None:
|
|
path.parent.mkdir(
|
|
parents=True,
|
|
exist_ok=True,
|
|
)
|
|
|
|
fieldnames = [
|
|
"complete",
|
|
"answerability_ok",
|
|
"bindings_ok",
|
|
"binding_score",
|
|
"paths_ok",
|
|
"path_score",
|
|
"count_ok",
|
|
"expected_count",
|
|
"actual_count",
|
|
"id",
|
|
"split",
|
|
"category",
|
|
"difficulty",
|
|
"strict",
|
|
"value_score",
|
|
"entity_score",
|
|
"relation_score",
|
|
"property_score",
|
|
"relation_choice_score",
|
|
"document_recall",
|
|
"source_recall",
|
|
"entity_set_ok",
|
|
"latency_seconds",
|
|
"error",
|
|
"question",
|
|
"expected_values",
|
|
"matched_values",
|
|
"expected_relations",
|
|
"actual_relations",
|
|
"expected_documents",
|
|
"actual_documents",
|
|
"expected_source_urls",
|
|
"actual_source_urls",
|
|
"fact_types",
|
|
"strategies",
|
|
"binding_results",
|
|
"path_results",
|
|
]
|
|
|
|
with path.open(
|
|
"w",
|
|
encoding="utf-8",
|
|
newline="",
|
|
) as file:
|
|
writer = csv.DictWriter(
|
|
file,
|
|
fieldnames=fieldnames,
|
|
lineterminator='\n',
|
|
)
|
|
|
|
writer.writeheader()
|
|
|
|
for row in rows:
|
|
output = {}
|
|
|
|
for field in fieldnames:
|
|
value = row.get(
|
|
field
|
|
)
|
|
|
|
if isinstance(
|
|
value,
|
|
(
|
|
list,
|
|
dict,
|
|
),
|
|
):
|
|
value = json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
)
|
|
|
|
output[
|
|
field
|
|
] = value
|
|
|
|
writer.writerow(
|
|
output
|
|
)
|
|
|
|
|
|
def print_summary(
|
|
summary: dict[str, Any],
|
|
*,
|
|
split: str,
|
|
) -> None:
|
|
print()
|
|
print(
|
|
"=" * 78
|
|
)
|
|
|
|
print(
|
|
"GRAPHRAG EVALUATION"
|
|
)
|
|
|
|
print(
|
|
"=" * 78
|
|
)
|
|
|
|
print(
|
|
f"Split: {split}"
|
|
)
|
|
|
|
print(
|
|
f"Questions: "
|
|
f"{summary['questions']}"
|
|
)
|
|
|
|
print(
|
|
f"Strict: "
|
|
f"{summary['strict_count']}/"
|
|
f"{summary['questions']} "
|
|
f"({summary['strict_accuracy']:.3%})"
|
|
)
|
|
|
|
print(
|
|
f"Errors: "
|
|
f"{summary['error_count']}"
|
|
)
|
|
|
|
print(
|
|
f"Value score: "
|
|
f"{summary['value_score']:.3f}"
|
|
)
|
|
|
|
print(
|
|
f"Entity score: "
|
|
f"{summary['entity_score']:.3f}"
|
|
)
|
|
|
|
print(
|
|
f"Relation score: "
|
|
f"{summary['relation_score']:.3f}"
|
|
)
|
|
|
|
print(
|
|
f"Property score: "
|
|
f"{summary['property_score']:.3f}"
|
|
)
|
|
|
|
print(
|
|
f"Relation choices: "
|
|
f"{summary['relation_choice_score']:.3f}"
|
|
)
|
|
|
|
print(
|
|
f"Document recall: "
|
|
f"{summary['document_recall']:.3f}"
|
|
)
|
|
|
|
print(
|
|
f"Source recall: "
|
|
f"{summary['source_recall']:.3f}"
|
|
)
|
|
|
|
print(
|
|
f"Latency mean: "
|
|
f"{summary['latency_mean_seconds']:.4f} s"
|
|
)
|
|
|
|
print(
|
|
f"Latency median: "
|
|
f"{summary['latency_median_seconds']:.4f} s"
|
|
)
|
|
|
|
print(
|
|
f"Latency p95: "
|
|
f"{summary['latency_p95_seconds']:.4f} s"
|
|
)
|
|
|
|
print(
|
|
"=" * 78
|
|
)
|
|
|
|
|
|
def validate_output_destination(args: argparse.Namespace) -> None:
|
|
suffix = args.split
|
|
if args.ids:
|
|
suffix += '_selected'
|
|
elif args.max_questions:
|
|
suffix += f'_first{args.max_questions}'
|
|
if not args.overwrite:
|
|
for extension in ('json', 'csv'):
|
|
path = args.output_dir / f'graphrag_{suffix}.{extension}'
|
|
if path.exists():
|
|
raise RuntimeError(f'Results already exist; use a fresh output directory: {path}')
|
|
elif (any(part.startswith(('final_', 'archive_')) for part in args.output_dir.parts)
|
|
or any((args.output_dir / name).exists() for name in ('REPORT.md', 'summary.json', 'freeze.json'))):
|
|
raise RuntimeError('Archived/final results are immutable; choose a fresh DEV directory.')
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
validate_output_destination(args)
|
|
metadata = run_metadata(args.questions)
|
|
audit = json.loads(args.audit.read_text()) if args.audit else None
|
|
if audit and (audit['dataset_sha256'] != metadata['fingerprints']['dataset_sha256']
|
|
or audit['inputs'] != metadata['fingerprints']['inputs']):
|
|
raise RuntimeError('Source audit does not match current benchmark and inputs.')
|
|
if audit and audit.get('source_corpus') is not None and (
|
|
audit['source_corpus'] != metadata['fingerprints']['source_corpus']):
|
|
raise RuntimeError('Independent source audit does not match original Markdown.')
|
|
|
|
all_questions = load_questions(
|
|
args.questions
|
|
)
|
|
source_oracle = load_source_oracle()
|
|
|
|
ids = {
|
|
value.strip()
|
|
for value in args.ids.split(
|
|
","
|
|
)
|
|
if value.strip()
|
|
}
|
|
|
|
questions = filter_questions(
|
|
all_questions,
|
|
split=args.split,
|
|
ids=(
|
|
ids
|
|
if ids
|
|
else None
|
|
),
|
|
max_questions=(
|
|
args.max_questions
|
|
),
|
|
)
|
|
|
|
if not questions:
|
|
raise RuntimeError(
|
|
"Pre zadaný výber sa nenašli otázky."
|
|
)
|
|
|
|
print()
|
|
print(
|
|
"GraphRAG dataset"
|
|
)
|
|
|
|
print(
|
|
"=" * 60
|
|
)
|
|
|
|
print(
|
|
f"File: {args.questions}"
|
|
)
|
|
|
|
print(
|
|
f"Total: {len(all_questions)}"
|
|
)
|
|
|
|
print(
|
|
f"Split: {args.split}"
|
|
)
|
|
|
|
print(
|
|
f"Selected: {len(questions)}"
|
|
)
|
|
|
|
print(
|
|
f"Limit: {args.limit}"
|
|
)
|
|
|
|
print(
|
|
f"Entities: {args.max_entities}"
|
|
)
|
|
|
|
print(
|
|
"=" * 60
|
|
)
|
|
|
|
settings = (
|
|
get_neo4j_settings()
|
|
)
|
|
|
|
rows: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
with create_neo4j_driver(
|
|
settings
|
|
) as driver:
|
|
driver.verify_connectivity()
|
|
|
|
with driver.session(
|
|
database=settings.database
|
|
) as session:
|
|
metadata['graph'] = graph_fingerprint(session)
|
|
if args.freeze_manifest:
|
|
verify_freeze(json.loads(args.freeze_manifest.read_text()), metadata, metadata['graph'],
|
|
limit=args.limit, max_entities=args.max_entities, audit=args.audit)
|
|
total = len(
|
|
questions
|
|
)
|
|
|
|
for index, question in enumerate(
|
|
questions,
|
|
start=1,
|
|
):
|
|
question_id = str(
|
|
question[
|
|
"id"
|
|
]
|
|
)
|
|
|
|
print(
|
|
f"[{index:04d}/{total:04d}] "
|
|
f"{question_id}: "
|
|
f"{question['question']}"
|
|
)
|
|
|
|
started = (
|
|
time.perf_counter()
|
|
)
|
|
|
|
error = None
|
|
|
|
try:
|
|
result = (
|
|
search_graph_with_session(
|
|
session,
|
|
str(
|
|
question[
|
|
"question"
|
|
]
|
|
),
|
|
limit=args.limit,
|
|
max_entities=(
|
|
args.max_entities
|
|
),
|
|
)
|
|
)
|
|
|
|
except Exception as exc:
|
|
result = {
|
|
"strategies": [],
|
|
"resolved_entities": [],
|
|
"paths": [],
|
|
"facts": [],
|
|
"documents": [],
|
|
}
|
|
|
|
error = (
|
|
f"{type(exc).__name__}: "
|
|
f"{exc}"
|
|
)
|
|
|
|
latency = (
|
|
time.perf_counter()
|
|
- started
|
|
)
|
|
|
|
row = evaluate_result(
|
|
question,
|
|
result,
|
|
latency_seconds=(
|
|
latency
|
|
),
|
|
error=error,
|
|
source_oracle=source_oracle,
|
|
)
|
|
|
|
rows.append(
|
|
row
|
|
)
|
|
|
|
state = (
|
|
"OK"
|
|
if row["strict"]
|
|
else "FAIL"
|
|
)
|
|
|
|
print(
|
|
f" {state} | "
|
|
f"value={row['value_score']:.2f} "
|
|
f"entity={row['entity_score']:.2f} "
|
|
f"rel={row['relation_score']:.2f} "
|
|
f"doc={row['document_recall']:.2f} "
|
|
f"{row['latency_seconds']:.4f}s"
|
|
)
|
|
|
|
if graph_fingerprint(session) != metadata['graph']:
|
|
raise RuntimeError('Graph changed during evaluation; results discarded.')
|
|
|
|
if run_metadata(args.questions)['fingerprints'] != metadata['fingerprints']:
|
|
raise RuntimeError('Code or input data changed during evaluation; results discarded.')
|
|
metadata['completed_at_utc'] = datetime.now(timezone.utc).isoformat()
|
|
if args.freeze_manifest:
|
|
metadata['freeze_sha256'] = sha256_file(args.freeze_manifest)
|
|
if args.audit:
|
|
metadata['audit_sha256'] = sha256_file(args.audit)
|
|
|
|
summary = aggregate_rows(
|
|
rows
|
|
)
|
|
|
|
by_category = aggregate_by_field(
|
|
rows,
|
|
"category",
|
|
)
|
|
|
|
by_difficulty = (
|
|
aggregate_by_field(
|
|
rows,
|
|
"difficulty",
|
|
)
|
|
)
|
|
|
|
payload = {
|
|
'evaluation_schema_version': 4,
|
|
'run_metadata': metadata,
|
|
"configuration": {
|
|
'evaluation_kind': (json.loads(args.freeze_manifest.read_text()).get('evaluation_kind', 'independent_holdout')
|
|
if args.freeze_manifest else 'development_regression'),
|
|
'published_only': True,
|
|
"questions_file": str(
|
|
args.questions
|
|
),
|
|
"split": args.split,
|
|
"dataset_question_count": len(
|
|
all_questions
|
|
),
|
|
"selected_question_count": len(
|
|
questions
|
|
),
|
|
"limit": args.limit,
|
|
"max_entities": (
|
|
args.max_entities
|
|
),
|
|
"ids": sorted(
|
|
ids
|
|
),
|
|
"max_questions": (
|
|
args.max_questions
|
|
),
|
|
},
|
|
"summary": summary,
|
|
"by_category": by_category,
|
|
"by_difficulty": (
|
|
by_difficulty
|
|
),
|
|
"questions": rows,
|
|
}
|
|
if audit:
|
|
consistent_ids = {row['id'] for row in audit['questions'] if row['source_consistent']}
|
|
consistent = [row for row in rows if row['id'] in consistent_ids]
|
|
payload['source_consistent_summary'] = aggregate_rows(consistent)
|
|
payload['source_consistent_by_category'] = aggregate_by_field(consistent, 'category')
|
|
payload['source_inconsistent_ids'] = [row['id'] for row in rows if row['id'] not in consistent_ids]
|
|
|
|
suffix = args.split
|
|
|
|
if ids:
|
|
suffix += "_selected"
|
|
|
|
elif args.max_questions:
|
|
suffix += (
|
|
f"_first"
|
|
f"{args.max_questions}"
|
|
)
|
|
|
|
json_path = (
|
|
args.output_dir
|
|
/ f"graphrag_{suffix}.json"
|
|
)
|
|
|
|
csv_path = (
|
|
args.output_dir
|
|
/ f"graphrag_{suffix}.csv"
|
|
)
|
|
|
|
save_json(
|
|
json_path,
|
|
payload,
|
|
)
|
|
|
|
save_csv(
|
|
csv_path,
|
|
rows,
|
|
)
|
|
|
|
print_summary(
|
|
summary,
|
|
split=args.split,
|
|
)
|
|
if audit:
|
|
print_summary(payload['source_consistent_summary'],
|
|
split=f'{args.split} — source-consistent subset (supplementary)')
|
|
print(f"Source-inconsistent annotations: {len(payload['source_inconsistent_ids'])}; "
|
|
'included in the primary score above.')
|
|
|
|
print()
|
|
print(
|
|
f"JSON: {json_path}"
|
|
)
|
|
|
|
print(
|
|
f"CSV: {csv_path}"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|