519 lines
14 KiB
Python
519 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from ..graph_db import (
|
|
create_neo4j_driver,
|
|
get_neo4j_settings,
|
|
)
|
|
from ..graph_search import (
|
|
DEFAULT_LIMIT,
|
|
DEFAULT_MAX_ENTITIES,
|
|
search_graph_with_session,
|
|
)
|
|
from .graphrag_evaluator import (
|
|
aggregate_by_field,
|
|
aggregate_rows,
|
|
evaluate_result,
|
|
filter_questions,
|
|
load_questions,
|
|
)
|
|
from .reproducibility import graph_fingerprint, run_metadata, sha256_file, verify_freeze
|
|
from .source_oracle import load_source_oracle
|
|
|
|
EVALUATION_DIR = Path(__file__).resolve().parents[3] / "experiments/graphrag"
|
|
|
|
QUESTIONS_PATH = EVALUATION_DIR / "graphrag_questions_v4.json"
|
|
|
|
RESULTS_DIR = EVALUATION_DIR / "results"
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description=("Vyhodnotenie GraphRAG vyhľadávania " "proti graphrag_questions.json.")
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--questions",
|
|
type=Path,
|
|
required=True,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
type=Path,
|
|
required=True,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--split",
|
|
choices=(
|
|
"dev",
|
|
"test",
|
|
"all",
|
|
),
|
|
default="dev",
|
|
help=("Predvolene sa vyhodnocuje iba DEV. " "TEST používaj až po zmrazení GraphRAG."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=DEFAULT_LIMIT,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-entities",
|
|
type=int,
|
|
default=DEFAULT_MAX_ENTITIES,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--ids",
|
|
type=str,
|
|
default="",
|
|
help=("Voliteľný zoznam ID oddelených čiarkou."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-questions",
|
|
type=int,
|
|
default=None,
|
|
help=("Voliteľne vyhodnotí iba prvých N " "otázok z vybraného splitu."),
|
|
)
|
|
|
|
parser.add_argument("--audit", type=Path, help="Predeclared source-consistency audit.")
|
|
parser.add_argument(
|
|
"--freeze-manifest",
|
|
type=Path,
|
|
help="Required for TEST/all; verifies code, data, graph and parameters.",
|
|
)
|
|
parser.add_argument(
|
|
"--overwrite",
|
|
action="store_true",
|
|
help="Replace a previous DEV output; frozen TEST outputs are immutable.",
|
|
)
|
|
args = parser.parse_args()
|
|
if args.split in {"test", "all"} and args.freeze_manifest is None:
|
|
parser.error("TEST/all requires --freeze-manifest after complete DEV evaluation.")
|
|
if args.split in {"test", "all"} and (args.ids or args.max_questions):
|
|
parser.error("Frozen holdout evaluation must cover the complete selected split.")
|
|
if args.split in {"test", "all"} and args.overwrite:
|
|
parser.error("Use a fresh output directory to reproduce a frozen holdout.")
|
|
|
|
if args.limit < 1:
|
|
parser.error("--limit musí byť aspoň 1.")
|
|
|
|
if args.max_entities < 1:
|
|
parser.error("--max-entities musí byť aspoň 1.")
|
|
|
|
if args.max_questions is not None and args.max_questions < 1:
|
|
parser.error("--max-questions musí byť aspoň 1.")
|
|
|
|
return args
|
|
|
|
|
|
def save_json(
|
|
path: Path,
|
|
payload: dict[str, Any],
|
|
) -> None:
|
|
path.parent.mkdir(
|
|
parents=True,
|
|
exist_ok=True,
|
|
)
|
|
|
|
path.write_text(
|
|
json.dumps(
|
|
payload,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def save_csv(
|
|
path: Path,
|
|
rows: list[dict[str, Any]],
|
|
) -> None:
|
|
path.parent.mkdir(
|
|
parents=True,
|
|
exist_ok=True,
|
|
)
|
|
|
|
fieldnames = [
|
|
"complete",
|
|
"answerability_ok",
|
|
"bindings_ok",
|
|
"binding_score",
|
|
"paths_ok",
|
|
"path_score",
|
|
"count_ok",
|
|
"expected_count",
|
|
"actual_count",
|
|
"id",
|
|
"split",
|
|
"category",
|
|
"difficulty",
|
|
"strict",
|
|
"value_score",
|
|
"entity_score",
|
|
"relation_score",
|
|
"property_score",
|
|
"relation_choice_score",
|
|
"document_recall",
|
|
"source_recall",
|
|
"entity_set_ok",
|
|
"latency_seconds",
|
|
"error",
|
|
"question",
|
|
"expected_values",
|
|
"matched_values",
|
|
"expected_relations",
|
|
"actual_relations",
|
|
"expected_documents",
|
|
"actual_documents",
|
|
"expected_source_urls",
|
|
"actual_source_urls",
|
|
"fact_types",
|
|
"strategies",
|
|
"binding_results",
|
|
"path_results",
|
|
]
|
|
|
|
with path.open(
|
|
"w",
|
|
encoding="utf-8",
|
|
newline="",
|
|
) as file:
|
|
writer = csv.DictWriter(
|
|
file,
|
|
fieldnames=fieldnames,
|
|
lineterminator="\n",
|
|
)
|
|
|
|
writer.writeheader()
|
|
|
|
for row in rows:
|
|
output = {}
|
|
|
|
for field in fieldnames:
|
|
value = row.get(field)
|
|
|
|
if isinstance(
|
|
value,
|
|
(
|
|
list,
|
|
dict,
|
|
),
|
|
):
|
|
value = json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
)
|
|
|
|
output[field] = value
|
|
|
|
writer.writerow(output)
|
|
|
|
|
|
def print_summary(
|
|
summary: dict[str, Any],
|
|
*,
|
|
split: str,
|
|
) -> None:
|
|
print()
|
|
print("=" * 78)
|
|
|
|
print("GRAPHRAG EVALUATION")
|
|
|
|
print("=" * 78)
|
|
|
|
print(f"Split: {split}")
|
|
|
|
print(f"Questions: " f"{summary['questions']}")
|
|
|
|
print(
|
|
f"Strict: "
|
|
f"{summary['strict_count']}/"
|
|
f"{summary['questions']} "
|
|
f"({summary['strict_accuracy']:.3%})"
|
|
)
|
|
|
|
print(f"Errors: " f"{summary['error_count']}")
|
|
|
|
print(f"Value score: " f"{summary['value_score']:.3f}")
|
|
|
|
print(f"Entity score: " f"{summary['entity_score']:.3f}")
|
|
|
|
print(f"Relation score: " f"{summary['relation_score']:.3f}")
|
|
|
|
print(f"Property score: " f"{summary['property_score']:.3f}")
|
|
|
|
print(f"Relation choices: " f"{summary['relation_choice_score']:.3f}")
|
|
|
|
print(f"Document recall: " f"{summary['document_recall']:.3f}")
|
|
|
|
print(f"Source recall: " f"{summary['source_recall']:.3f}")
|
|
|
|
print(f"Latency mean: " f"{summary['latency_mean_seconds']:.4f} s")
|
|
|
|
print(f"Latency median: " f"{summary['latency_median_seconds']:.4f} s")
|
|
|
|
print(f"Latency p95: " f"{summary['latency_p95_seconds']:.4f} s")
|
|
|
|
print("=" * 78)
|
|
|
|
|
|
def validate_output_destination(args: argparse.Namespace) -> None:
|
|
suffix = args.split
|
|
if args.ids:
|
|
suffix += "_selected"
|
|
elif args.max_questions:
|
|
suffix += f"_first{args.max_questions}"
|
|
if not args.overwrite:
|
|
for extension in ("json", "csv"):
|
|
path = args.output_dir / f"graphrag_{suffix}.{extension}"
|
|
if path.exists():
|
|
raise RuntimeError(f"Results already exist; use a fresh output directory: {path}")
|
|
elif any(part.startswith(("final_", "archive_")) for part in args.output_dir.parts) or any(
|
|
(args.output_dir / name).exists() for name in ("REPORT.md", "summary.json", "freeze.json")
|
|
):
|
|
raise RuntimeError("Archived/final results are immutable; choose a fresh DEV directory.")
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
validate_output_destination(args)
|
|
metadata = run_metadata(args.questions)
|
|
audit = json.loads(args.audit.read_text()) if args.audit else None
|
|
if audit and (
|
|
audit["dataset_sha256"] != metadata["fingerprints"]["dataset_sha256"]
|
|
or audit["inputs"] != metadata["fingerprints"]["inputs"]
|
|
):
|
|
raise RuntimeError("Source audit does not match current benchmark and inputs.")
|
|
if (
|
|
audit
|
|
and audit.get("source_corpus") is not None
|
|
and (audit["source_corpus"] != metadata["fingerprints"]["source_corpus"])
|
|
):
|
|
raise RuntimeError("Independent source audit does not match original Markdown.")
|
|
|
|
all_questions = load_questions(args.questions)
|
|
source_oracle = load_source_oracle()
|
|
|
|
ids = {value.strip() for value in args.ids.split(",") if value.strip()}
|
|
|
|
questions = filter_questions(
|
|
all_questions,
|
|
split=args.split,
|
|
ids=(ids if ids else None),
|
|
max_questions=(args.max_questions),
|
|
)
|
|
|
|
if not questions:
|
|
raise RuntimeError("Pre zadaný výber sa nenašli otázky.")
|
|
|
|
print()
|
|
print("GraphRAG dataset")
|
|
|
|
print("=" * 60)
|
|
|
|
print(f"File: {args.questions}")
|
|
|
|
print(f"Total: {len(all_questions)}")
|
|
|
|
print(f"Split: {args.split}")
|
|
|
|
print(f"Selected: {len(questions)}")
|
|
|
|
print(f"Limit: {args.limit}")
|
|
|
|
print(f"Entities: {args.max_entities}")
|
|
|
|
print("=" * 60)
|
|
|
|
settings = get_neo4j_settings()
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
|
|
with create_neo4j_driver(settings) as driver:
|
|
driver.verify_connectivity()
|
|
|
|
with driver.session(database=settings.database) as session:
|
|
metadata["graph"] = graph_fingerprint(session)
|
|
if args.freeze_manifest:
|
|
verify_freeze(
|
|
json.loads(args.freeze_manifest.read_text()),
|
|
metadata,
|
|
metadata["graph"],
|
|
limit=args.limit,
|
|
max_entities=args.max_entities,
|
|
audit=args.audit,
|
|
)
|
|
total = len(questions)
|
|
|
|
for index, question in enumerate(
|
|
questions,
|
|
start=1,
|
|
):
|
|
question_id = str(question["id"])
|
|
|
|
print(f"[{index:04d}/{total:04d}] " f"{question_id}: " f"{question['question']}")
|
|
|
|
started = time.perf_counter()
|
|
|
|
error = None
|
|
|
|
try:
|
|
result = search_graph_with_session(
|
|
session,
|
|
str(question["question"]),
|
|
limit=args.limit,
|
|
max_entities=(args.max_entities),
|
|
)
|
|
|
|
except Exception as exc:
|
|
result = {
|
|
"strategies": [],
|
|
"resolved_entities": [],
|
|
"paths": [],
|
|
"facts": [],
|
|
"documents": [],
|
|
}
|
|
|
|
error = f"{type(exc).__name__}: " f"{exc}"
|
|
|
|
latency = time.perf_counter() - started
|
|
|
|
row = evaluate_result(
|
|
question,
|
|
result,
|
|
latency_seconds=(latency),
|
|
error=error,
|
|
source_oracle=source_oracle,
|
|
)
|
|
|
|
rows.append(row)
|
|
|
|
state = "OK" if row["strict"] else "FAIL"
|
|
|
|
print(
|
|
f" {state} | "
|
|
f"value={row['value_score']:.2f} "
|
|
f"entity={row['entity_score']:.2f} "
|
|
f"rel={row['relation_score']:.2f} "
|
|
f"doc={row['document_recall']:.2f} "
|
|
f"{row['latency_seconds']:.4f}s"
|
|
)
|
|
|
|
if graph_fingerprint(session) != metadata["graph"]:
|
|
raise RuntimeError("Graph changed during evaluation; results discarded.")
|
|
|
|
if run_metadata(args.questions)["fingerprints"] != metadata["fingerprints"]:
|
|
raise RuntimeError("Code or input data changed during evaluation; results discarded.")
|
|
metadata["completed_at_utc"] = datetime.now(timezone.utc).isoformat()
|
|
if args.freeze_manifest:
|
|
metadata["freeze_sha256"] = sha256_file(args.freeze_manifest)
|
|
if args.audit:
|
|
metadata["audit_sha256"] = sha256_file(args.audit)
|
|
|
|
summary = aggregate_rows(rows)
|
|
|
|
by_category = aggregate_by_field(
|
|
rows,
|
|
"category",
|
|
)
|
|
|
|
by_difficulty = aggregate_by_field(
|
|
rows,
|
|
"difficulty",
|
|
)
|
|
|
|
payload = {
|
|
"evaluation_schema_version": 4,
|
|
"run_metadata": metadata,
|
|
"configuration": {
|
|
"evaluation_kind": (
|
|
json.loads(args.freeze_manifest.read_text()).get(
|
|
"evaluation_kind", "independent_holdout"
|
|
)
|
|
if args.freeze_manifest
|
|
else "development_regression"
|
|
),
|
|
"published_only": True,
|
|
"questions_file": str(args.questions),
|
|
"split": args.split,
|
|
"dataset_question_count": len(all_questions),
|
|
"selected_question_count": len(questions),
|
|
"limit": args.limit,
|
|
"max_entities": (args.max_entities),
|
|
"ids": sorted(ids),
|
|
"max_questions": (args.max_questions),
|
|
},
|
|
"summary": summary,
|
|
"by_category": by_category,
|
|
"by_difficulty": (by_difficulty),
|
|
"questions": rows,
|
|
}
|
|
if audit:
|
|
consistent_ids = {row["id"] for row in audit["questions"] if row["source_consistent"]}
|
|
consistent = [row for row in rows if row["id"] in consistent_ids]
|
|
payload["source_consistent_summary"] = aggregate_rows(consistent)
|
|
payload["source_consistent_by_category"] = aggregate_by_field(consistent, "category")
|
|
payload["source_inconsistent_ids"] = [
|
|
row["id"] for row in rows if row["id"] not in consistent_ids
|
|
]
|
|
|
|
suffix = args.split
|
|
|
|
if ids:
|
|
suffix += "_selected"
|
|
|
|
elif args.max_questions:
|
|
suffix += f"_first" f"{args.max_questions}"
|
|
|
|
json_path = args.output_dir / f"graphrag_{suffix}.json"
|
|
|
|
csv_path = args.output_dir / f"graphrag_{suffix}.csv"
|
|
|
|
save_json(
|
|
json_path,
|
|
payload,
|
|
)
|
|
|
|
save_csv(
|
|
csv_path,
|
|
rows,
|
|
)
|
|
|
|
print_summary(
|
|
summary,
|
|
split=args.split,
|
|
)
|
|
if audit:
|
|
print_summary(
|
|
payload["source_consistent_summary"],
|
|
split=f"{args.split} — source-consistent subset (supplementary)",
|
|
)
|
|
print(
|
|
f"Source-inconsistent annotations: {len(payload['source_inconsistent_ids'])}; "
|
|
"included in the primary score above."
|
|
)
|
|
|
|
print()
|
|
print(f"JSON: {json_path}")
|
|
|
|
print(f"CSV: {csv_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|