{
  "completed_at_utc": "2026-10-25T16:30:27.413476+00:00",
  "release": "v5.1",
  "release_tag": "graphrag-v5.1-frozen",
  "evaluation_kind": "historical_regression",
  "graph_dev": {
    "questions": 722,
    "source_checked_count": 722,
    "all_facts_supported_count": 722,
    "strict_count": 722,
    "strict_accuracy": 1.0,
    "error_count": 0,
    "value_score": 1.0,
    "entity_score": 1.0,
    "relation_score": 1.0,
    "property_score": 1.0,
    "relation_choice_score": 1.0,
    "document_recall": 1.0,
    "source_recall": 1.0,
    "binding_score": 1.0,
    "path_score": 1.0,
    "answerability_accuracy": 1.0,
    "latency_mean_seconds": 0.042052,
    "latency_median_seconds": 0.040355,
    "latency_p95_seconds": 0.052196
  },
  "graph_regression": {
    "questions": 320,
    "source_checked_count": 320,
    "all_facts_supported_count": 320,
    "strict_count": 320,
    "strict_accuracy": 1.0,
    "error_count": 0,
    "value_score": 1.0,
    "entity_score": 1.0,
    "relation_score": 1.0,
    "property_score": 1.0,
    "relation_choice_score": 1.0,
    "document_recall": 1.0,
    "source_recall": 1.0,
    "binding_score": 1.0,
    "path_score": 1.0,
    "answerability_accuracy": 1.0,
    "latency_mean_seconds": 0.042475,
    "latency_median_seconds": 0.042369,
    "latency_p95_seconds": 0.049721
  },
  "generation_dev": {
    "questions": 39,
    "correct": 38,
    "accuracy": 0.9743589743589743,
    "errors": 0,
    "first_pass_correct": 38,
    "format_repair_attempted": 0,
    "normalized_first_response": 0
  },
  "generation_regression": {
    "questions": 320,
    "correct": 317,
    "accuracy": 0.990625,
    "errors": 0,
    "first_pass_correct": 317,
    "format_repair_attempted": 0,
    "normalized_first_response": 0
  },
  "generation_regression_by_category": {
    "candidate_set_aggregation": {
      "questions": 32,
      "correct": 32,
      "accuracy": 1.0
    },
    "conjunctive_disambiguation": {
      "questions": 40,
      "correct": 40,
      "accuracy": 1.0
    },
    "document_author_category_multi_hop": {
      "questions": 24,
      "correct": 24,
      "accuracy": 1.0
    },
    "negative_lookup": {
      "questions": 32,
      "correct": 32,
      "accuracy": 1.0
    },
    "person_keyword_title": {
      "questions": 32,
      "correct": 30,
      "accuracy": 0.9375
    },
    "person_multiple_categories": {
      "questions": 24,
      "correct": 24,
      "accuracy": 1.0
    },
    "person_title_year": {
      "questions": 24,
      "correct": 23,
      "accuracy": 0.9583333333333334
    },
    "same_academic_year_pair": {
      "questions": 40,
      "correct": 40,
      "accuracy": 1.0
    },
    "same_start_year_pair": {
      "questions": 32,
      "correct": 32,
      "accuracy": 1.0
    },
    "shared_keyword_pair": {
      "questions": 40,
      "correct": 40,
      "accuracy": 1.0
    }
  },
  "generation_failure_reasons": {
    "unsupported_claim": 1,
    "answerability": 2
  },
  "comparison_v4_test_300": {
    "questions": 300,
    "v4_correct": 281,
    "v5_1_correct": 297
  },
  "actual_models": {
    "Qwen3.5-122B-A10B-Q4_K_M": 320
  },
  "all_attempt_token_usage": {
    "completion_tokens": 57707,
    "prompt_tokens": 362926,
    "total_tokens": 420633
  },
  "target": 0.92,
  "target_met": true,
  "automated_tests": {
    "passed": 525,
    "skipped": 4,
    "failed": 0
  },
  "development_overlap_in_final_generation": 39,
  "attempt_history": [
    {
      "name": "strict DEV before assignment-title fix",
      "correct": 719,
      "questions": 722
    },
    {
      "name": "v5 graph regression",
      "correct": 319,
      "questions": 320
    },
    {
      "name": "v5 generation DEV",
      "correct": 38,
      "questions": 39
    }
  ],
  "point_3_status": {
    "3.1": "Completed for v5.1 historical regression with declared limitations",
    "3.2": "Completed for v5.1 historical regression with declared limitations",
    "3.3": "Completed for v5.1 historical regression with declared limitations",
    "3.4": "Completed for v5.1 historical regression with declared limitations",
    "3.5": "Completed for v5.1 historical regression with declared limitations",
    "3.6": "Completed for v5.1 historical regression with declared limitations"
  },
  "limitations": [
    "Known questions, profiles, templates and corpus. This is not a new independent holdout.",
    "39 development questions occur in the final 320-case generation regression.",
    "Strict criteria and JSON handling changed relative to v4; comparisons are descriptive, not a controlled causal experiment.",
    "Independent rule-based raw-source oracle, not independent human annotation of the full dataset.",
    "Checks typed factual claims, bindings, citations and required prose values; not a semantic judge of every possible sentence.",
    "OpenWebUI API receives locally supplied graph context; native tool routing remains point 4.",
    "Shared-context contract forwarding and further cross-cutting work are deferred as requested.",
    "External model weights are controlled by the provider; repeated requests may differ."
  ]
}