{
  "completed_at_utc": "2026-10-25T02:04:11.837608+00:00",
  "graph_dev": {
    "correct": 722,
    "questions": 722,
    "accuracy": 1.0
  },
  "graph_test": {
    "correct": 320,
    "questions": 320,
    "accuracy": 1.0
  },
  "all_graph_context_facts_source_check": {
    "questions": 320,
    "source_consistent": 319,
    "inconsistent_ids": [
      "wide-080"
    ]
  },
  "generation_dev": {
    "questions": 20,
    "correct": 19,
    "accuracy": 0.95,
    "errors": 0
  },
  "generation_test": {
    "questions": 300,
    "correct": 281,
    "accuracy": 0.9366666666666666,
    "errors": 7
  },
  "generation_test_by_category": {
    "candidate_set_aggregation": {
      "questions": 30,
      "correct": 27,
      "accuracy": 0.9
    },
    "conjunctive_disambiguation": {
      "questions": 38,
      "correct": 37,
      "accuracy": 0.9736842105263158
    },
    "document_author_category_multi_hop": {
      "questions": 22,
      "correct": 20,
      "accuracy": 0.9090909090909091
    },
    "negative_lookup": {
      "questions": 30,
      "correct": 30,
      "accuracy": 1.0
    },
    "person_keyword_title": {
      "questions": 30,
      "correct": 21,
      "accuracy": 0.7
    },
    "person_multiple_categories": {
      "questions": 22,
      "correct": 21,
      "accuracy": 0.9545454545454546
    },
    "person_title_year": {
      "questions": 22,
      "correct": 21,
      "accuracy": 0.9545454545454546
    },
    "same_academic_year_pair": {
      "questions": 38,
      "correct": 37,
      "accuracy": 0.9736842105263158
    },
    "same_start_year_pair": {
      "questions": 30,
      "correct": 29,
      "accuracy": 0.9666666666666667
    },
    "shared_keyword_pair": {
      "questions": 38,
      "correct": 38,
      "accuracy": 1.0
    }
  },
  "generation_failure_reasons": {
    "unsupported_or_wrong_subject_claim": 3,
    "missing_required_prose_value": 3,
    "invalid_json": 7,
    "binding_missing_or_split": 6
  },
  "generation_latency_seconds": {
    "mean": 7.617359811466643,
    "median": 7.5836237964977045,
    "p95": 11.524189818002924
  },
  "generation_token_usage": {
    "completion_tokens": 52582,
    "prompt_tokens": 243107,
    "total_tokens": 295689
  },
  "target": 0.92,
  "target_met": true,
  "tests": {
    "passed": 448,
    "skipped": 4
  },
  "point_3_status": {
    "1": "complete_for_declared_v4_protocol",
    "2": "complete_for_declared_v4_protocol",
    "3": "complete_for_declared_v4_protocol",
    "4": "complete_for_declared_v4_protocol",
    "5": "complete_for_declared_v4_protocol",
    "6": "complete_for_declared_v4_protocol"
  },
  "attempt_history": [
    {
      "experiment": "graph_v4",
      "dev": 722,
      "test": 320,
      "correct": 320,
      "attempt": 1
    },
    {
      "experiment": "initial_external_diagnostic",
      "questions": 20,
      "correct_raw": 16,
      "role_after_exposure": "dev"
    },
    {
      "experiment": "revised_generation_dev",
      "questions": 20,
      "correct": 19
    },
    {
      "experiment": "generation_v4_test",
      "questions": 300,
      "correct": 281,
      "attempt": 1
    }
  ],
  "limitations": [
    "Known corpus, profiles, vocabulary and template families; no independent human annotation of all questions.",
    "Strict generation scoring includes malformed JSON and incomplete prose; not every failure is a factual falsehood.",
    "Retrieval strict checks required proofs, not every extra returned fact. Supplementary all-context source check is reported separately.",
    "OpenWebUI API with harness-supplied context, not native tool routing or browser use.",
    "External server model weights are not locally frozen; raw responses and returned model identity are retained."
  ]
}