118 lines
3.4 KiB
JSON
118 lines
3.4 KiB
JSON
{
|
|
"generated_at_utc": "2026-10-24T19:05:45.204482+00:00",
|
|
"freeze_sha256": "2277226cbefa6337e04b5642da1145e1a11c48bbb05cf90de7ee5a7dbc71d806",
|
|
"dataset_sha256": "b5c96e423fc9a5bcee0dcca38fdeac86b2f7cd1efebfcb0d7cd4cfb7fed0620e",
|
|
"primary_metric": "Strict accuracy over ALL questions; no exclusions.",
|
|
"supplementary_metric": "Strict accuracy on the source-consistent subset declared by static audit before TEST.",
|
|
"baseline_dev_same_evaluator": {
|
|
"questions": 286,
|
|
"strict_count": 106,
|
|
"strict_accuracy": 0.370629,
|
|
"error_count": 0,
|
|
"value_score": 0.772494,
|
|
"entity_score": 0.624126,
|
|
"relation_score": 0.853147,
|
|
"property_score": 0.793706,
|
|
"relation_choice_score": 0.870629,
|
|
"document_recall": 1.0,
|
|
"source_recall": 0.900058,
|
|
"latency_mean_seconds": 0.029989,
|
|
"latency_median_seconds": 0.028124,
|
|
"latency_p95_seconds": 0.041821
|
|
},
|
|
"dev": {
|
|
"questions": 286,
|
|
"strict_count": 209,
|
|
"strict_accuracy": 0.730769,
|
|
"error_count": 0,
|
|
"value_score": 0.751748,
|
|
"entity_score": 0.751748,
|
|
"relation_score": 0.751748,
|
|
"property_score": 0.835664,
|
|
"relation_choice_score": 0.968531,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"latency_mean_seconds": 0.032786,
|
|
"latency_median_seconds": 0.03221,
|
|
"latency_p95_seconds": 0.041831
|
|
},
|
|
"test": {
|
|
"questions": 214,
|
|
"strict_count": 152,
|
|
"strict_accuracy": 0.71028,
|
|
"error_count": 0,
|
|
"value_score": 0.735202,
|
|
"entity_score": 0.726636,
|
|
"relation_score": 0.738318,
|
|
"property_score": 0.794393,
|
|
"relation_choice_score": 0.953271,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"latency_mean_seconds": 0.029583,
|
|
"latency_median_seconds": 0.029331,
|
|
"latency_p95_seconds": 0.033782
|
|
},
|
|
"dev_source_consistent": {
|
|
"questions": 209,
|
|
"strict_count": 209,
|
|
"strict_accuracy": 1.0,
|
|
"error_count": 0,
|
|
"value_score": 1.0,
|
|
"entity_score": 1.0,
|
|
"relation_score": 1.0,
|
|
"property_score": 1.0,
|
|
"relation_choice_score": 1.0,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"latency_mean_seconds": 0.033557,
|
|
"latency_median_seconds": 0.033169,
|
|
"latency_p95_seconds": 0.041472
|
|
},
|
|
"test_source_consistent": {
|
|
"questions": 154,
|
|
"strict_count": 152,
|
|
"strict_accuracy": 0.987013,
|
|
"error_count": 0,
|
|
"value_score": 0.995671,
|
|
"entity_score": 0.987013,
|
|
"relation_score": 1.0,
|
|
"property_score": 0.987013,
|
|
"relation_choice_score": 0.987013,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"latency_mean_seconds": 0.030486,
|
|
"latency_median_seconds": 0.030564,
|
|
"latency_p95_seconds": 0.033949
|
|
},
|
|
"source_audit": {
|
|
"dev": {
|
|
"questions": 286,
|
|
"source_consistent": 209,
|
|
"issues": {
|
|
"keyword_not_in_frontmatter_catalog": 66,
|
|
"keyword_not_linked_to_expected_person": 5,
|
|
"exact_categories_omit_or_add_source_categories": 6
|
|
}
|
|
},
|
|
"test": {
|
|
"questions": 214,
|
|
"source_consistent": 154,
|
|
"issues": {
|
|
"keyword_not_in_frontmatter_catalog": 46,
|
|
"keyword_not_linked_to_expected_person": 11,
|
|
"exact_categories_omit_or_add_source_categories": 3
|
|
}
|
|
}
|
|
},
|
|
"unit_and_integration_tests": {
|
|
"passed": 359,
|
|
"skipped": 4,
|
|
"neo4j_integration_cases": 10
|
|
},
|
|
"holdout_failures_without_source_audit_issues": [
|
|
"g0065",
|
|
"g0129"
|
|
],
|
|
"post_test_code_changes": false
|
|
}
|