dp-zp-agent/tools/graphrag/evaluation/results/final_2026-10-24/holdout_failures.json

{
  "policy": "Recorded after the frozen TEST. No implementation or audit changes were made to improve these cases.",
  "diagnosis": "Both questions concern the same pair. An additional Martin Jancura document appears in the evidence; likely overly broad fuzzy person matching. Investigate in a future version using separate development cases and a new holdout.",
  "questions": [
    {
      "count_ok": true,
      "expected_count": null,
      "actual_count": null,
      "complete": true,
      "id": "g0065",
      "split": "test",
      "category": "same_academic_year_pair",
      "difficulty": "hard",
      "question": "V ktorom rovnakom roku je v ZP Wiki uvedená práca pri osobách Ján Malinovský a Martin Šarišský?",
      "latency_seconds": 0.034872,
      "error": null,
      "strict": false,
      "value_score": 0.666667,
      "entity_score": 0.0,
      "relation_score": 1.0,
      "property_score": 0.0,
      "relation_choice_score": 0.0,
      "document_recall": 1.0,
      "source_recall": 1.0,
      "expected_values": [
        "Ján Malinovský",
        "Martin Šarišský",
        "2026"
      ],
      "matched_values": [
        "Ján Malinovský",
        "Martin Šarišský"
      ],
      "expected_entities": [
        {
          "type": "year",
          "value": "2026"
        }
      ],
      "matched_entities": [],
      "expected_relations": [],
      "actual_relations": [],
      "matched_relations": [],
      "expected_properties": [
        "Thesis.year|Work.year",
        "Thesis.year|Work.year"
      ],
      "matched_properties": [],
      "relation_choices": [
        {
          "choices": [
            "AUTHORED",
            "HAS_WORK"
          ],
          "matched": false
        },
        {
          "choices": [
            "AUTHORED",
            "HAS_WORK"
          ],
          "matched": false
        }
      ],
      "expected_documents": [
        "pages/students/2021/martin_sarissky/README.md",
        "pages/students/2022/jan_malinovsky/README.md"
      ],
      "actual_documents": [
        "pages/students/2017/martin_jancura/README.md",
        "pages/students/2021/martin_sarissky/README.md",
        "pages/students/2022/jan_malinovsky/README.md"
      ],
      "matched_documents": [
        "pages/students/2021/martin_sarissky/README.md",
        "pages/students/2022/jan_malinovsky/README.md"
      ],
      "expected_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky",
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
      ],
      "actual_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
        "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky",
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
      ],
      "matched_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky",
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
      ],
      "entity_set_expected": [],
      "entity_set_actual": [],
      "entity_set_mode": null,
      "entity_set_ok": true,
      "strategies": [
        "entity_resolution"
      ],
      "fact_types": [
        "academic_year_comparison"
      ]
    },
    {
      "count_ok": true,
      "expected_count": null,
      "actual_count": null,
      "complete": true,
      "id": "g0129",
      "split": "test",
      "category": "same_academic_year_pair",
      "difficulty": "hard",
      "question": "Aký rok práce spája osoby Ján Malinovský a Martin Šarišský?",
      "latency_seconds": 0.032707,
      "error": null,
      "strict": false,
      "value_score": 0.666667,
      "entity_score": 0.0,
      "relation_score": 1.0,
      "property_score": 0.0,
      "relation_choice_score": 0.0,
      "document_recall": 1.0,
      "source_recall": 1.0,
      "expected_values": [
        "Ján Malinovský",
        "Martin Šarišský",
        "2026"
      ],
      "matched_values": [
        "Ján Malinovský",
        "Martin Šarišský"
      ],
      "expected_entities": [
        {
          "type": "year",
          "value": "2026"
        }
      ],
      "matched_entities": [],
      "expected_relations": [],
      "actual_relations": [],
      "matched_relations": [],
      "expected_properties": [
        "Thesis.year|Work.year",
        "Thesis.year|Work.year"
      ],
      "matched_properties": [],
      "relation_choices": [
        {
          "choices": [
            "AUTHORED",
            "HAS_WORK"
          ],
          "matched": false
        },
        {
          "choices": [
            "AUTHORED",
            "HAS_WORK"
          ],
          "matched": false
        }
      ],
      "expected_documents": [
        "pages/students/2021/martin_sarissky/README.md",
        "pages/students/2022/jan_malinovsky/README.md"
      ],
      "actual_documents": [
        "pages/students/2017/martin_jancura/README.md",
        "pages/students/2021/martin_sarissky/README.md",
        "pages/students/2022/jan_malinovsky/README.md"
      ],
      "matched_documents": [
        "pages/students/2021/martin_sarissky/README.md",
        "pages/students/2022/jan_malinovsky/README.md"
      ],
      "expected_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky",
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
      ],
      "actual_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
        "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky",
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
      ],
      "matched_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2021/martin_sarissky",
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
      ],
      "entity_set_expected": [],
      "entity_set_actual": [],
      "entity_set_mode": null,
      "entity_set_ok": true,
      "strategies": [
        "entity_resolution"
      ],
      "fact_types": [
        "academic_year_comparison"
      ]
    }
  ]
}