dp-zp-agent/experiments/user_questions_v1/review.json

{
  "assessment": "Supplemental AI source review, not human validation and not a replacement for frozen first-run scores.",
  "confirmed_backend_findings": [
    {
      "cases": [
        "T001",
        "T003",
        "T005",
        "T007",
        "T009"
      ],
      "engines": [
        "graphrag"
      ],
      "finding": "False no_answer for explicitly documented Bachelor project 2027 records.",
      "source_check": "Matching Work and KnowledgeWork exist for Adrian Ondra, type bachelor_project/year2027; test gold provides original headings.",
      "code_findings": [
        "extract_query_constraints mixes bachelor_thesis and bachelor_project when a question says bachelor project plus work year",
        "Quoted academic headings also pass the legacy quoted-title lookup"
      ]
    },
    {
      "cases": [
        "T002",
        "T010",
        "T012"
      ],
      "engines": [
        "graphrag"
      ],
      "finding": "Overview wording can return only title and year, omitting the explicitly requested academic work type.",
      "code_findings": [
        "Legacy mentions_work_listing is false for Zostav prehlad evidovanych prac; shared typed plan does not replace every legacy rendering branch"
      ]
    },
    {
      "cases": [
        "T021",
        "T023",
        "T025",
        "T027",
        "T029",
        "T031"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "A presentation constraint is falsely resolved as the person nezamienaj ich; source goals exist.",
      "source_check": "Source-bound requirements contain actual known people and full assignment bullets. The failed span is not a person."
    },
    {
      "cases": [
        "T022"
      ],
      "engines": [
        "graphrag"
      ],
      "finding": "Correct person-level join of alignment tag, bachelor_project and year2027, with both expected people and topics.",
      "source_check": "Yehor Piliavin and Kristian Sokolsky have the tag and explicit matching heading in raw Markdown."
    },
    {
      "cases": [
        "T036"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "Only five follow-up tasks are returned despite further source tasks fitting the shared budget; warning is retained but requested coverage is incomplete.",
      "code_findings": [
        "tools/knowledge/planned_answers.py uses take=5 for non-assignment facets independently of the query limit"
      ]
    },
    {
      "cases": [
        "T043"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "Asked frontend/backend/database technologies, but received general completed checklist tasks. Source explicitly provides React/Flask/Postgres/SQLAlchemy/Vite/socket.io/JWT; these are not selected."
    },
    {
      "cases": [
        "T044"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "Asked the long-context problem AND the attempted remedy; answer states the context problem but omits the documented hierarchical summarization."
    },
    {
      "cases": [
        "T061",
        "T062",
        "T063"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "Comparison says no supported goals exist for either work although source sections contain proposed goals. The backend should present the documented proposals with their uncertainty or explain that a formally confirmed assignment is absent; broad absence of data is misleading."
    },
    {
      "cases": [
        "T069",
        "T073"
      ],
      "engines": [
        "rag"
      ],
      "finding": "Relational requests can produce raw per-person metadata instead of explicitly computing the requested common result. T069 also includes a non-shared tag despite the only-shared condition."
    },
    {
      "cases": [
        "T080"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "Inflected partial first name Samuelovi is rejected by verification instead of asking which actual Samuel is meant."
    },
    {
      "cases": [
        "T081"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "A similarity question with the explicit canonical full name Adrian Ondra wrongly asks for a full student name."
    },
    {
      "cases": [
        "T085",
        "T086"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "Global discovery questions wrongly require a known student name even when the purpose is to discover that name from a documented activity."
    },
    {
      "cases": [
        "T087"
      ],
      "engines": [
        "rag",
        "graphrag"
      ],
      "finding": "The presentation request bez domyslania vysledkov is interpreted as an unsupported data-exclusion filter instead of a grounding rule.",
      "code_findings": [
        "tools/answers/question_plan.py treats any bez phrase without a work exclusion selector as unsupported exclusion"
      ]
    },
    {
      "finding": "Most content RAG questions enter deliver_content before BM25/E5/RRF. Their candidate route is the content checkpoint, so a repair of independent hybrid candidates alone does not improve those content questions. Separate route labels are required when comparing engines."
    }
  ],
  "confirmed_evaluator_findings": [
    {
      "cases": [
        "D01",
        "D02",
        "T002",
        "T010",
        "T018"
      ],
      "finding": "Some judge unsupported_time/unsupported_number proofs repeat the exact supported date/year or number. Such a verdict does not establish a backend hallucination."
    },
    {
      "cases": [
        "T002"
      ],
      "finding": "The judge missed a real omission of work type while wrongly rejecting the supported2027 work year."
    },
    {
      "finding": "Invalid schema/proof output is counted as judge_unavailable, not silently accepted; this prevents treating the conservative score as an accurate estimate of actual factual accuracy."
    },
    {
      "cases": [
        "T064"
      ],
      "finding": "The judge labels a supported Simona Bobrovcanova legal-text summarization goal and its correct citation as invented/unsupported. Raw source has an explicit Ciele section with that exact goal. This critic verdict is a false positive; first-run scoring remains unchanged."
    },
    {
      "cases": [
        "T069",
        "T073"
      ],
      "engines": [
        "rag"
      ],
      "finding": "Judge false acceptances also occur: T069 asks only the shared tag intersection but returns per-person full tag lists including a non-shared steering-vectors tag. T073 asks to lead with the common year, but the answer instead enumerates separate people and unrelated metadata. Supported facts alone are insufficient question coverage."
    }
  ],
  "closure": "Not closed: real false refusals remain; reliable independent answer accuracy is not established."
}