977 lines
39 KiB
Python
977 lines
39 KiB
Python
"""Build a source-grounded schema-v3 regression set and profile-disjoint holdout.
|
|
|
|
This module never imports graph construction/search code and never reads any
|
|
retrieval output. Legacy cases remain development-only. New TEST subjects are
|
|
student profiles absent from every legacy start entity. TEST is generated once
|
|
from raw source facts, before the corrected implementation is evaluated.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import itertools
|
|
import json
|
|
import random
|
|
import re
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .source_oracle import ROOT, DEFAULT_WIKI, load_source_oracle, norm, supporting_documents
|
|
|
|
BASE = ROOT / "experiments/graphrag"
|
|
SEED = 20261024
|
|
|
|
|
|
def fingerprint(value: Any) -> str:
|
|
return hashlib.sha256(
|
|
json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()
|
|
).hexdigest()
|
|
|
|
|
|
def people_for(question: dict, oracle: dict) -> list[str]:
|
|
names = {norm(name): name for name in oracle["people"]}
|
|
return [
|
|
names[norm(item["value"])]
|
|
for item in question["graph"].get("start_entities", [])
|
|
if item["type"] == "person" and norm(item["value"]) in names
|
|
]
|
|
|
|
|
|
def academics(
|
|
oracle: dict,
|
|
person: str,
|
|
*,
|
|
title: str | None = None,
|
|
year: int | None = None,
|
|
kind: str | None = None,
|
|
) -> list[dict]:
|
|
rows = oracle["people"][person]["academics"]
|
|
return [
|
|
row
|
|
for row in rows
|
|
if (title is None or norm(row["title"]) == norm(title))
|
|
and (year is None or row["year"] == year)
|
|
and (kind is None or row["academic_kind"] == kind)
|
|
]
|
|
|
|
|
|
def has_tag(oracle: dict, person: str, keyword: str) -> bool:
|
|
return norm(keyword) in {norm(tag) for tag in oracle["people"][person]["tags"]}
|
|
|
|
|
|
def academic_binding(
|
|
person: str, record: dict, *, title: bool = False, keyword: str | None = None
|
|
) -> dict:
|
|
binding = {
|
|
"person": person,
|
|
"academic_kind": record["academic_kind"],
|
|
"academic_type": record["academic_type"],
|
|
"year": record["year"],
|
|
}
|
|
if title:
|
|
binding["title"] = record["title"]
|
|
if keyword is not None:
|
|
binding["keyword"] = keyword
|
|
return {key: value for key, value in binding.items() if value is not None}
|
|
|
|
|
|
def path_patterns(bindings: list[dict]) -> list[dict]:
|
|
patterns = []
|
|
for index, binding in enumerate(bindings):
|
|
if any(key in binding for key in ["year", "title", "academic_kind", "academic_type"]):
|
|
kind = binding.get("academic_kind", "Academic")
|
|
relation = (
|
|
"AUTHORED"
|
|
if kind == "Thesis"
|
|
else "HAS_WORK" if kind == "Work" else "AUTHORED|HAS_WORK"
|
|
)
|
|
patterns.append(
|
|
{
|
|
"binding_index": index,
|
|
"node_kinds": ["Person", kind, "Document"],
|
|
"relationships": [relation, "EVIDENCED_BY"],
|
|
}
|
|
)
|
|
for field, kind, relation in [
|
|
("keyword", "Keyword", "HAS_KEYWORD"),
|
|
("category", "Category", "IN_CATEGORY"),
|
|
("document_author", "DocumentAuthor", "AUTHORED_DOCUMENT"),
|
|
]:
|
|
if field in binding:
|
|
patterns.append(
|
|
{
|
|
"binding_index": index,
|
|
"node_kinds": ["Person", "Document", kind],
|
|
"relationships": ["HAS_DOCUMENT", relation],
|
|
}
|
|
)
|
|
return patterns
|
|
|
|
|
|
def binding_source_evidence(binding: dict, oracle: dict) -> list[dict]:
|
|
person = oracle["people"][binding["person"]]
|
|
evidence = []
|
|
for field in ["keyword", "category", "document_author"]:
|
|
if field not in binding:
|
|
continue
|
|
docs = supporting_documents(oracle, binding["person"], **{field: binding[field]})
|
|
if binding.get("document_path"):
|
|
docs = [p for p in docs if p == binding["document_path"]]
|
|
for path in docs:
|
|
evidence.append(
|
|
{
|
|
"document_path": path,
|
|
"sha256": oracle["documents"][path]["sha256"],
|
|
"field": {
|
|
"keyword": "taxonomy.tag",
|
|
"category": "taxonomy.category",
|
|
"document_author": "taxonomy.author",
|
|
}[field],
|
|
"value": binding[field],
|
|
}
|
|
)
|
|
if any(key in binding for key in ["year", "title", "academic_kind", "academic_type"]):
|
|
rows = academics(
|
|
oracle,
|
|
binding["person"],
|
|
title=binding.get("title"),
|
|
year=binding.get("year"),
|
|
kind=binding.get("academic_kind"),
|
|
)
|
|
for row in rows:
|
|
if binding.get("academic_type") and row["academic_type"] != binding["academic_type"]:
|
|
continue
|
|
evidence.append(
|
|
{
|
|
"document_path": row["document_path"],
|
|
"sha256": oracle["documents"][row["document_path"]]["sha256"],
|
|
"field": "academic_record",
|
|
"academic_kind": row["academic_kind"],
|
|
"academic_type": row["academic_type"],
|
|
"title": row["title"],
|
|
"year": row["year"],
|
|
"heading_line": row["heading_line"],
|
|
"title_line": row["title_line"],
|
|
"year_basis": row["year_basis"],
|
|
}
|
|
)
|
|
if "start_year" in binding:
|
|
evidence.append(
|
|
{
|
|
"document_path": person["profile"],
|
|
"sha256": oracle["documents"][person["profile"]]["sha256"],
|
|
"field": "start_year",
|
|
"value": binding["start_year"],
|
|
"basis": person.get("start_year_basis", "explicit_text"),
|
|
"confidence": "documented",
|
|
}
|
|
)
|
|
return evidence
|
|
|
|
|
|
def make_case(
|
|
case_id: str,
|
|
split: str,
|
|
category: str,
|
|
question: str,
|
|
names: list[str],
|
|
oracle: dict,
|
|
bindings: list[dict],
|
|
*,
|
|
no_answer: bool = False,
|
|
reason: str | None = None,
|
|
expected_set: list[str] | None = None,
|
|
set_field: str | None = None,
|
|
count: int | None = None,
|
|
context_documents: list[str] | None = None,
|
|
negative_constraints: dict | None = None,
|
|
) -> dict:
|
|
known = [name for name in names if name in oracle["people"]]
|
|
profiles = sorted({oracle["people"][name]["profile"] for name in known})
|
|
paths = set(context_documents or profiles)
|
|
evidence = [item for binding in bindings for item in binding_source_evidence(binding, oracle)]
|
|
# Multiple documents can independently prove the same tag/category. Gold
|
|
# sources are alternatives for each bound proof, not an obligation to cite
|
|
# every duplicate occurrence. Context profiles still cover all candidates.
|
|
evidence_choices = []
|
|
roles = {
|
|
"taxonomy.tag": "keyword_document",
|
|
"taxonomy.category": "category_document",
|
|
"taxonomy.author": "author_document",
|
|
"academic_record": "academic_document",
|
|
"start_year": "person_document",
|
|
}
|
|
if not no_answer:
|
|
for index, binding in enumerate(bindings):
|
|
per_role: dict[str, set[str]] = {}
|
|
for item in binding_source_evidence(binding, oracle):
|
|
per_role.setdefault(roles[item["field"]], set()).add(item["document_path"])
|
|
for role, alternatives in per_role.items():
|
|
evidence_choices.append(
|
|
{
|
|
"binding_index": index,
|
|
"role": role,
|
|
"any_of": [
|
|
{
|
|
"document_path": path,
|
|
"source_url": oracle["documents"][path]["source_url"],
|
|
}
|
|
for path in sorted(alternatives)
|
|
],
|
|
}
|
|
)
|
|
expected_values = []
|
|
expected_entities = []
|
|
if not no_answer:
|
|
for binding in bindings:
|
|
for field in [
|
|
"person",
|
|
"keyword",
|
|
"category",
|
|
"document_author",
|
|
"title",
|
|
"year",
|
|
"start_year",
|
|
]:
|
|
if field in binding:
|
|
value = str(binding[field])
|
|
expected_values.append(value)
|
|
kind = (
|
|
(
|
|
"thesis_title"
|
|
if binding.get("academic_kind") == "Thesis"
|
|
else "work_title"
|
|
)
|
|
if field == "title"
|
|
else field
|
|
)
|
|
expected_entities.append({"type": kind, "value": value})
|
|
if count is not None:
|
|
expected_values.append(str(count))
|
|
graph = {
|
|
"schema_version": 3,
|
|
"task": category,
|
|
"start_entities": [{"type": "person", "value": name} for name in names],
|
|
"expected_bindings": bindings if not no_answer else [],
|
|
"expected_evidence_documents": evidence_choices,
|
|
"expected_path_patterns": path_patterns(bindings) if not no_answer else [],
|
|
"expected_no_answer": no_answer,
|
|
"expected_status": "no_answer" if no_answer else "ok",
|
|
"expected_entities": expected_entities,
|
|
"expected_relations": [],
|
|
"expected_properties": [],
|
|
"source_profiles": profiles,
|
|
}
|
|
if no_answer:
|
|
graph["no_answer_basis"] = reason or "No source record satisfies every requested condition."
|
|
graph["negative_constraints"] = negative_constraints or {}
|
|
# Closed-world negative: retain all original metadata considered, not a fabricated positive fact.
|
|
evidence = [
|
|
{
|
|
"document_path": path,
|
|
"sha256": oracle["documents"][path]["sha256"],
|
|
"field": "closed_world_metadata",
|
|
"tags": oracle["documents"][path]["tags"],
|
|
"categories": oracle["documents"][path]["categories"],
|
|
}
|
|
for name in known
|
|
for path in oracle["people"][name]["documents"]
|
|
]
|
|
if expected_set is not None:
|
|
graph.update(
|
|
expected_entity_set=sorted(expected_set, key=norm),
|
|
entity_set_mode="exact",
|
|
entity_set_field=set_field or "person",
|
|
)
|
|
if count is not None:
|
|
graph["expected_count"] = count
|
|
graph["partition_group"] = fingerprint(
|
|
{
|
|
"category": category,
|
|
"people": sorted(names, key=norm),
|
|
"bindings": bindings,
|
|
"negative": no_answer,
|
|
"requested": question if no_answer else None,
|
|
}
|
|
)[:20]
|
|
return {
|
|
"id": case_id,
|
|
"split": split,
|
|
"category": category,
|
|
"difficulty": "hard",
|
|
"question": question,
|
|
"should_answer": not no_answer,
|
|
"expected_documents": sorted(paths),
|
|
"expected_source_urls": sorted(oracle["documents"][path]["source_url"] for path in paths),
|
|
"expected_answer_contains": list(dict.fromkeys(expected_values)),
|
|
"graph": graph,
|
|
"source_evidence": evidence,
|
|
"evidence_basis": ["independent_raw_markdown_oracle_v2"],
|
|
}
|
|
|
|
|
|
def correct_legacy(question: dict, oracle: dict) -> dict:
|
|
names = people_for(question, oracle)
|
|
cat = question["category"]
|
|
old = question["graph"]
|
|
entities = old.get("expected_entities", [])
|
|
keyword = next((item["value"] for item in entities if item["type"] == "keyword"), None)
|
|
years = [int(item["value"]) for item in entities if item["type"] == "year"]
|
|
bindings = []
|
|
expected_set = None
|
|
set_field = None
|
|
count = None
|
|
reason = None
|
|
if cat == "shared_keyword_pair":
|
|
if keyword and all(has_tag(oracle, name, keyword) for name in names):
|
|
bindings = [{"person": name, "keyword": keyword} for name in names]
|
|
else:
|
|
reason = "Requested shared keyword is absent from the raw frontmatter of at least one named person."
|
|
elif cat == "same_start_year_pair":
|
|
bindings = [
|
|
{"person": name, "start_year": oracle["people"][name]["start_year"]} for name in names
|
|
]
|
|
elif cat == "same_academic_year_pair":
|
|
common = set.intersection(
|
|
*[
|
|
{row["year"] for row in academics(oracle, name) if row["year"] is not None}
|
|
for name in names
|
|
]
|
|
)
|
|
bindings = [{"person": name, "year": year} for year in sorted(common) for name in names]
|
|
if not common:
|
|
reason = "No shared academic year is stated in the original sources."
|
|
elif cat in {"person_title_year", "person_keyword_title"}:
|
|
titles = [item for item in entities if item["type"] in {"thesis_title", "work_title"}]
|
|
for item in titles:
|
|
kind = "Thesis" if item["type"] == "thesis_title" else "Work"
|
|
rows = academics(oracle, names[0], title=item["value"], kind=kind)
|
|
if not rows and norm(item["value"]) in norm(
|
|
oracle["documents"][oracle["people"][names[0]]["profile"]]["raw_text"]
|
|
):
|
|
raise RuntimeError(
|
|
f"Oracle extraction needs review for supported raw title: {question['id']}"
|
|
)
|
|
if years:
|
|
rows = [row for row in rows if row["year"] in years]
|
|
if cat == "person_keyword_title" and not has_tag(oracle, names[0], keyword or ""):
|
|
reason = "Requested keyword is not a raw frontmatter tag of this person."
|
|
break
|
|
for row in rows:
|
|
binding = academic_binding(
|
|
names[0],
|
|
row,
|
|
title=True,
|
|
keyword=keyword if cat == "person_keyword_title" else None,
|
|
)
|
|
if cat == "person_keyword_title" and not years:
|
|
binding.pop("year", None)
|
|
bindings.append(binding)
|
|
if not bindings and not reason:
|
|
reason = "Requested title/type/year combination is not independently supported by the source."
|
|
elif cat == "document_author_category_multi_hop":
|
|
category = next(item["value"] for item in entities if item["type"] == "category")
|
|
for path in supporting_documents(oracle, names[0], category=category):
|
|
author = oracle["documents"][path]["author"]
|
|
if author:
|
|
bindings.append(
|
|
{
|
|
"person": names[0],
|
|
"category": category,
|
|
"document_author": author,
|
|
"document_path": path,
|
|
"same_document": True,
|
|
}
|
|
)
|
|
if not bindings:
|
|
reason = "No same document supports both the category and metadata author."
|
|
elif cat == "person_multiple_categories":
|
|
expected_set = oracle["people"][names[0]]["categories"]
|
|
set_field = "category"
|
|
bindings = [{"person": names[0], "category": category} for category in expected_set]
|
|
elif cat == "candidate_set_aggregation":
|
|
query_years = [int(x) for x in re.findall(r"\b20\d{2}\b", question["question"])]
|
|
year = query_years[-1]
|
|
matches = [name for name in names if academics(oracle, name, year=year)]
|
|
bindings = [{"person": name, "year": year} for name in matches]
|
|
expected_set = matches
|
|
set_field = "person"
|
|
count = len(matches)
|
|
elif cat == "conjunctive_disambiguation":
|
|
year = years[0]
|
|
matches = [
|
|
name
|
|
for name in names
|
|
if has_tag(oracle, name, keyword) and academics(oracle, name, year=year)
|
|
]
|
|
bindings = [{"person": name, "keyword": keyword, "year": year} for name in matches]
|
|
if not matches:
|
|
reason = "No named person satisfies both raw frontmatter keyword and academic year."
|
|
else:
|
|
raise ValueError(f"Unsupported legacy category: {cat}")
|
|
case = make_case(
|
|
question["id"],
|
|
"dev",
|
|
cat,
|
|
question["question"],
|
|
names,
|
|
oracle,
|
|
bindings,
|
|
no_answer=reason is not None,
|
|
reason=reason,
|
|
expected_set=expected_set,
|
|
set_field=set_field,
|
|
count=count,
|
|
negative_constraints=(
|
|
{"keyword": keyword, "require_all_people": True}
|
|
if cat == "shared_keyword_pair"
|
|
else (
|
|
{"keyword": keyword}
|
|
if cat == "person_keyword_title"
|
|
else (
|
|
{"keyword": keyword, "year": years[0]}
|
|
if cat == "conjunctive_disambiguation"
|
|
else {"shared_academic_year": True} if cat == "same_academic_year_pair" else {}
|
|
)
|
|
)
|
|
),
|
|
)
|
|
case["legacy_split"] = question["split"]
|
|
case["legacy_role"] = "previously_seen_regression_only"
|
|
return case
|
|
|
|
|
|
def independent_pool(oracle: dict, names: list[str]) -> dict[str, list[dict]]:
|
|
pool: dict[str, list[dict]] = {
|
|
category: []
|
|
for category in [
|
|
"shared_keyword_pair",
|
|
"same_academic_year_pair",
|
|
"same_start_year_pair",
|
|
"person_title_year",
|
|
"person_keyword_title",
|
|
"document_author_category_multi_hop",
|
|
"person_multiple_categories",
|
|
"candidate_set_aggregation",
|
|
"conjunctive_disambiguation",
|
|
]
|
|
}
|
|
|
|
def add(category: str, text: str, people: list[str], bindings: list[dict], **kwargs):
|
|
pool[category].append(
|
|
make_case("", "test", category, text, people, oracle, bindings, **kwargs)
|
|
)
|
|
|
|
for left, right in itertools.combinations(names, 2):
|
|
common_tags = sorted(
|
|
{
|
|
norm(tag): tag
|
|
for tag in oracle["people"][left]["tags"]
|
|
if has_tag(oracle, right, tag)
|
|
}.values(),
|
|
key=norm,
|
|
)
|
|
for tag in common_tags:
|
|
add(
|
|
"shared_keyword_pair",
|
|
f"Over spoločné kľúčové slovo {tag} pri osobách {left} a {right}.",
|
|
[left, right],
|
|
[{"person": name, "keyword": tag} for name in [left, right]],
|
|
)
|
|
shared_years = {row["year"] for row in academics(oracle, left)} & {
|
|
row["year"] for row in academics(oracle, right)
|
|
} - {None}
|
|
# A query for an unspecified shared year expects all supported intersections.
|
|
if shared_years:
|
|
add(
|
|
"same_academic_year_pair",
|
|
f"Aké spoločné roky práce spájajú osoby {left} a {right}?",
|
|
[left, right],
|
|
[
|
|
{"person": name, "year": year}
|
|
for year in sorted(shared_years)
|
|
for name in [left, right]
|
|
],
|
|
)
|
|
start_l, start_r = (
|
|
oracle["people"][left]["start_year"],
|
|
oracle["people"][right]["start_year"],
|
|
)
|
|
if start_l is not None and start_l == start_r:
|
|
add(
|
|
"same_start_year_pair",
|
|
f"Porovnaj rovnaký rok začiatku štúdia osôb {left} a {right}.",
|
|
[left, right],
|
|
[{"person": name, "start_year": start_l} for name in [left, right]],
|
|
)
|
|
for name in names:
|
|
person = oracle["people"][name]
|
|
if person["categories"]:
|
|
add(
|
|
"person_multiple_categories",
|
|
f"Vypíš všetky kategórie dokumentov osoby {name}.",
|
|
[name],
|
|
[{"person": name, "category": cat} for cat in person["categories"]],
|
|
expected_set=person["categories"],
|
|
set_field="category",
|
|
)
|
|
for category in person["categories"]:
|
|
for path in supporting_documents(oracle, name, category=category):
|
|
author = oracle["documents"][path]["author"]
|
|
if author:
|
|
add(
|
|
"document_author_category_multi_hop",
|
|
f"Kto je autor dokumentu osoby {name}, ktorý patrí do kategórie {category}?",
|
|
[name],
|
|
[
|
|
{
|
|
"person": name,
|
|
"category": category,
|
|
"document_author": author,
|
|
"document_path": path,
|
|
"same_document": True,
|
|
}
|
|
],
|
|
)
|
|
# Title-year cases use only explicit, nonconflicting title records.
|
|
for record in person["academics"]:
|
|
if not record["title"] or record["year"] is None or record in person["year_conflicts"]:
|
|
continue
|
|
year = record["year"]
|
|
kind_label = (
|
|
"diplomového projektu" if record["academic_type"] == "diploma_project" else "práce"
|
|
)
|
|
add(
|
|
"person_title_year",
|
|
f"Aký názov {kind_label} a rok práce sú pri osobe {name} uvedené pre rok {year}?",
|
|
[name],
|
|
[academic_binding(name, record, title=True)],
|
|
)
|
|
for tag in person["tags"]:
|
|
binding = academic_binding(name, record, title=True, keyword=tag)
|
|
# Pin the academic year in the question as the independent source can contain several titles.
|
|
add(
|
|
"person_keyword_title",
|
|
f"Aký názov práce osoby {name} v roku {year} súvisí s kľúčovým slovom {tag}?",
|
|
[name],
|
|
[binding],
|
|
)
|
|
# Candidate groups are source-derived; the query mentions every candidate.
|
|
for year in sorted(
|
|
{
|
|
row["year"]
|
|
for name in names
|
|
for row in academics(oracle, name)
|
|
if row["year"] is not None
|
|
}
|
|
):
|
|
matching = [name for name in names if academics(oracle, name, year=year)]
|
|
other = [name for name in names if name not in matching]
|
|
for selected in itertools.islice(itertools.combinations(matching, 2), 12):
|
|
if other:
|
|
candidates = list(selected) + [other[0]]
|
|
add(
|
|
"candidate_set_aggregation",
|
|
f'Z kandidátov {", ".join(candidates)} vyber osoby s prácou v roku {year}. Koľko ich je?',
|
|
candidates,
|
|
[{"person": name, "year": year} for name in selected],
|
|
expected_set=list(selected),
|
|
set_field="person",
|
|
count=len(selected),
|
|
)
|
|
for name in names:
|
|
person = oracle["people"][name]
|
|
for year in sorted(
|
|
{record["year"] for record in person["academics"] if record["year"] is not None}
|
|
):
|
|
for tag in person["tags"]:
|
|
distractors = [
|
|
other
|
|
for other in names
|
|
if other != name
|
|
and (has_tag(oracle, other, tag) != bool(academics(oracle, other, year=year)))
|
|
]
|
|
for other in distractors[:2]:
|
|
candidates = [name, other]
|
|
add(
|
|
"conjunctive_disambiguation",
|
|
f"Ktorá z osôb {name} a {other} má kľúčové slovo {tag} a jej práca je z roku {year}?",
|
|
candidates,
|
|
[{"person": name, "keyword": tag, "year": year}],
|
|
)
|
|
return pool
|
|
|
|
|
|
def necessary_negatives(oracle: dict, names: list[str], split: str) -> list[dict]:
|
|
cases = []
|
|
prefix = "dev" if split == "dev" else "holdout"
|
|
for i, name in enumerate(names[:4]):
|
|
tag = next(iter(oracle["people"][name]["tags"]), "nlp")
|
|
cases.append(
|
|
make_case(
|
|
f"{prefix}-negative-year-{i}",
|
|
split,
|
|
"negative_lookup",
|
|
f"Aký názov práce má osoba {name}, ak jej práca je z roku 2099 a jej dokument obsahuje kľúčové slovo {tag}?",
|
|
[name],
|
|
oracle,
|
|
[],
|
|
no_answer=True,
|
|
reason="No source academic record has year 2099.",
|
|
negative_constraints={"year": 2099, "keyword": tag},
|
|
)
|
|
)
|
|
for i, name in enumerate(names[4:6]):
|
|
cases.append(
|
|
make_case(
|
|
f"{prefix}-negative-tag-{i}",
|
|
split,
|
|
"negative_lookup",
|
|
f"Aký názov práce má osoba {name}, ak jej dokument obsahuje kľúčové slovo neexistujucitagvzor?",
|
|
[name],
|
|
oracle,
|
|
[],
|
|
no_answer=True,
|
|
reason="Requested tag absent from complete raw frontmatter catalog.",
|
|
negative_constraints={"keyword": "neexistujucitagvzor"},
|
|
)
|
|
)
|
|
for i, unknown in enumerate(
|
|
["Zuzana Neexistujúca Vzorová", "Roman Neskutočný Testovací"]
|
|
if split == "dev"
|
|
else ["Tibor Neexistujúci Skúšobný", "Lenka Neskutočná Vzorová"]
|
|
):
|
|
cases.append(
|
|
make_case(
|
|
f"{prefix}-negative-person-{i}",
|
|
split,
|
|
"negative_lookup",
|
|
f"Aký názov diplomovej práce má osoba {unknown}?",
|
|
[unknown],
|
|
oracle,
|
|
[],
|
|
no_answer=True,
|
|
reason="Explicit full name absent from all raw profile frontmatter titles.",
|
|
negative_constraints={"unknown_person": unknown},
|
|
)
|
|
)
|
|
candidates = names[:3]
|
|
cases.append(
|
|
make_case(
|
|
f"{prefix}-zero-count",
|
|
split,
|
|
"candidate_set_aggregation",
|
|
f'Z kandidátov {", ".join(candidates)} vyber osoby s prácou v roku 2099. Koľko ich je?',
|
|
candidates,
|
|
oracle,
|
|
[],
|
|
expected_set=[],
|
|
set_field="person",
|
|
count=0,
|
|
)
|
|
)
|
|
return cases
|
|
|
|
|
|
def validate_case(case: dict, oracle: dict) -> list[str]:
|
|
issues = []
|
|
graph = case["graph"]
|
|
names = people_for(case, oracle)
|
|
if set(graph.get("source_profiles", [])) != {
|
|
oracle["people"][name]["profile"] for name in names
|
|
}:
|
|
issues.append("incorrect_source_profile_partition")
|
|
for binding in graph.get("expected_bindings", []):
|
|
person = binding["person"]
|
|
if person not in oracle["people"]:
|
|
issues.append("unknown_bound_person")
|
|
continue
|
|
if (
|
|
"start_year" in binding
|
|
and oracle["people"][person]["start_year"] != binding["start_year"]
|
|
):
|
|
issues.append("wrong_start_year")
|
|
metadata = {
|
|
key: binding[key]
|
|
for key in ["keyword", "category", "document_author"]
|
|
if key in binding
|
|
}
|
|
if binding.get("same_document") and not supporting_documents(oracle, person, **metadata):
|
|
issues.append("metadata_not_connected_in_same_document")
|
|
for key, value in metadata.items():
|
|
if not supporting_documents(oracle, person, **{key: value}):
|
|
issues.append("unsupported_" + key)
|
|
if any(key in binding for key in ["title", "year", "academic_type", "academic_kind"]):
|
|
records = academics(
|
|
oracle,
|
|
person,
|
|
title=binding.get("title"),
|
|
year=binding.get("year"),
|
|
kind=binding.get("academic_kind"),
|
|
)
|
|
if binding.get("academic_type"):
|
|
records = [r for r in records if r["academic_type"] == binding["academic_type"]]
|
|
if not records:
|
|
issues.append("unsupported_academic_binding")
|
|
if graph.get("expected_no_answer"):
|
|
if (
|
|
graph.get("expected_bindings")
|
|
or case.get("expected_answer_contains")
|
|
or graph.get("expected_entities")
|
|
):
|
|
issues.append("negative_contains_positive_expectations")
|
|
constraints = graph.get("negative_constraints", {})
|
|
names = people_for(case, oracle)
|
|
if constraints.get("unknown_person"):
|
|
if norm(constraints["unknown_person"]) in {norm(name) for name in oracle["people"]}:
|
|
issues.append("negative_person_actually_exists")
|
|
elif constraints.get("missing_start_year"):
|
|
if names and all(oracle["people"][name]["start_year"] is not None for name in names):
|
|
issues.append("negative_start_year_actually_documented")
|
|
elif constraints.get("shared_academic_year"):
|
|
year_sets = [
|
|
{r["year"] for r in academics(oracle, name) if r["year"] is not None}
|
|
for name in names
|
|
]
|
|
if year_sets and set.intersection(*year_sets):
|
|
issues.append("negative_shared_year_actually_exists")
|
|
elif constraints.get("require_all_people"):
|
|
if not names or all(
|
|
has_tag(oracle, name, constraints.get("keyword", "")) for name in names
|
|
):
|
|
issues.append("negative_shared_keyword_actually_exists")
|
|
elif constraints:
|
|
matching = []
|
|
for name in names:
|
|
valid = not constraints.get("keyword") or has_tag(
|
|
oracle, name, constraints["keyword"]
|
|
)
|
|
if any(
|
|
key in constraints
|
|
for key in ["year", "title", "academic_kind", "title_required"]
|
|
):
|
|
records = academics(
|
|
oracle,
|
|
name,
|
|
year=constraints.get("year"),
|
|
title=constraints.get("title"),
|
|
kind=constraints.get("academic_kind"),
|
|
)
|
|
if constraints.get("academic_type"):
|
|
records = [
|
|
r for r in records if r["academic_type"] == constraints["academic_type"]
|
|
]
|
|
if constraints.get("title_required"):
|
|
records = [r for r in records if r.get("title")]
|
|
valid = valid and bool(records)
|
|
if valid:
|
|
matching.append(name)
|
|
if matching:
|
|
issues.append("negative_conjunction_actually_satisfied")
|
|
else:
|
|
issues.append("negative_without_machine_verifiable_false_premise")
|
|
if graph.get("expected_entity_set") is not None and graph.get("entity_set_field") == "category":
|
|
names = people_for(case, oracle)
|
|
if names and set(graph["expected_entity_set"]) != set(
|
|
oracle["people"][names[0]]["categories"]
|
|
):
|
|
issues.append("incomplete_category_set")
|
|
if graph.get("expected_count") is not None:
|
|
years = re.findall(r"\b\d{4}\b", case["question"])
|
|
if not years:
|
|
issues.append("count_without_supported_year_constraint")
|
|
else:
|
|
matches = {name for name in names if academics(oracle, name, year=int(years[-1]))}
|
|
if (
|
|
matches != set(graph.get("expected_entity_set", []))
|
|
or len(matches) != graph["expected_count"]
|
|
):
|
|
issues.append("incorrect_candidate_set_or_count")
|
|
return sorted(set(issues))
|
|
|
|
|
|
def audit_dataset(questions: list[dict], oracle: dict, dataset_path: Path) -> dict:
|
|
rows = [
|
|
{
|
|
"id": question["id"],
|
|
"split": question["split"],
|
|
"source_consistent": not (issues := validate_case(question, oracle)),
|
|
"issues": issues,
|
|
"category": question["category"],
|
|
}
|
|
for question in questions
|
|
]
|
|
inputs = {
|
|
name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
|
for name in ["data/documents.json", "data/chunks.json"]
|
|
}
|
|
return {
|
|
"method": "Independent original Markdown frontmatter/heading/title oracle; no graph parser, chunks or retrieval results used for labels.",
|
|
"dataset_sha256": hashlib.sha256(dataset_path.read_bytes()).hexdigest(),
|
|
"inputs": inputs,
|
|
"source_corpus": oracle["source_corpus"],
|
|
"questions": rows,
|
|
"by_split": {
|
|
split: {
|
|
"questions": len(selected),
|
|
"source_consistent": sum(row["source_consistent"] for row in selected),
|
|
"issues": dict(Counter(issue for row in selected for issue in row["issues"])),
|
|
}
|
|
for split in ["dev", "test"]
|
|
for selected in [[row for row in rows if row["split"] == split]]
|
|
},
|
|
}
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--wiki-root", type=Path, default=DEFAULT_WIKI)
|
|
parser.add_argument("--legacy", type=Path, default=BASE / "graphrag_questions.json")
|
|
parser.add_argument("--output", type=Path, default=BASE / "graphrag_questions_v2.json")
|
|
parser.add_argument("--artifacts", type=Path, default=BASE / "results/v2_preparation")
|
|
args = parser.parse_args()
|
|
oracle = load_source_oracle(args.wiki_root)
|
|
legacy = json.loads(args.legacy.read_text())
|
|
corrected = [correct_legacy(question, oracle) for question in legacy]
|
|
legacy_names = {name for question in legacy for name in people_for(question, oracle)}
|
|
exposed_path = BASE / "results/final_2026-10-24/holdout_failures.json"
|
|
previously_exposed = set()
|
|
if exposed_path.exists():
|
|
for row in json.loads(exposed_path.read_text()).get("questions", []):
|
|
previously_exposed.update(
|
|
set(row.get("actual_documents", [])) - set(row.get("expected_documents", []))
|
|
)
|
|
reserved_names = sorted(
|
|
[
|
|
name
|
|
for name, person in oracle["people"].items()
|
|
if name not in legacy_names
|
|
and person["profile"] not in previously_exposed
|
|
and person["profile"].startswith("pages/students/")
|
|
],
|
|
key=norm,
|
|
)
|
|
dev_negatives = necessary_negatives(oracle, sorted(legacy_names, key=norm), "dev")
|
|
test_pool = independent_pool(oracle, reserved_names)
|
|
rng = random.Random(SEED)
|
|
holdout = []
|
|
for category, cases in test_pool.items():
|
|
unique = {case["graph"]["partition_group"]: case for case in cases}
|
|
candidates = list(unique.values())
|
|
rng.shuffle(candidates)
|
|
if len(candidates) < 8:
|
|
raise RuntimeError(
|
|
f"Insufficient independent source cases for {category}: {len(candidates)}"
|
|
)
|
|
for case in candidates[:8]:
|
|
case["id"] = f"h{len(holdout)+1:04d}"
|
|
holdout.append(case)
|
|
negative_cases = necessary_negatives(oracle, reserved_names, "test")
|
|
# A zero-result count is an answerable numeric result, not a no-answer.
|
|
zero_count = negative_cases.pop()
|
|
count_index = next(
|
|
i for i, case in enumerate(holdout) if case["category"] == "candidate_set_aggregation"
|
|
)
|
|
zero_count["id"] = holdout[count_index]["id"]
|
|
holdout[count_index] = zero_count
|
|
holdout.extend(negative_cases)
|
|
dev = corrected + dev_negatives
|
|
questions = dev + holdout
|
|
all_issues = {
|
|
case["id"]: validate_case(case, oracle) for case in questions if validate_case(case, oracle)
|
|
}
|
|
if all_issues:
|
|
raise RuntimeError(f"Source consistency issues: {all_issues}")
|
|
dev_profiles = {p for case in dev for p in case["graph"]["source_profiles"]}
|
|
test_profiles = {p for case in holdout for p in case["graph"]["source_profiles"]}
|
|
assert not dev_profiles & test_profiles
|
|
assert not (
|
|
{case["graph"]["partition_group"] for case in dev}
|
|
& {case["graph"]["partition_group"] for case in holdout}
|
|
)
|
|
write_json(args.output, questions)
|
|
corrections = []
|
|
old_audit_path = BASE / "results/final_2026-10-24/source_audit.json"
|
|
old_issues = {
|
|
row["id"]: row["issues"]
|
|
for row in json.loads(old_audit_path.read_text())["questions"]
|
|
if row["issues"]
|
|
}
|
|
for old, new in zip(legacy, corrected):
|
|
corrections.append(
|
|
{
|
|
"id": old["id"],
|
|
"legacy_split": old["split"],
|
|
"new_split": "dev",
|
|
"v1_source_audit_issues": old_issues.get(old["id"], []),
|
|
"old_should_answer": old.get("should_answer", True),
|
|
"new_should_answer": new["should_answer"],
|
|
"old_expected_values": old["expected_answer_contains"],
|
|
"new_expected_values": new["expected_answer_contains"],
|
|
"new_expected_bindings": new["graph"]["expected_bindings"],
|
|
"no_answer_basis": new["graph"].get("no_answer_basis"),
|
|
"source_evidence": new["source_evidence"],
|
|
}
|
|
)
|
|
write_json(
|
|
args.artifacts / "legacy_corrections.json",
|
|
{
|
|
"legacy_count": len(legacy),
|
|
"v1_flagged_count": len(old_issues),
|
|
"resolved_v1_flagged_count": sum(
|
|
not validate_case(new, oracle) for new in corrected if new["id"] in old_issues
|
|
),
|
|
"negative_count": sum(not case["should_answer"] for case in corrected),
|
|
"questions": corrections,
|
|
},
|
|
)
|
|
write_json(args.artifacts / "source_audit.json", audit_dataset(questions, oracle, args.output))
|
|
partition = {
|
|
"seed": SEED,
|
|
"strategy": "All previously seen 500 cases are DEV regression. TEST profiles never appeared in legacy start_entities; all real candidate profiles are disjoint.",
|
|
"dev_count": len(dev),
|
|
"test_count": len(holdout),
|
|
"previously_exposed_profiles_excluded_from_test": sorted(previously_exposed),
|
|
"dev_profiles": sorted(dev_profiles),
|
|
"test_profiles": sorted(test_profiles),
|
|
"dev_groups": sorted({case["graph"]["partition_group"] for case in dev}),
|
|
"test_groups": sorted({case["graph"]["partition_group"] for case in holdout}),
|
|
"dev_categories": dict(Counter(case["category"] for case in dev)),
|
|
"test_categories": dict(Counter(case["category"] for case in holdout)),
|
|
"source_corpus": oracle["source_corpus"],
|
|
"dataset_sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(),
|
|
"limitations": [
|
|
"Same source corpus and shared tag/category/author vocabulary.",
|
|
"Template families are known; TEST persons and their fact groups are withheld.",
|
|
"Oracle is independent of graph extraction but still rule-based; provenance permits human review.",
|
|
],
|
|
}
|
|
write_json(args.artifacts / "partition.json", partition)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"dataset": str(args.output),
|
|
"dev": len(dev),
|
|
"test": len(holdout),
|
|
"reserved_profile_count": len(test_profiles),
|
|
"source_issues": 0,
|
|
"corrected_legacy_negatives": sum(not q["should_answer"] for q in corrected),
|
|
"resolved_original_issues": len(old_issues),
|
|
"test_categories": partition["test_categories"],
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|