345 lines
14 KiB
Python
345 lines
14 KiB
Python
"""Answer guards using only actual retrieval context; no benchmark or oracle."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from html import unescape
|
|
from html.parser import HTMLParser
|
|
import unicodedata
|
|
|
|
|
|
def normalized(value) -> str:
|
|
text = unicodedata.normalize("NFKD", str(value or ""))
|
|
return " ".join(
|
|
re.findall(
|
|
r"[a-z0-9]+", "".join(c for c in text if not unicodedata.combining(c)).casefold()
|
|
)
|
|
)
|
|
|
|
|
|
class _HTMLLinks(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.links = set()
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
self.links.update(value for key, value in attrs if key in {"href", "src"} and value)
|
|
|
|
|
|
def text_urls(text: str) -> set[str]:
|
|
"""Inspect absolute, inline, reference-style Markdown and HTML links."""
|
|
text = unescape(text)
|
|
urls = set()
|
|
for url in re.findall(r'(?:[a-zA-Z][a-zA-Z0-9+.-]*://|www\.)[^\s<>"\x27`“”„«»‘’\]]+', text):
|
|
url = url.rstrip(".,;:!?")
|
|
while url.endswith(")") and url.count(")") > url.count("("):
|
|
url = url[:-1]
|
|
urls.add(url)
|
|
for match in re.finditer(r"\]\(\s*", text):
|
|
tail = text[match.end() :]
|
|
if tail.startswith("<"):
|
|
urls.add(tail[1:].split(">", 1)[0])
|
|
continue
|
|
level = 0
|
|
end = 0
|
|
for end, char in enumerate(tail):
|
|
if char.isspace() or (char == ")" and level == 0):
|
|
break
|
|
if char == "(":
|
|
level += 1
|
|
if char == ")":
|
|
level -= 1
|
|
else:
|
|
end = len(tail)
|
|
if tail[:end]:
|
|
urls.add(tail[:end])
|
|
for match in re.finditer(r"^\s{0,3}\[[^]\n]+\]:\s*(?:<([^>]+)>|(\S+))", text, flags=re.M):
|
|
urls.add(match[1] or match[2])
|
|
parser = _HTMLLinks()
|
|
parser.feed(text)
|
|
return urls | parser.links
|
|
|
|
|
|
def citation_check(answer: dict, allowed: set[str]) -> dict:
|
|
declared = answer.get("source_urls", [])
|
|
if not isinstance(declared, list) or any(not isinstance(url, str) for url in declared):
|
|
return {"citations_ok": False, "unsupported_urls": [], "reason": "invalid_source_urls"}
|
|
used = set(declared) | text_urls(str(answer.get("answer", "")))
|
|
unsupported = sorted(used - allowed)
|
|
return {"citations_ok": not unsupported, "unsupported_urls": unsupported}
|
|
|
|
|
|
def allowed_urls(context: dict) -> set[str]:
|
|
return {row["source_url"] for row in context.get("sources", []) if row.get("source_url")}
|
|
|
|
|
|
def joined_candidates(facts: list[dict], intent: dict) -> list[dict]:
|
|
"""Join requested academic fields and tag by person, retaining real sources."""
|
|
needs_title = intent.get("mentions_title")
|
|
needs_year = intent.get("mentions_academic_year") or intent.get("years")
|
|
if not (needs_title or needs_year) or not (
|
|
intent.get("mentions_keyword") or intent.get("keyword_phrase")
|
|
):
|
|
return []
|
|
result = []
|
|
for record in facts:
|
|
if (needs_title and not record.get("title")) or (
|
|
not needs_title and record.get("year") is None
|
|
):
|
|
continue
|
|
for tag in facts:
|
|
if normalized(tag.get("person")) != normalized(record.get("person")) or not tag.get(
|
|
"keyword"
|
|
):
|
|
continue
|
|
fields = ["person", "title", "year"] if needs_title else ["person", "year"]
|
|
candidate = {k: record[k] for k in fields if record.get(k) is not None}
|
|
if record.get("work_type"):
|
|
candidate["work_type"] = record["work_type"]
|
|
candidate["keyword"] = tag["keyword"]
|
|
candidate["evidence_documents"] = sorted(
|
|
{p for p in [record.get("document_path"), tag.get("document_path")] if p}
|
|
)
|
|
candidate["source_urls"] = sorted(
|
|
{u for u in [record.get("source_url"), tag.get("source_url")] if u}
|
|
)
|
|
if candidate not in result:
|
|
result.append(candidate)
|
|
return result
|
|
|
|
|
|
def coverage_issues(answer: dict, context: dict) -> list[str]:
|
|
incomplete = context.get("truncated") is True or context.get("complete") is False
|
|
unknown = context.get("complete") is None
|
|
clauses = [normalized(part) for part in re.split(r"[.!?;\n]+", answer.get("answer", ""))]
|
|
caveat = any(
|
|
re.search(
|
|
r"(?:vypis|zoznam|vysled\w*|dokazy) (?:je |su |nie je |nie su )?(?:neupln\w*|orezan\w*)"
|
|
r"|nie (?:je|su) (?:to )?upln\w*|nemozno zarucit uplnost|nemusi byt upln",
|
|
clause,
|
|
)
|
|
for clause in clauses
|
|
)
|
|
issues = []
|
|
if incomplete and answer.get("status") == "ok" and not caveat:
|
|
issues.append("missing_incompleteness_notice")
|
|
if incomplete or unknown:
|
|
for clause in clauses:
|
|
for match in re.finditer(
|
|
r"(?:upln\w*|kompletn\w*) (?:zoznam|vypis|prehlad)"
|
|
r"|vset(?:ci|ky|kych) (?:student\w*|osob\w*|prac\w*)",
|
|
clause,
|
|
):
|
|
prefix = clause[: match.start()]
|
|
# Negation must qualify this particular assertion. A caveat in
|
|
# another sentence never licenses a positive exhaustive claim.
|
|
if not re.search(
|
|
r"(?:nie (?:je|su)(?: to)?|nejde o|neuvadzam|"
|
|
r"nemozno (?:zarucit|uviest)|nemozem (?:zarucit|poskytnut))\s*$",
|
|
prefix,
|
|
):
|
|
issues.append("unsupported_exhaustive_statement")
|
|
return list(dict.fromkeys(issues))
|
|
|
|
|
|
def prose_issues(answer: dict, context: dict | None = None) -> list[str]:
|
|
"""Conservative numeric consistency check; final prose is rendered from claims."""
|
|
text = str(answer.get("answer", ""))
|
|
if context is not None and answer.get("status") == "no_answer":
|
|
from tools.answers.presentation import no_answer_text
|
|
|
|
expected = no_answer_text(context)
|
|
# These years describe the failed query, never a positive factual claim.
|
|
# Strip only the exact trusted template; appended prose stays checked.
|
|
if text.startswith(expected):
|
|
text = text[len(expected) :]
|
|
for url in sorted(text_urls(text), key=len, reverse=True):
|
|
text = text.replace(url, "")
|
|
# A bounded ordered-list marker is presentation, not a factual count/year.
|
|
# Accept only a contiguous sequence starting at one, with at most one item
|
|
# per structured claim. Other numbers still need explicit factual support.
|
|
expected = 1
|
|
|
|
def list_marker(match):
|
|
nonlocal expected
|
|
if int(match[1]) == expected and expected <= len(answer.get("claims", [])):
|
|
expected += 1
|
|
return ""
|
|
return match[0]
|
|
|
|
text = re.sub(r"^\s*(\d+)[.)]\s+", list_marker, text, flags=re.M)
|
|
allowed_numbers = set()
|
|
for claim in answer.get("claims", []):
|
|
for key, value in claim.items():
|
|
if value is None:
|
|
continue
|
|
if key in {"year", "start_year"}:
|
|
allowed_numbers.add(str(value))
|
|
else:
|
|
text = re.sub(re.escape(str(value)), "", text, flags=re.I)
|
|
if answer.get("count") is not None:
|
|
allowed_numbers.add(str(answer["count"]))
|
|
unsupported = set(re.findall(r"\b\d+\b", text)) - allowed_numbers
|
|
return ["unsupported_prose_number"] if unsupported else []
|
|
|
|
|
|
def person_token_pattern(token: str) -> str:
|
|
"""Bounded case endings of a complete name token, never fuzzy prefixes."""
|
|
token = normalized(token)
|
|
suffix = r"(?:a|u|ovi|om|e|ou|y|ovej|oveho|ym|emu)?"
|
|
variants = [re.escape(token) + (suffix if token.isalpha() else "")]
|
|
if token.isalpha() and len(token) >= 3:
|
|
if token.endswith("a"):
|
|
variants.append(re.escape(token[:-1]) + r"(?:u|y|e|ou|ovi|om|ej)")
|
|
elif token.endswith("y"):
|
|
variants.append(re.escape(token[:-1]) + r"(?:eho|emu|om|ym)")
|
|
return "(?:" + "|".join(variants) + ")"
|
|
|
|
|
|
def person_pattern(person: str) -> str:
|
|
return (
|
|
r"\b"
|
|
+ r"\s+".join(person_token_pattern(token) for token in normalized(person).split())
|
|
+ r"\b"
|
|
)
|
|
|
|
|
|
def text_contains_value(text: str, value: str, *, person: bool = False) -> bool:
|
|
if not person:
|
|
return normalized(value) in normalized(text)
|
|
return bool(re.search(person_pattern(value), normalized(text)))
|
|
|
|
|
|
def _same_claim_value(key: str, actual, expected) -> bool:
|
|
# Opaque identifiers must not collapse paths/punctuation as prose does.
|
|
return actual == expected if key == "record_id" else normalized(actual) == normalized(expected)
|
|
|
|
|
|
def answer_context_issues(answer: dict, context: dict, *, check_text: bool = True) -> list[str]:
|
|
issues = []
|
|
if not citation_check(answer, allowed_urls(context))["citations_ok"]:
|
|
issues.append("unsupported_citation")
|
|
facts = context.get("facts", [])
|
|
aggregation = context.get("aggregation") or {}
|
|
if answer.get("count") is not None:
|
|
if aggregation.get("complete") is not True:
|
|
issues.append("unsupported_total_count")
|
|
elif answer["count"] != aggregation.get("count"):
|
|
issues.append("incorrect_context_count")
|
|
claims = answer.get("claims", [])
|
|
cited = set(answer.get("source_urls", []))
|
|
if answer.get("status") == "ok" and facts and not claims:
|
|
issues.append("missing_required_claims")
|
|
if claims and not cited:
|
|
issues.append("missing_required_citations")
|
|
if answer.get("status") == "no_answer" and claims:
|
|
issues.append("claims_on_no_answer")
|
|
if check_text and not str(answer.get("answer", "")).strip():
|
|
issues.append("empty_answer_text")
|
|
for claim in answer.get("claims", []):
|
|
person = normalized(claim.get("person"))
|
|
rows = [f for f in facts if normalized(f.get("person")) == person]
|
|
if not rows:
|
|
issues.append("unsupported_person")
|
|
continue
|
|
for key in ["keyword", "category", "document_author", "start_year"]:
|
|
if claim.get(key) is not None and not any(
|
|
normalized(row.get(key)) == normalized(claim[key]) for row in rows
|
|
):
|
|
issues.append("unsupported_" + key)
|
|
for key in [
|
|
"title",
|
|
"year",
|
|
"start_year",
|
|
"keyword",
|
|
"category",
|
|
"document_author",
|
|
"record_id",
|
|
"work_type",
|
|
"work_label",
|
|
]:
|
|
if claim.get(key) is None:
|
|
continue
|
|
evidence = {
|
|
row.get("source_url")
|
|
for row in rows
|
|
if _same_claim_value(key, row.get(key), claim[key]) and row.get("source_url")
|
|
}
|
|
if evidence and not evidence & cited:
|
|
issues.append("missing_claim_source")
|
|
academic = {
|
|
k: claim[k]
|
|
for k in ["title", "year", "record_id", "work_type", "work_label"]
|
|
if claim.get(k) is not None
|
|
}
|
|
if academic and not any(
|
|
all(_same_claim_value(k, row.get(k), v) for k, v in academic.items()) for row in rows
|
|
):
|
|
issues.append("unsupported_academic_binding")
|
|
if academic:
|
|
bound_rows = [
|
|
row
|
|
for row in rows
|
|
if all(_same_claim_value(k, row.get(k), v) for k, v in academic.items())
|
|
]
|
|
if bound_rows and not any(row.get("source_url") in cited for row in bound_rows):
|
|
issues.append("missing_academic_record_source")
|
|
if claim.get("category") is not None and claim.get("document_author") is not None:
|
|
if not any(
|
|
a.get("document_path") and a.get("document_path") == b.get("document_path")
|
|
for a in rows
|
|
if normalized(a.get("category")) == normalized(claim["category"])
|
|
for b in rows
|
|
if normalized(b.get("document_author")) == normalized(claim["document_author"])
|
|
):
|
|
issues.append("disconnected_author_category")
|
|
required_records = context.get("required_academic_records", [])
|
|
if context.get("status") == "ok" and required_records:
|
|
if answer.get("status") != "ok":
|
|
issues.append("rejected_supported_academic_listing")
|
|
fields = ["person", "record_id", "title", "year", "work_type", "work_label"]
|
|
for record in required_records:
|
|
if not any(
|
|
all(
|
|
_same_claim_value(key, claim.get(key), record[key])
|
|
for key in fields
|
|
if record.get(key) is not None
|
|
)
|
|
for claim in claims
|
|
):
|
|
issues.append("missing_academic_record_details")
|
|
break
|
|
for shared in context.get("shared_properties", []):
|
|
for person in shared["people"]:
|
|
if not any(
|
|
normalized(c.get("person")) == normalized(person)
|
|
and normalized(c.get(shared["field"])) == normalized(shared["value"])
|
|
for c in claims
|
|
):
|
|
issues.append("missing_shared_property_evidence")
|
|
candidates = context.get("answer_candidates", [])
|
|
if context.get("status") == "ok" and candidates:
|
|
if answer.get("status") == "no_answer":
|
|
issues.append("rejected_supported_person_join")
|
|
elif not all(
|
|
any(
|
|
all(
|
|
normalized(claim.get(k)) == normalized(candidate[k])
|
|
for k in ["person", "title", "year", "keyword", "work_type"]
|
|
if k in candidate
|
|
)
|
|
for claim in answer.get("claims", [])
|
|
)
|
|
for candidate in candidates
|
|
):
|
|
issues.append("split_or_missing_person_join")
|
|
if (
|
|
context.get("status") in {"no_answer", "ambiguous", "error"}
|
|
and answer.get("status") == "ok"
|
|
):
|
|
issues.append("unsupported_positive_status")
|
|
issues.extend(prose_issues(answer, context))
|
|
if check_text or str(answer.get("answer", "")).strip():
|
|
issues.extend(coverage_issues(answer, context))
|
|
return list(dict.fromkeys(issues))
|