"""Answer guards using only actual retrieval context; no benchmark or oracle.""" from __future__ import annotations import re from html import unescape from html.parser import HTMLParser import unicodedata def normalized(value) -> str: text = unicodedata.normalize("NFKD", str(value or "")) return " ".join( re.findall( r"[a-z0-9]+", "".join(c for c in text if not unicodedata.combining(c)).casefold() ) ) class _HTMLLinks(HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) self.links = set() def handle_starttag(self, tag, attrs): self.links.update(value for key, value in attrs if key in {"href", "src"} and value) def text_urls(text: str) -> set[str]: """Inspect absolute, inline, reference-style Markdown and HTML links.""" text = unescape(text) urls = set() for url in re.findall(r'(?:[a-zA-Z][a-zA-Z0-9+.-]*://|www\.)[^\s<>"\x27`“”„«»‘’\]]+', text): url = url.rstrip(".,;:!?") while url.endswith(")") and url.count(")") > url.count("("): url = url[:-1] urls.add(url) for match in re.finditer(r"\]\(\s*", text): tail = text[match.end() :] if tail.startswith("<"): urls.add(tail[1:].split(">", 1)[0]) continue level = 0 end = 0 for end, char in enumerate(tail): if char.isspace() or (char == ")" and level == 0): break if char == "(": level += 1 if char == ")": level -= 1 else: end = len(tail) if tail[:end]: urls.add(tail[:end]) for match in re.finditer(r"^\s{0,3}\[[^]\n]+\]:\s*(?:<([^>]+)>|(\S+))", text, flags=re.M): urls.add(match[1] or match[2]) parser = _HTMLLinks() parser.feed(text) return urls | parser.links def citation_check(answer: dict, allowed: set[str]) -> dict: declared = answer.get("source_urls", []) if not isinstance(declared, list) or any(not isinstance(url, str) for url in declared): return {"citations_ok": False, "unsupported_urls": [], "reason": "invalid_source_urls"} used = set(declared) | text_urls(str(answer.get("answer", ""))) unsupported = sorted(used - allowed) return {"citations_ok": not unsupported, "unsupported_urls": unsupported} def allowed_urls(context: dict) -> set[str]: return {row["source_url"] for row in context.get("sources", []) if row.get("source_url")} def joined_candidates(facts: list[dict], intent: dict) -> list[dict]: """Join requested academic fields and tag by person, retaining real sources.""" needs_title = intent.get("mentions_title") needs_year = intent.get("mentions_academic_year") or intent.get("years") if not (needs_title or needs_year) or not ( intent.get("mentions_keyword") or intent.get("keyword_phrase") ): return [] result = [] for record in facts: if (needs_title and not record.get("title")) or ( not needs_title and record.get("year") is None ): continue for tag in facts: if normalized(tag.get("person")) != normalized(record.get("person")) or not tag.get( "keyword" ): continue fields = ["person", "title", "year"] if needs_title else ["person", "year"] candidate = {k: record[k] for k in fields if record.get(k) is not None} if record.get("work_type"): candidate["work_type"] = record["work_type"] candidate["keyword"] = tag["keyword"] candidate["evidence_documents"] = sorted( {p for p in [record.get("document_path"), tag.get("document_path")] if p} ) candidate["source_urls"] = sorted( {u for u in [record.get("source_url"), tag.get("source_url")] if u} ) if candidate not in result: result.append(candidate) return result def coverage_issues(answer: dict, context: dict) -> list[str]: incomplete = context.get("truncated") is True or context.get("complete") is False unknown = context.get("complete") is None clauses = [normalized(part) for part in re.split(r"[.!?;\n]+", answer.get("answer", ""))] caveat = any( re.search( r"(?:vypis|zoznam|vysled\w*|dokazy) (?:je |su |nie je |nie su )?(?:neupln\w*|orezan\w*)" r"|nie (?:je|su) (?:to )?upln\w*|nemozno zarucit uplnost|nemusi byt upln", clause, ) for clause in clauses ) issues = [] if incomplete and answer.get("status") == "ok" and not caveat: issues.append("missing_incompleteness_notice") if incomplete or unknown: for clause in clauses: for match in re.finditer( r"(?:upln\w*|kompletn\w*) (?:zoznam|vypis|prehlad)" r"|vset(?:ci|ky|kych) (?:student\w*|osob\w*|prac\w*)", clause, ): prefix = clause[: match.start()] # Negation must qualify this particular assertion. A caveat in # another sentence never licenses a positive exhaustive claim. if not re.search( r"(?:nie (?:je|su)(?: to)?|nejde o|neuvadzam|" r"nemozno (?:zarucit|uviest)|nemozem (?:zarucit|poskytnut))\s*$", prefix, ): issues.append("unsupported_exhaustive_statement") return list(dict.fromkeys(issues)) def prose_issues(answer: dict, context: dict | None = None) -> list[str]: """Conservative numeric consistency check; final prose is rendered from claims.""" text = str(answer.get("answer", "")) if context is not None and answer.get("status") == "no_answer": from tools.answers.presentation import no_answer_text expected = no_answer_text(context) # These years describe the failed query, never a positive factual claim. # Strip only the exact trusted template; appended prose stays checked. if text.startswith(expected): text = text[len(expected) :] for url in sorted(text_urls(text), key=len, reverse=True): text = text.replace(url, "") # A bounded ordered-list marker is presentation, not a factual count/year. # Accept only a contiguous sequence starting at one, with at most one item # per structured claim. Other numbers still need explicit factual support. expected = 1 def list_marker(match): nonlocal expected if int(match[1]) == expected and expected <= len(answer.get("claims", [])): expected += 1 return "" return match[0] text = re.sub(r"^\s*(\d+)[.)]\s+", list_marker, text, flags=re.M) allowed_numbers = set() for claim in answer.get("claims", []): for key, value in claim.items(): if value is None: continue if key in {"year", "start_year"}: allowed_numbers.add(str(value)) else: text = re.sub(re.escape(str(value)), "", text, flags=re.I) if answer.get("count") is not None: allowed_numbers.add(str(answer["count"])) unsupported = set(re.findall(r"\b\d+\b", text)) - allowed_numbers return ["unsupported_prose_number"] if unsupported else [] def person_token_pattern(token: str) -> str: """Bounded case endings of a complete name token, never fuzzy prefixes.""" token = normalized(token) suffix = r"(?:a|u|ovi|om|e|ou|y|ovej|oveho|ym|emu)?" variants = [re.escape(token) + (suffix if token.isalpha() else "")] if token.isalpha() and len(token) >= 3: if token.endswith("a"): variants.append(re.escape(token[:-1]) + r"(?:u|y|e|ou|ovi|om|ej)") elif token.endswith("y"): variants.append(re.escape(token[:-1]) + r"(?:eho|emu|om|ym)") return "(?:" + "|".join(variants) + ")" def person_pattern(person: str) -> str: return ( r"\b" + r"\s+".join(person_token_pattern(token) for token in normalized(person).split()) + r"\b" ) def text_contains_value(text: str, value: str, *, person: bool = False) -> bool: if not person: return normalized(value) in normalized(text) return bool(re.search(person_pattern(value), normalized(text))) def _same_claim_value(key: str, actual, expected) -> bool: # Opaque identifiers must not collapse paths/punctuation as prose does. return actual == expected if key == "record_id" else normalized(actual) == normalized(expected) def answer_context_issues(answer: dict, context: dict, *, check_text: bool = True) -> list[str]: issues = [] if not citation_check(answer, allowed_urls(context))["citations_ok"]: issues.append("unsupported_citation") facts = context.get("facts", []) aggregation = context.get("aggregation") or {} if answer.get("count") is not None: if aggregation.get("complete") is not True: issues.append("unsupported_total_count") elif answer["count"] != aggregation.get("count"): issues.append("incorrect_context_count") claims = answer.get("claims", []) cited = set(answer.get("source_urls", [])) if answer.get("status") == "ok" and facts and not claims: issues.append("missing_required_claims") if claims and not cited: issues.append("missing_required_citations") if answer.get("status") == "no_answer" and claims: issues.append("claims_on_no_answer") if check_text and not str(answer.get("answer", "")).strip(): issues.append("empty_answer_text") for claim in answer.get("claims", []): person = normalized(claim.get("person")) rows = [f for f in facts if normalized(f.get("person")) == person] if not rows: issues.append("unsupported_person") continue for key in ["keyword", "category", "document_author", "start_year"]: if claim.get(key) is not None and not any( normalized(row.get(key)) == normalized(claim[key]) for row in rows ): issues.append("unsupported_" + key) for key in [ "title", "year", "start_year", "keyword", "category", "document_author", "record_id", "work_type", "work_label", ]: if claim.get(key) is None: continue evidence = { row.get("source_url") for row in rows if _same_claim_value(key, row.get(key), claim[key]) and row.get("source_url") } if evidence and not evidence & cited: issues.append("missing_claim_source") academic = { k: claim[k] for k in ["title", "year", "record_id", "work_type", "work_label"] if claim.get(k) is not None } if academic and not any( all(_same_claim_value(k, row.get(k), v) for k, v in academic.items()) for row in rows ): issues.append("unsupported_academic_binding") if academic: bound_rows = [ row for row in rows if all(_same_claim_value(k, row.get(k), v) for k, v in academic.items()) ] if bound_rows and not any(row.get("source_url") in cited for row in bound_rows): issues.append("missing_academic_record_source") if claim.get("category") is not None and claim.get("document_author") is not None: if not any( a.get("document_path") and a.get("document_path") == b.get("document_path") for a in rows if normalized(a.get("category")) == normalized(claim["category"]) for b in rows if normalized(b.get("document_author")) == normalized(claim["document_author"]) ): issues.append("disconnected_author_category") required_records = context.get("required_academic_records", []) if context.get("status") == "ok" and required_records: if answer.get("status") != "ok": issues.append("rejected_supported_academic_listing") fields = ["person", "record_id", "title", "year", "work_type", "work_label"] for record in required_records: if not any( all( _same_claim_value(key, claim.get(key), record[key]) for key in fields if record.get(key) is not None ) for claim in claims ): issues.append("missing_academic_record_details") break for shared in context.get("shared_properties", []): for person in shared["people"]: if not any( normalized(c.get("person")) == normalized(person) and normalized(c.get(shared["field"])) == normalized(shared["value"]) for c in claims ): issues.append("missing_shared_property_evidence") candidates = context.get("answer_candidates", []) if context.get("status") == "ok" and candidates: if answer.get("status") == "no_answer": issues.append("rejected_supported_person_join") elif not all( any( all( normalized(claim.get(k)) == normalized(candidate[k]) for k in ["person", "title", "year", "keyword", "work_type"] if k in candidate ) for claim in answer.get("claims", []) ) for candidate in candidates ): issues.append("split_or_missing_person_join") if ( context.get("status") in {"no_answer", "ambiguous", "error"} and answer.get("status") == "ok" ): issues.append("unsupported_positive_status") issues.extend(prose_issues(answer, context)) if check_text or str(answer.get("answer", "")).strip(): issues.extend(coverage_issues(answer, context)) return list(dict.fromkeys(issues))