402 lines
15 KiB
Python
402 lines
15 KiB
Python
"""Parse provenance before asking a model to identify concepts.
|
|
|
|
Work years are never meeting dates. Partial dates stay partial; directory years
|
|
and document authors are never promoted to dates or supervisors.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import date
|
|
import hashlib
|
|
import re
|
|
|
|
from tools.rag.common import clean_heading_text
|
|
from tools.rag.search_core import make_source_url
|
|
from tools.answers.context_policy import normalized
|
|
from tools.answers.rag_evidence import academic_scope
|
|
from tools.answers.content_selection import effective_role, evidence_phase, reference_only
|
|
from tools.knowledge.evidence import actor_binding, temporal_fields, EvidenceAssertion, EXTRACTION_SCHEMA, claim_eligible
|
|
|
|
PROFILE = re.compile(r"(?:pages/)?students/\d{4}/[^/]+/README\.md$")
|
|
DATE = re.compile(r"(?<!\d)(\d{1,2})\.(\d{1,2})[.:](?:\s*([12]\d{3}))?(?!\d)")
|
|
LIFECYCLE = re.compile(
|
|
r"\b(?:ukoncil\w* predcasne|prerusil\w* studium|odstupil\w*|praca (?:bola )?zrusena)\b"
|
|
)
|
|
|
|
|
|
def identifier(*values) -> str:
|
|
return hashlib.sha256("\0".join(str(v) for v in values).encode()).hexdigest()[:32]
|
|
|
|
|
|
def supervisor_name(unit: dict) -> str | None:
|
|
return advisor_name(unit, "supervisor")
|
|
|
|
|
|
def advisor_name(unit: dict, role: str) -> str | None:
|
|
if unit["kind"] != role or unit.get("section_scope") in {"example", "references"}:
|
|
return None
|
|
plain = re.sub(r"[*_]", "", unit["text"]).strip().lstrip("-+ ")
|
|
label = (
|
|
r"(?:meno ved[uú]ceho(?: pr[aá]ce)?|ved[uú]c[iaí](?: pr[aá]ce)?|[šs]kolite[lľ](?:ka)?)"
|
|
if role == "supervisor"
|
|
else r"(?:meno konzultanta|konzultant(?:ka)?(?: pr[aá]ce)?)"
|
|
)
|
|
match = re.match(r"^" + label + r"\s*[:\-]?\s*(.+)$", plain, re.I)
|
|
if not match:
|
|
return None
|
|
value = match[1]
|
|
value = re.sub(r"\[([^]]+)\]\([^)]*\)", r"\1", value)
|
|
value = re.sub(r"\b(?:ing|mgr|bc|rndr|phdr|phd|doc|prof)\.?\b", " ", value, flags=re.I)
|
|
words = re.findall(r"[^\W\d_]+(?:-[^\W\d_]+)*", value, re.UNICODE)
|
|
return " ".join(words) if 2 <= len(words) <= 4 and all(w[0].isupper() for w in words) else None
|
|
|
|
|
|
def with_work_context(units: list[dict]) -> list[dict]:
|
|
"""Join a status only to titles/topics of the same profile and exact work."""
|
|
from collections import defaultdict
|
|
|
|
topics = defaultdict(list)
|
|
for row in units:
|
|
if (
|
|
row["kind"] == "topic"
|
|
and row.get("work_label")
|
|
and claim_eligible(row)
|
|
and not row["text"].rstrip().endswith(":")
|
|
):
|
|
topics[(row["source_path"], row["work_label"])].append(row)
|
|
return [
|
|
{
|
|
**row,
|
|
"context_quotes": [
|
|
u
|
|
for u in topics[(row["source_path"], row.get("work_label"))]
|
|
if u["id"] != row["id"]
|
|
][:2],
|
|
}
|
|
for row in units
|
|
]
|
|
|
|
|
|
def meeting_date(line: str) -> dict | None:
|
|
text = normalized(line)
|
|
if re.match(r"^(?:ulohy (?:na|do|pre)|do dalsie\w* stretnutia|zasobnik uloh)", text):
|
|
return None
|
|
dated_status = bool(
|
|
re.match(r"^(?:stav|stav pr[aá]ce)\s+\d{1,2}\.\d{1,2}", line.strip(" *#_"), re.I)
|
|
)
|
|
if (
|
|
not dated_status
|
|
and not re.search(r"\b(?:stretnutie|revizia|konzultacia)\b", text)
|
|
and not DATE.fullmatch(line.strip().rstrip(":"))
|
|
):
|
|
return None
|
|
match = DATE.search(line)
|
|
if not match:
|
|
return {"date": None, "date_label": line.strip(), "date_precision": "unknown"}
|
|
day, month = int(match[1]), int(match[2])
|
|
try:
|
|
date(int(match[3] or "2000"), month, day)
|
|
except ValueError:
|
|
return {"date": None, "date_label": line.strip(), "date_precision": "unknown"}
|
|
return {
|
|
"date": f"{int(match[3]):04}-{month:02}-{day:02}" if match[3] else None,
|
|
"date_label": match[0],
|
|
"date_precision": "day" if match[3] else "day_month",
|
|
}
|
|
|
|
|
|
def profile_units(document: dict, *, include_records: bool = False) -> list[dict]:
|
|
"""Keep literal source units, line numbers, work identity and explicit dates."""
|
|
person, path = document["title"].strip(), document["path"]
|
|
scope = None
|
|
work_label = None
|
|
work_level = None
|
|
role, source_phase = "other", "unknown"
|
|
timestamp = {"date": None, "date_label": None, "date_precision": "unknown"}
|
|
units, buffer = [], []
|
|
start = 0
|
|
fence = None
|
|
meeting_id = None
|
|
section_scope = "profile"
|
|
|
|
def provenance(row):
|
|
row.update(actor_binding(row))
|
|
row.update(temporal_fields(row["text"], row["phase"]))
|
|
row["record_date"] = row.get("date")
|
|
row["record_date_label"] = row.get("date_label")
|
|
row["section_scope"] = section_scope
|
|
row["work_id"] = identifier(path, work_label) if work_label else None
|
|
row["end_line"] = row["line"] + row["text"].count("\n")
|
|
row["assertion_schema"] = EXTRACTION_SCHEMA
|
|
EvidenceAssertion.model_validate(row)
|
|
return row
|
|
|
|
def record(text, number, kind):
|
|
return provenance({
|
|
"id": identifier(path, work_label, number, kind, text),
|
|
"person": person,
|
|
"source_path": path,
|
|
"source_url": make_source_url(path),
|
|
"line": number + document.get("body_line_offset", 0),
|
|
"work_label": work_label,
|
|
"work_type": scope[0] if scope else None,
|
|
"work_year": scope[1] if scope else None,
|
|
"text": text,
|
|
"kind": kind,
|
|
"phase": "unknown",
|
|
**timestamp,
|
|
"meeting_id": meeting_id,
|
|
})
|
|
|
|
def flush():
|
|
nonlocal role
|
|
if not buffer:
|
|
return
|
|
text = "\n".join(buffer).strip()
|
|
buffer.clear()
|
|
plain = re.sub(r"^\s*(?:[-*+]|\d+[.)])\s+", "", text)
|
|
lifecycle = bool(LIFECYCLE.search(normalized(text)))
|
|
words = normalized(text).split()
|
|
if (
|
|
not words
|
|
or (len(words) < 3 and not lifecycle and not include_records)
|
|
or re.fullmatch(r"https?://\S+", plain)
|
|
):
|
|
if role == "topic":
|
|
role = "other"
|
|
return
|
|
current_role = effective_role(text, role)
|
|
observed = re.search(
|
|
r"\b(?:dosiahli sme|dosiahol|dosiahla|ziskan\w*|spojen\w* datasety|urobene|"
|
|
r"dokoncena|dokonceny|dotrenovan\w*|preroben\w*|vyskusany|vyskusane)\b",
|
|
normalized(text),
|
|
)
|
|
if current_role == "other" and timestamp["date_label"] and observed:
|
|
current_role = "progress"
|
|
if (
|
|
current_role == "other"
|
|
and timestamp["date_label"]
|
|
and re.search(
|
|
r"\b(?:ziad\w* pokrok|nefung\w*|problem\w*|nepodar\w*|rozpracovan\w*|pracoval\w*)\b",
|
|
normalized(text),
|
|
)
|
|
):
|
|
current_role = "progress"
|
|
phase = evidence_phase(text, current_role, source_phase)
|
|
if (
|
|
current_role == "progress"
|
|
and observed
|
|
and not re.search(
|
|
r"\b(?:treba|bude|budu|planuj\w*|nie je|nebol\w*|zatial nie)\b", normalized(text)
|
|
)
|
|
):
|
|
phase = "result"
|
|
checked = bool(re.match(r"^\s*[-*+]\s+\[[xX]\]", text))
|
|
if checked and current_role != "assignment" and not re.search(
|
|
r"\b(?:ma implementovat|ma vyhodnotit|mal by|bude|treba|nie je)\b", normalized(text)
|
|
):
|
|
phase = "result"
|
|
if phase == "result" and re.search(
|
|
r"\b(?:rozroben\w*|rozpracovan\w*|nedokoncen\w*)\b", normalized(text)
|
|
):
|
|
phase = "progress"
|
|
kind = {
|
|
"assignment": "assignment",
|
|
"task": "plan",
|
|
"progress": "progress",
|
|
"topic": "topic",
|
|
}.get(current_role, "context")
|
|
label = normalized(text.split(":", 1)[0]) if ":" in text else ""
|
|
if label.startswith("nazov ") or label in {"nazov", "tema"}:
|
|
kind = "topic"
|
|
if phase == "result":
|
|
kind = "result"
|
|
if reference_only(text):
|
|
kind, phase = "reference", "unknown"
|
|
if lifecycle:
|
|
kind, phase = "lifecycle", "unknown"
|
|
if re.search(
|
|
r"\b(?:meno veduceho|veduci(?: prace)?|veduca(?: prace)?|skolitel)\b", normalized(text)
|
|
):
|
|
kind, phase = "supervisor", "unknown"
|
|
if re.match(r"^(?:meno konzultanta|konzultant(?:ka)?\b)", normalized(plain)):
|
|
kind, phase = "consultant", "unknown"
|
|
units.append(
|
|
provenance({
|
|
"id": identifier(path, work_label, start, text),
|
|
"person": person,
|
|
"source_path": path,
|
|
"source_url": make_source_url(path),
|
|
"line": start + document.get("body_line_offset", 0),
|
|
"work_label": work_label,
|
|
"work_type": scope[0] if scope else None,
|
|
"work_year": scope[1] if scope else None,
|
|
"text": text,
|
|
"kind": kind,
|
|
"phase": phase,
|
|
"completion_basis": "checked_task" if checked else "source_statement",
|
|
**({"meeting_id": meeting_id} if include_records else {}),
|
|
**timestamp,
|
|
})
|
|
)
|
|
if role == "topic":
|
|
role = "other"
|
|
|
|
role_labels = {
|
|
"assignment": {"zadanie", "navrh na zadanie", "ciel", "ciele", "ciele na semester"},
|
|
"topic": {"tema", "zameranie", "nazov", "nazov prace", "predbezny nazov", "navrh na nazov"},
|
|
"progress": {"stav", "priebezny stav", "hotove", "urobene", "vysledky", "vystupy"},
|
|
"task": {"ulohy", "zasobnik uloh", "treba dokoncit", "plan"},
|
|
}
|
|
for number, line in enumerate(document["content"].splitlines(), 1):
|
|
if re.match(r"^\s*(```|~~~)", line):
|
|
flush()
|
|
fence = None if fence else line.strip()[:3]
|
|
continue
|
|
if fence:
|
|
continue
|
|
heading = re.match(r"^\s{0,3}#{1,6}\s+(.+)", line)
|
|
heading_level = len(line.lstrip()) - len(line.lstrip().lstrip("#"))
|
|
if heading and (new_scope := academic_scope(heading[1])):
|
|
flush()
|
|
scope, work_label = new_scope, clean_heading_text(heading[1])
|
|
work_level = heading_level
|
|
role, source_phase = "other", "unknown"
|
|
timestamp = {"date": None, "date_label": None, "date_precision": "unknown"}
|
|
meeting_id = None
|
|
section_scope = "work"
|
|
if include_records:
|
|
units.append(record(line.strip(), number, "work_record"))
|
|
continue
|
|
if (new_date := meeting_date(line)) is not None:
|
|
flush()
|
|
timestamp = new_date
|
|
meeting_id = identifier(path, work_label, number, line.strip())
|
|
section_scope = "meeting_note"
|
|
if include_records:
|
|
units.append(record(line.strip(), number, "meeting"))
|
|
role, source_phase = "other", "unknown"
|
|
continue
|
|
label = normalized(line.strip().strip("*_# ").rstrip(":"))
|
|
if label in {"zasobnik uloh", "dlhodobe ulohy", "dlhodoby plan", "tyka sa dalsieho obdobia"}:
|
|
flush()
|
|
section_scope = "backlog" if label != "tyka sa dalsieho obdobia" else "future_period"
|
|
timestamp = {"date": None, "date_label": None, "date_precision": "unknown"}
|
|
meeting_id = None
|
|
role, source_phase = "task", "unknown"
|
|
continue
|
|
if label in {
|
|
"napr",
|
|
"napriklad",
|
|
"priklady",
|
|
"priklady otazok",
|
|
"zdroje dokumentov",
|
|
"zdroje",
|
|
"suvisiace stranky",
|
|
"suvisiace prace",
|
|
}:
|
|
flush()
|
|
role, source_phase = "other", "unknown"
|
|
section_scope = "example" if label in {"napr","napriklad","priklady","priklady otazok"} else "references"
|
|
continue
|
|
next_role = next(
|
|
(
|
|
key
|
|
for key, labels in role_labels.items()
|
|
if label in labels
|
|
or key == "assignment"
|
|
and label.startswith("zadanie ")
|
|
or key == "task"
|
|
and re.match(r"^(?:ulohy (?:na|do|pre)|do dalsie\w* stretnutia)", label)
|
|
),
|
|
None,
|
|
)
|
|
if next_role:
|
|
flush()
|
|
role = next_role
|
|
if section_scope in {"example","references"} and not meeting_id:
|
|
section_scope = "work" if work_label else "profile"
|
|
if next_role == "task" and meeting_id:
|
|
section_scope = "followup_task"
|
|
elif meeting_id:
|
|
section_scope = "meeting_note"
|
|
source_phase = (
|
|
"result" if label in {"hotove", "urobene", "vysledky", "vystupy"} else "unknown"
|
|
)
|
|
continue
|
|
if heading and work_level is not None and heading_level <= work_level:
|
|
flush()
|
|
scope, work_label, work_level = None, None, None
|
|
role, source_phase = "other", "unknown"
|
|
timestamp = {"date": None, "date_label": None, "date_precision": "unknown"}
|
|
meeting_id = None
|
|
section_scope = "profile"
|
|
inline_label = normalized(line.split(":", 1)[0]) if ":" in line else ""
|
|
if (
|
|
LIFECYCLE.search(normalized(line))
|
|
or inline_label
|
|
in {
|
|
"nazov",
|
|
"nazov prace",
|
|
"predbezny nazov",
|
|
"tema",
|
|
"veduci",
|
|
"veduci prace",
|
|
"meno veduceho",
|
|
"meno veduceho prace",
|
|
"skolitel",
|
|
"konzultant",
|
|
"konzultant prace",
|
|
"meno konzultanta",
|
|
"repozitar",
|
|
}
|
|
or re.match(r"^\s*(?:konzultant(?:ka)?|vedúci|veduci)\s+", line, re.I)
|
|
):
|
|
flush()
|
|
if not line.strip() or re.match(r"^\s*(?:[-*+]\s+|\d+[.)]\s+)", line):
|
|
flush()
|
|
if line.strip():
|
|
if not buffer:
|
|
start = number
|
|
buffer.append(line)
|
|
flush()
|
|
return units
|
|
|
|
|
|
def source_lifecycle_notes(wiki_root, source_digest):
|
|
"""Read explicit lifecycle qualifications only from the indexed snapshot."""
|
|
return _source_lifecycle_notes(str(wiki_root), source_digest)
|
|
|
|
|
|
from functools import lru_cache
|
|
|
|
|
|
@lru_cache(maxsize=2)
|
|
def _source_lifecycle_notes(wiki_root, source_digest):
|
|
import json
|
|
from pathlib import Path
|
|
from tools.sync.sync_indexes import source_files
|
|
from tools.rag.common import load_zpwiki_page
|
|
|
|
root = Path(wiki_root)
|
|
if not (root / "pages").is_dir():
|
|
return []
|
|
files = source_files(root)
|
|
if hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest() != source_digest:
|
|
return [] # Never attach qualifications from a different source version.
|
|
notes = []
|
|
for path in files:
|
|
if not PROFILE.fullmatch(path):
|
|
continue
|
|
if not LIFECYCLE.search(normalized((root / path).read_text())):
|
|
continue
|
|
document = load_zpwiki_page(root / path, wiki_root=root)
|
|
if document.get("published") is False:
|
|
continue
|
|
notes.extend(
|
|
{**row, "evidence_role": "lifecycle"}
|
|
for row in profile_units(document)
|
|
if row["kind"] == "lifecycle"
|
|
)
|
|
return notes
|