"""Parse provenance before asking a model to identify concepts. Work years are never meeting dates. Partial dates stay partial; directory years and document authors are never promoted to dates or supervisors. """ from __future__ import annotations from datetime import date import hashlib import re from tools.rag.common import clean_heading_text from tools.rag.search_core import make_source_url from tools.answer_evaluation.context_policy import normalized from tools.answer_evaluation.rag_evidence import academic_scope from tools.answer_evaluation.content_selection import effective_role, evidence_phase PROFILE = re.compile(r'(?:pages/)?students/\d{4}/[^/]+/README\.md$') DATE = re.compile(r'(? str: return hashlib.sha256('\0'.join(str(v) for v in values).encode()).hexdigest()[:32] def supervisor_name(unit: dict) -> str | None: if unit['kind']!='supervisor' or ':' not in unit['text']: return None value=unit['text'].split(':',1)[1] value=re.sub(r'\[([^]]+)\]\([^)]*\)',r'\1',value) value=re.sub(r'\b(?:ing|mgr|bc|rndr|phdr|phd|doc|prof)\.?\b',' ',value,flags=re.I) words=re.findall(r'[^\W\d_]+(?:-[^\W\d_]+)*',value,re.UNICODE) return ' '.join(words) if 2<=len(words)<=4 and all(w[0].isupper() for w in words) else None def with_work_context(units: list[dict]) -> list[dict]: """Join a status only to titles/topics of the same profile and exact work.""" from collections import defaultdict topics=defaultdict(list) for row in units: if row['kind']=='topic' and row.get('work_label') and not row['text'].rstrip().endswith(':'): topics[(row['source_path'],row['work_label'])].append(row) return [{**row,'context_quotes':[u for u in topics[(row['source_path'],row.get('work_label'))] if u['id']!=row['id']][:2]} for row in units] def meeting_date(line: str) -> dict | None: text = normalized(line) if not re.search(r'\b(?:stretnutie|revizia|konzultacia)\b', text) and not DATE.fullmatch(line.strip().rstrip(':')): return None match = DATE.search(line) if not match: return {'date': None, 'date_label': line.strip(), 'date_precision': 'unknown'} day, month = int(match[1]), int(match[2]) try: date(int(match[3] or '2000'), month, day) except ValueError: return {'date': None, 'date_label': line.strip(), 'date_precision': 'unknown'} return {'date': f'{int(match[3]):04}-{month:02}-{day:02}' if match[3] else None, 'date_label': match[0], 'date_precision': 'day' if match[3] else 'day_month'} def profile_units(document: dict) -> list[dict]: """Keep literal source units, line numbers, work identity and explicit dates.""" person, path = document['title'].strip(), document['path'] scope = None work_label = None work_level = None role, source_phase = 'other', 'unknown' timestamp = {'date': None, 'date_label': None, 'date_precision': 'unknown'} units, buffer = [], [] start = 0 fence = None def flush(): nonlocal role if not buffer: return text = '\n'.join(buffer).strip() buffer.clear() plain = re.sub(r'^\s*(?:[-*+]|\d+[.)])\s+', '', text) if len(normalized(text).split()) < 3 or re.fullmatch(r'https?://\S+', plain): if role=='topic':role='other' return current_role = effective_role(text, role) observed=re.search(r'\b(?:dosiahli sme|dosiahol|dosiahla|ziskan\w*|spojen\w* datasety|urobene|' r'dokoncena|dokonceny|dotrenovan\w*|preroben\w*|vyskusany|vyskusane)\b',normalized(text)) if current_role=='other' and timestamp['date_label'] and observed: current_role='progress' if current_role=='other' and timestamp['date_label'] and re.search( r'\b(?:ziad\w* pokrok|nefung\w*|problem\w*|nepodar\w*|rozpracovan\w*|pracoval\w*)\b',normalized(text)): current_role='progress' phase = evidence_phase(text, current_role, source_phase) if current_role=='progress' and observed and not re.search( r'\b(?:treba|bude|budu|planuj\w*|nie je|nebol\w*|zatial nie)\b',normalized(text)): phase='result' checked = bool(re.match(r'^\s*[-*+]\s+\[[xX]\]',text)) if checked: phase='result' if phase == 'result' and re.search(r'\b(?:rozroben\w*|rozpracovan\w*|nedokoncen\w*)\b', normalized(text)): phase = 'progress' kind = {'assignment': 'assignment', 'task': 'plan', 'progress': 'progress','topic':'topic'}.get(current_role, 'context') label=normalized(text.split(':',1)[0]) if ':' in text else '' if label.startswith('nazov ') or label in {'nazov','tema'}: kind='topic' if phase == 'result': kind = 'result' if re.search(r'\b(?:meno veduceho|veduci(?: prace)?|veduca(?: prace)?|skolitel)\b', normalized(text)): kind, phase = 'supervisor', 'unknown' units.append({'id': identifier(path, work_label, start, text), 'person': person, 'source_path': path, 'source_url': make_source_url(path), 'line': start+document.get('body_line_offset',0), 'work_label': work_label, 'work_type': scope[0] if scope else None, 'work_year': scope[1] if scope else None, 'text': text, 'kind': kind, 'phase': phase, 'completion_basis':'checked_task' if checked else 'source_statement', **timestamp}) if role=='topic': role='other' role_labels = {'assignment': {'zadanie', 'navrh na zadanie', 'ciel', 'ciele', 'ciele na semester'}, 'topic': {'tema','zameranie','nazov','nazov prace'}, 'progress': {'stav', 'priebezny stav', 'hotove', 'urobene', 'vysledky','vystupy'}, 'task': {'ulohy', 'zasobnik uloh', 'treba dokoncit', 'plan'}} for number, line in enumerate(document['content'].splitlines(), 1): if re.match(r'^\s*(```|~~~)', line): flush() fence = None if fence else line.strip()[:3] continue if fence: continue heading = re.match(r'^\s{0,3}#{1,6}\s+(.+)', line) heading_level=len(line.lstrip())-len(line.lstrip().lstrip('#')) if heading and (new_scope := academic_scope(heading[1])): flush() scope, work_label = new_scope, clean_heading_text(heading[1]) work_level=heading_level role, source_phase = 'other', 'unknown' timestamp = {'date': None, 'date_label': None, 'date_precision': 'unknown'} continue if (new_date := meeting_date(line)) is not None: flush() timestamp = new_date role, source_phase = 'other', 'unknown' continue label = normalized(line.strip().strip('*_# ').rstrip(':')) if label in {'napr','napriklad','priklady','zdroje dokumentov','zdroje','suvisiace stranky','suvisiace prace'}: flush() role, source_phase = 'other', 'unknown' continue next_role = next((key for key, labels in role_labels.items() if label in labels or key == 'assignment' and label.startswith('zadanie ') or key == 'task' and re.match(r'^(?:ulohy (?:na|do|pre)|do dalsie\w* stretnutia)', label)), None) if next_role: flush() role = next_role source_phase = 'result' if label in {'hotove', 'urobene', 'vysledky','vystupy'} else 'unknown' continue if heading and work_level is not None and heading_level<=work_level: flush() scope,work_label,work_level=None,None,None role,source_phase='other','unknown' timestamp={'date':None,'date_label':None,'date_precision':'unknown'} if not line.strip() or re.match(r'^\s*(?:[-*+]\s+|\d+[.)]\s+)', line): flush() if line.strip(): if not buffer: start = number buffer.append(line) flush() return units