234 lines
12 KiB
Python
234 lines
12 KiB
Python
"""Parse provenance before asking a model to identify concepts.
|
|
|
|
Work years are never meeting dates. Partial dates stay partial; directory years
|
|
and document authors are never promoted to dates or supervisors.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import date
|
|
import hashlib
|
|
import re
|
|
|
|
from tools.rag.common import clean_heading_text
|
|
from tools.rag.search_core import make_source_url
|
|
from tools.answer_evaluation.context_policy import normalized
|
|
from tools.answer_evaluation.rag_evidence import academic_scope
|
|
from tools.answer_evaluation.content_selection import effective_role, evidence_phase, reference_only
|
|
|
|
PROFILE = re.compile(r'(?:pages/)?students/\d{4}/[^/]+/README\.md$')
|
|
DATE = re.compile(r'(?<!\d)(\d{1,2})\.(\d{1,2})[.:](?:\s*([12]\d{3}))?(?!\d)')
|
|
LIFECYCLE = re.compile(r'\b(?:ukoncil\w* predcasne|prerusil\w* studium|odstupil\w*|praca (?:bola )?zrusena)\b')
|
|
|
|
|
|
def identifier(*values) -> str:
|
|
return hashlib.sha256('\0'.join(str(v) for v in values).encode()).hexdigest()[:32]
|
|
|
|
|
|
def supervisor_name(unit: dict) -> str | None:
|
|
return advisor_name(unit, 'supervisor')
|
|
|
|
|
|
def advisor_name(unit: dict, role: str) -> str | None:
|
|
if unit['kind'] != role:
|
|
return None
|
|
plain=re.sub(r'[*_]','',unit['text']).strip().lstrip('-+ ')
|
|
label = (r'(?:meno ved[uú]ceho(?: pr[aá]ce)?|ved[uú]c[iaí](?: pr[aá]ce)?|[šs]kolite[lľ](?:ka)?)'
|
|
if role == 'supervisor' else r'(?:meno konzultanta|konzultant(?:ka)?(?: pr[aá]ce)?)')
|
|
match=re.match(r'^'+label+r'\s*[:\-]?\s*(.+)$',plain,re.I)
|
|
if not match:return None
|
|
value=match[1]
|
|
value=re.sub(r'\[([^]]+)\]\([^)]*\)',r'\1',value)
|
|
value=re.sub(r'\b(?:ing|mgr|bc|rndr|phdr|phd|doc|prof)\.?\b',' ',value,flags=re.I)
|
|
words=re.findall(r'[^\W\d_]+(?:-[^\W\d_]+)*',value,re.UNICODE)
|
|
return ' '.join(words) if 2<=len(words)<=4 and all(w[0].isupper() for w in words) else None
|
|
|
|
|
|
def with_work_context(units: list[dict]) -> list[dict]:
|
|
"""Join a status only to titles/topics of the same profile and exact work."""
|
|
from collections import defaultdict
|
|
topics=defaultdict(list)
|
|
for row in units:
|
|
if row['kind']=='topic' and row.get('work_label') and not row['text'].rstrip().endswith(':'):
|
|
topics[(row['source_path'],row['work_label'])].append(row)
|
|
return [{**row,'context_quotes':[u for u in topics[(row['source_path'],row.get('work_label'))]
|
|
if u['id']!=row['id']][:2]} for row in units]
|
|
|
|
|
|
def meeting_date(line: str) -> dict | None:
|
|
text = normalized(line)
|
|
dated_status=bool(re.match(r'^(?:stav|stav pr[aá]ce)\s+\d{1,2}\.\d{1,2}',line.strip(' *#_'),re.I))
|
|
if not dated_status and not re.search(r'\b(?:stretnutie|revizia|konzultacia)\b', text) and not DATE.fullmatch(line.strip().rstrip(':')):
|
|
return None
|
|
match = DATE.search(line)
|
|
if not match:
|
|
return {'date': None, 'date_label': line.strip(), 'date_precision': 'unknown'}
|
|
day, month = int(match[1]), int(match[2])
|
|
try:
|
|
date(int(match[3] or '2000'), month, day)
|
|
except ValueError:
|
|
return {'date': None, 'date_label': line.strip(), 'date_precision': 'unknown'}
|
|
return {'date': f'{int(match[3]):04}-{month:02}-{day:02}' if match[3] else None,
|
|
'date_label': match[0], 'date_precision': 'day' if match[3] else 'day_month'}
|
|
|
|
|
|
def profile_units(document: dict, *, include_records: bool = False) -> list[dict]:
|
|
"""Keep literal source units, line numbers, work identity and explicit dates."""
|
|
person, path = document['title'].strip(), document['path']
|
|
scope = None
|
|
work_label = None
|
|
work_level = None
|
|
role, source_phase = 'other', 'unknown'
|
|
timestamp = {'date': None, 'date_label': None, 'date_precision': 'unknown'}
|
|
units, buffer = [], []
|
|
start = 0
|
|
fence = None
|
|
meeting_id = None
|
|
|
|
def record(text, number, kind):
|
|
return {'id':identifier(path,work_label,number,kind,text),'person':person,
|
|
'source_path':path,'source_url':make_source_url(path),
|
|
'line':number+document.get('body_line_offset',0),'work_label':work_label,
|
|
'work_type':scope[0] if scope else None,'work_year':scope[1] if scope else None,
|
|
'text':text,'kind':kind,'phase':'unknown',**timestamp,'meeting_id':meeting_id}
|
|
|
|
def flush():
|
|
nonlocal role
|
|
if not buffer:
|
|
return
|
|
text = '\n'.join(buffer).strip()
|
|
buffer.clear()
|
|
plain = re.sub(r'^\s*(?:[-*+]|\d+[.)])\s+', '', text)
|
|
lifecycle=bool(LIFECYCLE.search(normalized(text)))
|
|
words=normalized(text).split()
|
|
if not words or (len(words) < 3 and not lifecycle and not include_records) or re.fullmatch(r'https?://\S+', plain):
|
|
if role=='topic':role='other'
|
|
return
|
|
current_role = effective_role(text, role)
|
|
observed=re.search(r'\b(?:dosiahli sme|dosiahol|dosiahla|ziskan\w*|spojen\w* datasety|urobene|'
|
|
r'dokoncena|dokonceny|dotrenovan\w*|preroben\w*|vyskusany|vyskusane)\b',normalized(text))
|
|
if current_role=='other' and timestamp['date_label'] and observed:
|
|
current_role='progress'
|
|
if current_role=='other' and timestamp['date_label'] and re.search(
|
|
r'\b(?:ziad\w* pokrok|nefung\w*|problem\w*|nepodar\w*|rozpracovan\w*|pracoval\w*)\b',normalized(text)):
|
|
current_role='progress'
|
|
phase = evidence_phase(text, current_role, source_phase)
|
|
if current_role=='progress' and observed and not re.search(
|
|
r'\b(?:treba|bude|budu|planuj\w*|nie je|nebol\w*|zatial nie)\b',normalized(text)):
|
|
phase='result'
|
|
checked = bool(re.match(r'^\s*[-*+]\s+\[[xX]\]',text))
|
|
if checked:
|
|
phase='result'
|
|
if phase == 'result' and re.search(r'\b(?:rozroben\w*|rozpracovan\w*|nedokoncen\w*)\b', normalized(text)):
|
|
phase = 'progress'
|
|
kind = {'assignment': 'assignment', 'task': 'plan', 'progress': 'progress','topic':'topic'}.get(current_role, 'context')
|
|
label=normalized(text.split(':',1)[0]) if ':' in text else ''
|
|
if label.startswith('nazov ') or label in {'nazov','tema'}:
|
|
kind='topic'
|
|
if phase == 'result':
|
|
kind = 'result'
|
|
if reference_only(text):kind,phase='reference','unknown'
|
|
if lifecycle:kind,phase='lifecycle','unknown'
|
|
if re.search(r'\b(?:meno veduceho|veduci(?: prace)?|veduca(?: prace)?|skolitel)\b', normalized(text)):
|
|
kind, phase = 'supervisor', 'unknown'
|
|
if re.match(r'^(?:meno konzultanta|konzultant(?:ka)?\b)', normalized(plain)):
|
|
kind, phase = 'consultant', 'unknown'
|
|
units.append({'id': identifier(path, work_label, start, text), 'person': person,
|
|
'source_path': path, 'source_url': make_source_url(path), 'line': start+document.get('body_line_offset',0),
|
|
'work_label': work_label, 'work_type': scope[0] if scope else None,
|
|
'work_year': scope[1] if scope else None, 'text': text,
|
|
'kind': kind, 'phase': phase, 'completion_basis':'checked_task' if checked else 'source_statement',
|
|
**({'meeting_id':meeting_id} if include_records else {}), **timestamp})
|
|
if role=='topic':
|
|
role='other'
|
|
|
|
role_labels = {'assignment': {'zadanie', 'navrh na zadanie', 'ciel', 'ciele', 'ciele na semester'},
|
|
'topic': {'tema','zameranie','nazov','nazov prace','predbezny nazov','navrh na nazov'},
|
|
'progress': {'stav', 'priebezny stav', 'hotove', 'urobene', 'vysledky','vystupy'},
|
|
'task': {'ulohy', 'zasobnik uloh', 'treba dokoncit', 'plan'}}
|
|
for number, line in enumerate(document['content'].splitlines(), 1):
|
|
if re.match(r'^\s*(```|~~~)', line):
|
|
flush()
|
|
fence = None if fence else line.strip()[:3]
|
|
continue
|
|
if fence:
|
|
continue
|
|
heading = re.match(r'^\s{0,3}#{1,6}\s+(.+)', line)
|
|
heading_level=len(line.lstrip())-len(line.lstrip().lstrip('#'))
|
|
if heading and (new_scope := academic_scope(heading[1])):
|
|
flush()
|
|
scope, work_label = new_scope, clean_heading_text(heading[1])
|
|
work_level=heading_level
|
|
role, source_phase = 'other', 'unknown'
|
|
timestamp = {'date': None, 'date_label': None, 'date_precision': 'unknown'}
|
|
meeting_id = None
|
|
if include_records:units.append(record(line.strip(),number,'work_record'))
|
|
continue
|
|
if (new_date := meeting_date(line)) is not None:
|
|
flush()
|
|
timestamp = new_date
|
|
meeting_id = identifier(path,work_label,number,line.strip())
|
|
if include_records:units.append(record(line.strip(),number,'meeting'))
|
|
role, source_phase = 'other', 'unknown'
|
|
continue
|
|
label = normalized(line.strip().strip('*_# ').rstrip(':'))
|
|
if label in {'napr','napriklad','priklady','zdroje dokumentov','zdroje','suvisiace stranky','suvisiace prace'}:
|
|
flush()
|
|
role, source_phase = 'other', 'unknown'
|
|
continue
|
|
next_role = next((key for key, labels in role_labels.items()
|
|
if label in labels or key == 'assignment' and label.startswith('zadanie ')
|
|
or key == 'task' and re.match(r'^(?:ulohy (?:na|do|pre)|do dalsie\w* stretnutia)', label)), None)
|
|
if next_role:
|
|
flush()
|
|
role = next_role
|
|
source_phase = 'result' if label in {'hotove', 'urobene', 'vysledky','vystupy'} else 'unknown'
|
|
continue
|
|
if heading and work_level is not None and heading_level<=work_level:
|
|
flush()
|
|
scope,work_label,work_level=None,None,None
|
|
role,source_phase='other','unknown'
|
|
timestamp={'date':None,'date_label':None,'date_precision':'unknown'}
|
|
meeting_id=None
|
|
inline_label=normalized(line.split(':',1)[0]) if ':' in line else ''
|
|
if (LIFECYCLE.search(normalized(line)) or inline_label in {
|
|
'nazov','nazov prace','predbezny nazov','tema','veduci','veduci prace','meno veduceho',
|
|
'meno veduceho prace','skolitel','konzultant','konzultant prace','meno konzultanta','repozitar'}
|
|
or re.match(r'^\s*(?:konzultant(?:ka)?|vedúci|veduci)\s+',line,re.I)):
|
|
flush()
|
|
if not line.strip() or re.match(r'^\s*(?:[-*+]\s+|\d+[.)]\s+)', line):
|
|
flush()
|
|
if line.strip():
|
|
if not buffer:
|
|
start = number
|
|
buffer.append(line)
|
|
flush()
|
|
return units
|
|
|
|
|
|
def source_lifecycle_notes(wiki_root, source_digest):
|
|
"""Read explicit lifecycle qualifications only from the indexed snapshot."""
|
|
return _source_lifecycle_notes(str(wiki_root),source_digest)
|
|
|
|
|
|
from functools import lru_cache
|
|
|
|
@lru_cache(maxsize=2)
|
|
def _source_lifecycle_notes(wiki_root, source_digest):
|
|
import json
|
|
from pathlib import Path
|
|
from tools.sync.sync_indexes import source_files
|
|
from tools.rag.common import load_zpwiki_page
|
|
root=Path(wiki_root)
|
|
if not (root/'pages').is_dir():return []
|
|
files=source_files(root)
|
|
if hashlib.sha256(json.dumps(files,sort_keys=True).encode()).hexdigest()!=source_digest:
|
|
return [] # Never attach qualifications from a different source version.
|
|
notes=[]
|
|
for path in files:
|
|
if not PROFILE.fullmatch(path):continue
|
|
if not LIFECYCLE.search(normalized((root/path).read_text())):continue
|
|
document=load_zpwiki_page(root/path,wiki_root=root)
|
|
if document.get('published') is False:continue
|
|
notes.extend({**row,'evidence_role':'lifecycle'} for row in profile_units(document) if row['kind']=='lifecycle')
|
|
return notes
|