1904 lines
39 KiB
Python
1904 lines
39 KiB
Python
from __future__ import annotations
|
|
|
|
from tools.publication import effective_published
|
|
|
|
import re
|
|
import unicodedata
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
|
|
GRAPH_SCOPE = "zpwiki"
|
|
|
|
THESIS_TYPES = {
|
|
"bachelor_thesis",
|
|
"diploma_thesis",
|
|
"doctoral_thesis",
|
|
}
|
|
|
|
WORK_TYPE_LABELS = {
|
|
"bachelor_project": "Bakalársky projekt",
|
|
"bachelor_thesis": "Bakalárska práca",
|
|
"diploma_thesis": "Diplomová práca",
|
|
"diploma_project": "Diplomový projekt",
|
|
"team_project": "Tímový projekt",
|
|
"research_project": "Výskumný projekt",
|
|
"doctoral_thesis": "Dizertačná práca",
|
|
}
|
|
|
|
WORK_HEADING_PATTERNS = (
|
|
("bachelor_project", "bakalarsky projekt", None),
|
|
("bachelor_thesis", "bakalarska praca", "bp"),
|
|
("diploma_thesis", "diplomova praca", "dp"),
|
|
("diploma_project", "diplomovy projekt", None),
|
|
("team_project", "timovy projekt", "tp"),
|
|
("research_project", "vyskumny projekt", "vp"),
|
|
("research_project", "vedecky projekt", "vp"),
|
|
("doctoral_thesis", "dizertacna praca", "phd"),
|
|
)
|
|
|
|
TITLE_LABELS = {
|
|
"pisomna praca",
|
|
"nazov",
|
|
"nazov prace",
|
|
"nazov bakalarskej prace",
|
|
"nazov diplomovej prace",
|
|
"predbezny nazov",
|
|
"tema",
|
|
"navrh na temu",
|
|
"navrh na nazov",
|
|
"navrh na nazov bp",
|
|
"navrh na nazov dp",
|
|
"navrh na nazov bakalarskej prace",
|
|
"navrh na nazov bakalarkej prace",
|
|
"navrh na nazov diplomovej prace",
|
|
}
|
|
|
|
FALLBACK_TITLE_LABELS = {
|
|
"bachelor_thesis": {
|
|
"navrh na nazov bp",
|
|
"navrh na nazov bakalarskej prace",
|
|
"navrh na nazov bakalarkej prace",
|
|
"navrh na temu",
|
|
},
|
|
"diploma_thesis": {
|
|
"navrh na nazov dp",
|
|
"navrh na nazov diplomovej prace",
|
|
"navrh na temu",
|
|
},
|
|
}
|
|
|
|
TITLE_BLOCK_BOUNDARY_PREFIXES = (
|
|
"zadanie",
|
|
"navrh na zadanie",
|
|
"navrh na zadamie",
|
|
"ciel",
|
|
"ciele",
|
|
"ulohy",
|
|
"uloha",
|
|
"zasobnik",
|
|
"stretnutie",
|
|
"sttetnutie",
|
|
"virtualne stretnutie",
|
|
"stav",
|
|
"myslienky",
|
|
"napady",
|
|
"namety",
|
|
"stare zadanie",
|
|
"analyza ulohy",
|
|
)
|
|
|
|
NON_TITLE_PREFIXES = (
|
|
"odporucany cas konzult",
|
|
"odporucany termin konzult",
|
|
"cas konzult",
|
|
"termin konzult",
|
|
"pozriet si",
|
|
"prejst si",
|
|
"konzultacie",
|
|
"projekt odlozeny",
|
|
"chceme vytvorit",
|
|
"dokument",
|
|
"sekcia",
|
|
"rok zaciatku studia",
|
|
"meno veduceho",
|
|
"veduci",
|
|
"konzultant",
|
|
"spolupraca",
|
|
"repozitar",
|
|
"git repozitar",
|
|
"crzp",
|
|
"literatura",
|
|
"suvisiace materialy",
|
|
"praca suvisi",
|
|
"do buducnosti",
|
|
"do dalsieho",
|
|
"do buduceho",
|
|
"poznamky",
|
|
"kriteria",
|
|
"projektove stranky",
|
|
"pisomna praca",
|
|
"vystupy",
|
|
"hotove",
|
|
"ukoncil predcasne",
|
|
"praca uspesne obhajena",
|
|
"projekt zakonceny",
|
|
"dp sa preklada",
|
|
"na maise",
|
|
"je potrebne",
|
|
"todo",
|
|
"to-do",
|
|
"v roku",
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorkHeading:
|
|
work_type: str
|
|
heading: str
|
|
category_prefix: str | None
|
|
|
|
|
|
def normalize_text(value: str) -> str:
|
|
decomposed = unicodedata.normalize(
|
|
"NFKD",
|
|
str(value),
|
|
)
|
|
|
|
without_marks = "".join(
|
|
char
|
|
for char in decomposed
|
|
if not unicodedata.combining(char)
|
|
)
|
|
|
|
return re.sub(
|
|
r"\s+",
|
|
" ",
|
|
without_marks.lower(),
|
|
).strip()
|
|
|
|
|
|
def normalize_key(value: str) -> str:
|
|
normalized = normalize_text(value)
|
|
|
|
normalized = re.sub(
|
|
r"[^a-z0-9]+",
|
|
"-",
|
|
normalized,
|
|
)
|
|
|
|
return normalized.strip("-")
|
|
|
|
|
|
def source_url_from_path(path: str) -> str:
|
|
normalized = path.replace("\\", "/")
|
|
|
|
if normalized.startswith("pages/"):
|
|
normalized = normalized[len("pages/"):]
|
|
|
|
if normalized.endswith("/README.md"):
|
|
normalized = normalized[:-len("/README.md")]
|
|
elif normalized.endswith(".md"):
|
|
normalized = normalized[:-3]
|
|
|
|
return (
|
|
"https://zp.kemt.fei.tuke.sk/"
|
|
+ normalized.strip("/")
|
|
)
|
|
|
|
|
|
def document_kind(path: str) -> str:
|
|
normalized = path.replace("\\", "/")
|
|
|
|
if normalized.startswith("pages/students/"):
|
|
return "student"
|
|
|
|
if normalized.startswith("pages/interns/"):
|
|
return "intern"
|
|
|
|
if normalized.startswith("pages/topics/"):
|
|
return "topic"
|
|
|
|
return "other"
|
|
|
|
|
|
def person_profile_path(
|
|
path: str,
|
|
) -> str | None:
|
|
normalized = path.replace("\\", "/")
|
|
parts = normalized.split("/")
|
|
|
|
if (
|
|
len(parts) >= 4
|
|
and parts[0] == "pages"
|
|
and parts[1] == "students"
|
|
and re.fullmatch(
|
|
r"20\d{2}",
|
|
parts[2],
|
|
)
|
|
and parts[3]
|
|
):
|
|
return (
|
|
"pages/students/"
|
|
+ parts[2]
|
|
+ "/"
|
|
+ parts[3]
|
|
+ "/README.md"
|
|
)
|
|
|
|
if (
|
|
len(parts) >= 3
|
|
and parts[0] == "pages"
|
|
and parts[1] == "interns"
|
|
and parts[2]
|
|
):
|
|
return (
|
|
"pages/interns/"
|
|
+ parts[2]
|
|
+ "/README.md"
|
|
)
|
|
|
|
return None
|
|
|
|
|
|
def person_kind_from_profile_path(
|
|
path: str,
|
|
) -> str | None:
|
|
normalized = path.replace("\\", "/")
|
|
profile_path = person_profile_path(
|
|
normalized
|
|
)
|
|
|
|
if profile_path != normalized:
|
|
return None
|
|
|
|
if normalized.startswith(
|
|
"pages/students/"
|
|
):
|
|
return "student"
|
|
|
|
if normalized.startswith(
|
|
"pages/interns/"
|
|
):
|
|
return "intern"
|
|
|
|
return None
|
|
|
|
|
|
def extract_full_years(value: str) -> list[int]:
|
|
years = [
|
|
int(match)
|
|
for match in re.findall(
|
|
r"\b(20\d{2})\b",
|
|
value,
|
|
)
|
|
]
|
|
|
|
for match in re.finditer(
|
|
r"\b(20\d{2})\s*/\s*(\d{2})\b",
|
|
value,
|
|
):
|
|
first = int(match.group(1))
|
|
second_short = int(match.group(2))
|
|
|
|
second = (
|
|
(first // 100) * 100
|
|
+ second_short
|
|
)
|
|
|
|
if second not in years:
|
|
years.append(second)
|
|
|
|
return sorted(set(years))
|
|
|
|
|
|
def parse_category(
|
|
category: str,
|
|
) -> tuple[str, int | None]:
|
|
match = re.fullmatch(
|
|
r"(bp|dp|tp|vp|phd)(20\d{2})",
|
|
category.lower(),
|
|
)
|
|
|
|
if not match:
|
|
return "other", None
|
|
|
|
prefix = match.group(1)
|
|
year = int(match.group(2))
|
|
|
|
category_types = {
|
|
"bp": "bachelor_thesis",
|
|
"dp": "diploma_thesis",
|
|
"tp": "team_project",
|
|
"vp": "research_project",
|
|
"phd": "doctoral_thesis",
|
|
}
|
|
|
|
return category_types[prefix], year
|
|
|
|
|
|
def find_work_heading(
|
|
heading: str,
|
|
) -> WorkHeading | None:
|
|
normalized = normalize_text(heading)
|
|
|
|
# Preparation/planning notes are not an academic work record. In
|
|
# particular, a year in such a heading must not become the work's year.
|
|
if normalized.startswith(('priprava na ', 'priprava k ')):
|
|
return None
|
|
|
|
for (
|
|
work_type,
|
|
phrase,
|
|
category_prefix,
|
|
) in WORK_HEADING_PATTERNS:
|
|
if phrase in normalized:
|
|
return WorkHeading(
|
|
work_type=work_type,
|
|
heading=heading,
|
|
category_prefix=category_prefix,
|
|
)
|
|
|
|
return None
|
|
|
|
|
|
def category_years(
|
|
categories: list[str],
|
|
prefix: str | None,
|
|
) -> list[int]:
|
|
if prefix is None:
|
|
return []
|
|
|
|
pattern = re.compile(
|
|
rf"^{re.escape(prefix)}(20\d{{2}})$",
|
|
flags=re.IGNORECASE,
|
|
)
|
|
|
|
result = []
|
|
|
|
for category in categories:
|
|
match = pattern.fullmatch(category)
|
|
|
|
if match:
|
|
result.append(
|
|
int(match.group(1))
|
|
)
|
|
|
|
return sorted(set(result))
|
|
|
|
|
|
def resolve_work_year(
|
|
heading: WorkHeading,
|
|
categories: list[str],
|
|
) -> int | None:
|
|
heading_years = extract_full_years(
|
|
heading.heading
|
|
)
|
|
|
|
category_candidates = category_years(
|
|
categories,
|
|
heading.category_prefix,
|
|
)
|
|
|
|
if heading.work_type == "diploma_project":
|
|
if heading_years:
|
|
return heading_years[-1]
|
|
|
|
return None
|
|
|
|
common = sorted(
|
|
set(heading_years)
|
|
& set(category_candidates)
|
|
)
|
|
|
|
if common:
|
|
return common[-1]
|
|
|
|
if len(heading_years) > 1:
|
|
return heading_years[-1]
|
|
|
|
if heading_years:
|
|
return heading_years[-1]
|
|
|
|
if category_candidates:
|
|
return category_candidates[-1]
|
|
|
|
return None
|
|
|
|
|
|
def clean_markdown_line(
|
|
line: str,
|
|
) -> str:
|
|
value = line.strip()
|
|
|
|
value = value.replace(
|
|
"*",
|
|
"",
|
|
).replace(
|
|
"`",
|
|
"",
|
|
)
|
|
|
|
return value.strip()
|
|
|
|
|
|
def strip_list_prefix(
|
|
line: str,
|
|
) -> str:
|
|
value = clean_markdown_line(
|
|
line
|
|
)
|
|
|
|
value = re.sub(
|
|
r"^[-+•]\s+(?:\[[ xX-]\]\s*)?",
|
|
"",
|
|
value,
|
|
)
|
|
|
|
return value.strip()
|
|
|
|
|
|
def is_numbered_or_bulleted_line(
|
|
line: str,
|
|
) -> bool:
|
|
value = clean_markdown_line(
|
|
line
|
|
)
|
|
|
|
if re.match(
|
|
r"^\d+[\.\)]\s*",
|
|
value,
|
|
):
|
|
return True
|
|
|
|
return bool(
|
|
re.match(
|
|
r"^[-+•]\s+",
|
|
value,
|
|
)
|
|
)
|
|
|
|
|
|
def is_title_candidate(
|
|
line: str,
|
|
) -> bool:
|
|
value = strip_list_prefix(
|
|
line
|
|
)
|
|
|
|
if not value:
|
|
return False
|
|
|
|
normalized = normalize_text(
|
|
value
|
|
)
|
|
|
|
if normalized.startswith(
|
|
NON_TITLE_PREFIXES
|
|
):
|
|
return False
|
|
|
|
if value.startswith(
|
|
(
|
|
"http://",
|
|
"https://",
|
|
"[",
|
|
"./",
|
|
"../",
|
|
)
|
|
):
|
|
return False
|
|
|
|
if (
|
|
"http://" in value
|
|
or "https://" in value
|
|
):
|
|
return False
|
|
|
|
if re.match(
|
|
r"^\d+[\.\)]\s*",
|
|
value,
|
|
):
|
|
return False
|
|
|
|
if value.endswith(
|
|
":"
|
|
):
|
|
return False
|
|
|
|
if len(value) < 6:
|
|
return False
|
|
|
|
if len(value) > 180:
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def immediate_title_after_label(
|
|
lines: list[str],
|
|
start_index: int,
|
|
) -> str | None:
|
|
for raw_line in lines[start_index:]:
|
|
if not raw_line.strip():
|
|
continue
|
|
|
|
if is_numbered_or_bulleted_line(
|
|
raw_line
|
|
):
|
|
return None
|
|
|
|
candidate = strip_list_prefix(
|
|
raw_line
|
|
)
|
|
|
|
if not is_title_candidate(
|
|
candidate
|
|
):
|
|
return None
|
|
|
|
return candidate.rstrip(
|
|
" ,"
|
|
)
|
|
|
|
return None
|
|
|
|
|
|
def extract_labeled_title(
|
|
texts: list[str],
|
|
labels: set[str],
|
|
) -> str | None:
|
|
for text in texts:
|
|
lines = text.splitlines()
|
|
|
|
for index, raw_line in enumerate(
|
|
lines
|
|
):
|
|
line = strip_list_prefix(
|
|
raw_line
|
|
)
|
|
|
|
if not line:
|
|
continue
|
|
|
|
before, separator, after = (
|
|
line.partition(
|
|
":"
|
|
)
|
|
)
|
|
|
|
if not separator:
|
|
continue
|
|
|
|
normalized_before = (
|
|
normalize_text(
|
|
before
|
|
)
|
|
)
|
|
|
|
if normalized_before not in labels:
|
|
continue
|
|
|
|
title = re.sub(r'\[([^\]]+)\]\([^)]*\)', r'\1', after).strip()
|
|
|
|
if title:
|
|
if is_title_candidate(
|
|
title
|
|
):
|
|
return title.rstrip(
|
|
" ,"
|
|
)
|
|
|
|
continue
|
|
|
|
candidate = (
|
|
immediate_title_after_label(
|
|
lines,
|
|
index + 1,
|
|
)
|
|
)
|
|
|
|
if candidate:
|
|
return candidate
|
|
|
|
return None
|
|
|
|
|
|
def extract_leading_title(
|
|
texts: list[str],
|
|
) -> str | None:
|
|
for text in texts:
|
|
candidates: list[str] = []
|
|
saw_content = False
|
|
|
|
for raw_line in text.splitlines():
|
|
if not raw_line.strip():
|
|
continue
|
|
|
|
line = clean_markdown_line(
|
|
raw_line
|
|
)
|
|
stripped = strip_list_prefix(
|
|
raw_line
|
|
)
|
|
normalized = normalize_text(
|
|
stripped
|
|
)
|
|
|
|
if normalized.startswith(
|
|
(
|
|
"dokument:",
|
|
"sekcia:",
|
|
"rok zaciatku studia",
|
|
)
|
|
):
|
|
continue
|
|
|
|
if (
|
|
stripped.startswith(
|
|
"["
|
|
)
|
|
and "](" in stripped
|
|
):
|
|
continue
|
|
|
|
if normalized.startswith(
|
|
(
|
|
"spolupraca",
|
|
"suvisiaca praca",
|
|
"suvisiaca dizertacna praca",
|
|
)
|
|
):
|
|
continue
|
|
|
|
saw_content = True
|
|
|
|
if is_numbered_or_bulleted_line(
|
|
raw_line
|
|
):
|
|
if candidates:
|
|
return candidates[-1]
|
|
|
|
return None
|
|
|
|
if normalized.startswith(
|
|
TITLE_BLOCK_BOUNDARY_PREFIXES
|
|
):
|
|
if candidates:
|
|
return candidates[-1]
|
|
|
|
return None
|
|
|
|
if not is_title_candidate(
|
|
line
|
|
):
|
|
continue
|
|
|
|
candidates.append(
|
|
stripped.rstrip(
|
|
" ,"
|
|
)
|
|
)
|
|
|
|
if candidates:
|
|
return candidates[-1]
|
|
|
|
if saw_content:
|
|
return None
|
|
|
|
return None
|
|
|
|
|
|
def extract_work_title(
|
|
texts: list[str],
|
|
) -> str | None:
|
|
title = extract_labeled_title(
|
|
texts,
|
|
TITLE_LABELS - {"tema", "navrh na temu", "pisomna praca"},
|
|
)
|
|
|
|
if title:
|
|
return title
|
|
|
|
title = extract_labeled_title(texts, {"tema", "navrh na temu", "pisomna praca"})
|
|
if title:
|
|
return title
|
|
|
|
return extract_leading_title(
|
|
texts
|
|
)
|
|
|
|
|
|
def extract_explicit_proposal_title(
|
|
work_type: str,
|
|
chunks: list[dict[str, Any]],
|
|
) -> tuple[str | None, list[str]]:
|
|
labels = FALLBACK_TITLE_LABELS.get(
|
|
work_type
|
|
)
|
|
|
|
if not labels:
|
|
return None, []
|
|
|
|
candidates: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
for chunk in sorted(
|
|
chunks,
|
|
key=lambda item: int(
|
|
item.get(
|
|
"chunk_index",
|
|
0,
|
|
)
|
|
),
|
|
):
|
|
text = str(
|
|
chunk.get(
|
|
"text",
|
|
"",
|
|
)
|
|
)
|
|
|
|
if not text:
|
|
continue
|
|
|
|
lines = text.splitlines()
|
|
|
|
for index, raw_line in enumerate(
|
|
lines
|
|
):
|
|
line = strip_list_prefix(
|
|
raw_line
|
|
)
|
|
|
|
if not line:
|
|
continue
|
|
|
|
before, separator, after = (
|
|
line.partition(
|
|
":"
|
|
)
|
|
)
|
|
|
|
if not separator:
|
|
continue
|
|
|
|
normalized_before = (
|
|
normalize_text(
|
|
before
|
|
)
|
|
)
|
|
|
|
if normalized_before not in labels:
|
|
continue
|
|
|
|
title = after.strip()
|
|
|
|
if title:
|
|
if not is_title_candidate(
|
|
title
|
|
):
|
|
continue
|
|
|
|
title = title.rstrip(
|
|
" ,"
|
|
)
|
|
else:
|
|
title = (
|
|
immediate_title_after_label(
|
|
lines,
|
|
index + 1,
|
|
)
|
|
)
|
|
|
|
if not title:
|
|
continue
|
|
|
|
normalized_title = (
|
|
normalize_text(
|
|
title
|
|
)
|
|
)
|
|
|
|
candidate = candidates.setdefault(
|
|
normalized_title,
|
|
{
|
|
"title": title,
|
|
"chunk_ids": set(),
|
|
},
|
|
)
|
|
|
|
chunk_id = chunk.get(
|
|
"chunk_id"
|
|
)
|
|
|
|
if chunk_id:
|
|
candidate[
|
|
"chunk_ids"
|
|
].add(
|
|
str(chunk_id)
|
|
)
|
|
|
|
if len(candidates) != 1:
|
|
return None, []
|
|
|
|
candidate = next(
|
|
iter(candidates.values())
|
|
)
|
|
|
|
return (
|
|
str(candidate["title"]),
|
|
sorted(
|
|
candidate["chunk_ids"]
|
|
),
|
|
)
|
|
|
|
|
|
ROOT_PROFILE_TITLE_LABELS = {
|
|
"nazov",
|
|
"nazov prace",
|
|
"nazov bakalarskej prace",
|
|
"nazov diplomovej prace",
|
|
"predbezny nazov",
|
|
}
|
|
|
|
|
|
def extract_profile_root_title(
|
|
chunks: list[dict[str, Any]],
|
|
) -> tuple[str | None, list[str]]:
|
|
candidates: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
for chunk in sorted(
|
|
chunks,
|
|
key=lambda item: int(
|
|
item.get(
|
|
"chunk_index",
|
|
0,
|
|
)
|
|
),
|
|
):
|
|
text = str(
|
|
chunk.get(
|
|
"text",
|
|
"",
|
|
)
|
|
)
|
|
|
|
if not text:
|
|
continue
|
|
|
|
has_section_marker = any(
|
|
normalize_text(
|
|
strip_list_prefix(
|
|
line
|
|
)
|
|
).startswith(
|
|
"sekcia:"
|
|
)
|
|
for line in text.splitlines()
|
|
)
|
|
|
|
if has_section_marker:
|
|
continue
|
|
|
|
title = extract_labeled_title(
|
|
[text],
|
|
ROOT_PROFILE_TITLE_LABELS,
|
|
)
|
|
|
|
if not title:
|
|
continue
|
|
|
|
normalized_title = normalize_text(
|
|
title
|
|
)
|
|
|
|
candidate = candidates.setdefault(
|
|
normalized_title,
|
|
{
|
|
"title": title,
|
|
"chunk_ids": set(),
|
|
},
|
|
)
|
|
|
|
chunk_id = chunk.get(
|
|
"chunk_id"
|
|
)
|
|
|
|
if chunk_id:
|
|
candidate[
|
|
"chunk_ids"
|
|
].add(
|
|
str(chunk_id)
|
|
)
|
|
|
|
if len(candidates) != 1:
|
|
return None, []
|
|
|
|
candidate = next(
|
|
iter(
|
|
candidates.values()
|
|
)
|
|
)
|
|
|
|
return (
|
|
str(
|
|
candidate["title"]
|
|
),
|
|
sorted(
|
|
candidate[
|
|
"chunk_ids"
|
|
]
|
|
),
|
|
)
|
|
|
|
|
|
def latest_thesis_category(
|
|
categories: list[str],
|
|
) -> tuple[str, int] | None:
|
|
candidates: list[
|
|
tuple[str, int]
|
|
] = []
|
|
|
|
for category in categories:
|
|
work_type, year = (
|
|
parse_category(
|
|
category
|
|
)
|
|
)
|
|
|
|
if (
|
|
work_type
|
|
not in THESIS_TYPES
|
|
or year is None
|
|
):
|
|
continue
|
|
|
|
candidates.append(
|
|
(
|
|
work_type,
|
|
year,
|
|
)
|
|
)
|
|
|
|
if not candidates:
|
|
return None
|
|
|
|
latest_year = max(
|
|
year
|
|
for _, year
|
|
in candidates
|
|
)
|
|
|
|
latest = sorted(
|
|
{
|
|
(
|
|
work_type,
|
|
year,
|
|
)
|
|
for work_type, year
|
|
in candidates
|
|
if year == latest_year
|
|
}
|
|
)
|
|
|
|
if len(latest) != 1:
|
|
return None
|
|
|
|
return latest[0]
|
|
|
|
|
|
def extract_start_year(
|
|
path: str,
|
|
chunks: list[dict[str, Any]],
|
|
) -> int | None:
|
|
pattern = re.compile(
|
|
r"rok\s+zaciatku\s+studia"
|
|
r"\s*\*?\s*:\s*\*?\s*"
|
|
r"(20\d{2})"
|
|
)
|
|
|
|
for chunk in chunks:
|
|
normalized = normalize_text(
|
|
str(chunk.get("text", ""))
|
|
)
|
|
|
|
match = pattern.search(
|
|
normalized
|
|
)
|
|
|
|
if match:
|
|
return int(
|
|
match.group(1)
|
|
)
|
|
|
|
match = re.search(
|
|
r"pages/students/(20\d{2})/",
|
|
path.replace("\\", "/"),
|
|
)
|
|
|
|
if match:
|
|
return int(
|
|
match.group(1)
|
|
)
|
|
|
|
return None
|
|
|
|
|
|
def topic_key_from_document_path(
|
|
path: str,
|
|
) -> str | None:
|
|
normalized = path.replace("\\", "/")
|
|
|
|
if not normalized.startswith(
|
|
"pages/topics/"
|
|
):
|
|
return None
|
|
|
|
value = normalized[
|
|
len("pages/topics/"):
|
|
]
|
|
|
|
if value.endswith("/README.md"):
|
|
value = value[:-len("/README.md")]
|
|
elif value.endswith(".md"):
|
|
value = value[:-3]
|
|
|
|
if not value:
|
|
return None
|
|
|
|
return normalize_key(
|
|
value.split("/")[0]
|
|
)
|
|
|
|
|
|
def is_topic_root_document(
|
|
path: str,
|
|
) -> bool:
|
|
normalized = path.replace("\\", "/")
|
|
|
|
return bool(
|
|
re.fullmatch(
|
|
r"pages/topics/[^/]+/README\.md",
|
|
normalized,
|
|
)
|
|
)
|
|
|
|
|
|
def keyword_display_name(
|
|
value: str,
|
|
) -> str:
|
|
aliases = {
|
|
"nlp": "NLP",
|
|
"rag": "RAG",
|
|
"ner": "NER",
|
|
"qa": "QA",
|
|
"ie": "IE",
|
|
"lm": "LM",
|
|
"llm": "LLM",
|
|
"nmt": "NMT",
|
|
"pos": "POS",
|
|
"gpu": "GPU",
|
|
"spacy": "Spacy",
|
|
"db": "DB",
|
|
}
|
|
|
|
key = normalize_key(value)
|
|
|
|
if key in aliases:
|
|
return aliases[key]
|
|
|
|
return str(value).strip()
|
|
|
|
|
|
def extract_works(
|
|
document: dict[str, Any],
|
|
chunks: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
grouped: dict[
|
|
tuple[str, str],
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
categories = list(
|
|
document.get(
|
|
"categories",
|
|
[],
|
|
)
|
|
)
|
|
|
|
for chunk in sorted(
|
|
chunks,
|
|
key=lambda item: int(
|
|
item.get(
|
|
"chunk_index",
|
|
0,
|
|
)
|
|
),
|
|
):
|
|
heading_paths = chunk.get(
|
|
"heading_paths",
|
|
[],
|
|
)
|
|
|
|
chunk_headings: dict[
|
|
tuple[str, str],
|
|
WorkHeading,
|
|
] = {}
|
|
|
|
for path in heading_paths:
|
|
values = (
|
|
path
|
|
if isinstance(
|
|
path,
|
|
list,
|
|
)
|
|
else [path]
|
|
)
|
|
|
|
heading = None
|
|
|
|
for raw_heading in reversed(
|
|
values
|
|
):
|
|
if not raw_heading:
|
|
continue
|
|
|
|
heading = find_work_heading(
|
|
str(raw_heading)
|
|
)
|
|
|
|
if heading is not None:
|
|
break
|
|
|
|
if heading is None:
|
|
continue
|
|
|
|
key = (
|
|
heading.work_type,
|
|
normalize_text(
|
|
heading.heading
|
|
),
|
|
)
|
|
|
|
chunk_headings[key] = heading
|
|
|
|
for key, heading in (
|
|
chunk_headings.items()
|
|
):
|
|
entry = grouped.setdefault(
|
|
key,
|
|
{
|
|
"heading": heading,
|
|
"texts": [],
|
|
"chunk_ids": [],
|
|
},
|
|
)
|
|
|
|
text = str(
|
|
chunk.get(
|
|
"text",
|
|
"",
|
|
)
|
|
)
|
|
|
|
if text:
|
|
entry["texts"].append(
|
|
text
|
|
)
|
|
|
|
chunk_id = chunk.get(
|
|
"chunk_id"
|
|
)
|
|
|
|
if chunk_id:
|
|
entry["chunk_ids"].append(
|
|
str(chunk_id)
|
|
)
|
|
|
|
work_type_counts: dict[
|
|
str,
|
|
int,
|
|
] = defaultdict(int)
|
|
|
|
for entry in grouped.values():
|
|
work_type_counts[
|
|
entry["heading"].work_type
|
|
] += 1
|
|
|
|
result = []
|
|
|
|
for entry in grouped.values():
|
|
heading = entry["heading"]
|
|
|
|
year = resolve_work_year(
|
|
heading,
|
|
categories,
|
|
)
|
|
|
|
title = extract_work_title(
|
|
entry["texts"]
|
|
)
|
|
|
|
source_chunk_ids = set(
|
|
entry["chunk_ids"]
|
|
)
|
|
|
|
if (
|
|
title is None
|
|
and work_type_counts[
|
|
heading.work_type
|
|
] == 1
|
|
):
|
|
(
|
|
fallback_title,
|
|
fallback_chunk_ids,
|
|
) = extract_explicit_proposal_title(
|
|
heading.work_type,
|
|
chunks,
|
|
)
|
|
|
|
if fallback_title:
|
|
title = fallback_title
|
|
source_chunk_ids.update(
|
|
fallback_chunk_ids
|
|
)
|
|
|
|
result.append(
|
|
{
|
|
"work_type": heading.work_type,
|
|
"work_type_label": (
|
|
WORK_TYPE_LABELS[
|
|
heading.work_type
|
|
]
|
|
),
|
|
"heading": heading.heading,
|
|
"year": year,
|
|
"title": title,
|
|
"source_chunk_ids": sorted(
|
|
source_chunk_ids
|
|
),
|
|
}
|
|
)
|
|
|
|
(
|
|
root_title,
|
|
root_title_chunk_ids,
|
|
) = extract_profile_root_title(
|
|
chunks
|
|
)
|
|
|
|
thesis_category = (
|
|
latest_thesis_category(
|
|
categories
|
|
)
|
|
)
|
|
|
|
if (
|
|
root_title
|
|
and thesis_category
|
|
):
|
|
(
|
|
thesis_type,
|
|
thesis_year,
|
|
) = thesis_category
|
|
|
|
matching_theses = [
|
|
item
|
|
for item in result
|
|
if (
|
|
item[
|
|
"work_type"
|
|
]
|
|
== thesis_type
|
|
and item[
|
|
"year"
|
|
]
|
|
== thesis_year
|
|
)
|
|
]
|
|
|
|
if len(
|
|
matching_theses
|
|
) == 1:
|
|
existing = (
|
|
matching_theses[0]
|
|
)
|
|
|
|
if existing[
|
|
"title"
|
|
] is None:
|
|
existing[
|
|
"title"
|
|
] = root_title
|
|
|
|
existing[
|
|
"source_chunk_ids"
|
|
] = sorted(
|
|
set(
|
|
existing[
|
|
"source_chunk_ids"
|
|
]
|
|
)
|
|
| set(
|
|
root_title_chunk_ids
|
|
)
|
|
)
|
|
|
|
elif not matching_theses:
|
|
result.append(
|
|
{
|
|
"work_type": (
|
|
thesis_type
|
|
),
|
|
"work_type_label": (
|
|
WORK_TYPE_LABELS[
|
|
thesis_type
|
|
]
|
|
),
|
|
"heading": (
|
|
WORK_TYPE_LABELS[
|
|
thesis_type
|
|
]
|
|
+ " "
|
|
+ str(
|
|
thesis_year
|
|
)
|
|
),
|
|
"year": thesis_year,
|
|
"title": root_title,
|
|
"source_chunk_ids": (
|
|
root_title_chunk_ids
|
|
),
|
|
}
|
|
)
|
|
|
|
return sorted(
|
|
result,
|
|
key=lambda item: (
|
|
item["year"] or 0,
|
|
item["work_type"],
|
|
item["heading"],
|
|
),
|
|
)
|
|
|
|
|
|
def build_graph_payload(
|
|
documents: list[dict[str, Any]],
|
|
chunks: list[dict[str, Any]],
|
|
) -> dict[str, list[dict[str, Any]]]:
|
|
chunks_by_document: dict[
|
|
str,
|
|
list[dict[str, Any]],
|
|
] = defaultdict(list)
|
|
|
|
for chunk in chunks:
|
|
path = str(
|
|
chunk.get(
|
|
"document_path",
|
|
"",
|
|
)
|
|
)
|
|
|
|
if path:
|
|
chunks_by_document[
|
|
path
|
|
].append(chunk)
|
|
|
|
documents_by_path = {
|
|
str(document["path"]): document
|
|
for document in documents
|
|
}
|
|
|
|
people: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
document_authors: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
document_nodes: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
categories: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
topics: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
keywords: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
theses: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
works: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
person_documents = set()
|
|
document_author_documents = set()
|
|
document_categories = set()
|
|
document_keywords = set()
|
|
document_describes_topics = set()
|
|
person_theses = set()
|
|
person_works = set()
|
|
thesis_documents = set()
|
|
work_documents = set()
|
|
|
|
for document in documents:
|
|
path = str(
|
|
document["path"]
|
|
)
|
|
|
|
title = str(
|
|
document.get(
|
|
"title",
|
|
path,
|
|
)
|
|
)
|
|
|
|
kind = document_kind(
|
|
path
|
|
)
|
|
|
|
doc_categories = [
|
|
str(value)
|
|
for value in document.get(
|
|
"categories",
|
|
[],
|
|
)
|
|
]
|
|
|
|
doc_tags = [
|
|
str(value)
|
|
for value in document.get(
|
|
"tags",
|
|
[],
|
|
)
|
|
]
|
|
|
|
author = document.get(
|
|
"author"
|
|
)
|
|
|
|
document_nodes[path] = {
|
|
"path": path,
|
|
"name": title,
|
|
"title": title,
|
|
"source_url": (
|
|
source_url_from_path(
|
|
path
|
|
)
|
|
),
|
|
"document_kind": kind,
|
|
"published": effective_published(document.get("published")),
|
|
"categories": (
|
|
doc_categories
|
|
),
|
|
"tags": doc_tags,
|
|
"author": (
|
|
str(author)
|
|
if author
|
|
else None
|
|
),
|
|
"graph_scope": (
|
|
GRAPH_SCOPE
|
|
),
|
|
}
|
|
|
|
if author:
|
|
author_name = str(
|
|
author
|
|
)
|
|
|
|
author_id = normalize_key(
|
|
author_name
|
|
)
|
|
|
|
if author_id not in document_authors:
|
|
document_authors[
|
|
author_id
|
|
] = {
|
|
"id": author_id,
|
|
"name": author_name,
|
|
"graph_scope": (
|
|
GRAPH_SCOPE
|
|
),
|
|
}
|
|
|
|
document_author_documents.add(
|
|
(
|
|
author_id,
|
|
path,
|
|
)
|
|
)
|
|
|
|
for category in doc_categories:
|
|
category_type, year = (
|
|
parse_category(
|
|
category
|
|
)
|
|
)
|
|
|
|
categories[category] = {
|
|
"name": category,
|
|
"category_type": (
|
|
category_type
|
|
),
|
|
"year": year,
|
|
"graph_scope": (
|
|
GRAPH_SCOPE
|
|
),
|
|
}
|
|
|
|
document_categories.add(
|
|
(
|
|
path,
|
|
category,
|
|
)
|
|
)
|
|
|
|
for tag in doc_tags:
|
|
keyword_id = normalize_key(
|
|
tag
|
|
)
|
|
|
|
if not keyword_id:
|
|
continue
|
|
|
|
if keyword_id not in keywords:
|
|
keywords[keyword_id] = {
|
|
"id": keyword_id,
|
|
"name": (
|
|
keyword_display_name(
|
|
tag
|
|
)
|
|
),
|
|
"graph_scope": (
|
|
GRAPH_SCOPE
|
|
),
|
|
}
|
|
|
|
document_keywords.add(
|
|
(
|
|
path,
|
|
keyword_id,
|
|
)
|
|
)
|
|
|
|
described_topic = (
|
|
topic_key_from_document_path(
|
|
path
|
|
)
|
|
)
|
|
|
|
if described_topic:
|
|
if (
|
|
described_topic not in topics
|
|
or is_topic_root_document(
|
|
path
|
|
)
|
|
):
|
|
topics[
|
|
described_topic
|
|
] = {
|
|
"id": described_topic,
|
|
"name": title,
|
|
"source_document": path,
|
|
"graph_scope": (
|
|
GRAPH_SCOPE
|
|
),
|
|
}
|
|
|
|
document_describes_topics.add(
|
|
(
|
|
path,
|
|
described_topic,
|
|
)
|
|
)
|
|
|
|
profile_path = person_profile_path(
|
|
path
|
|
)
|
|
|
|
if profile_path is None:
|
|
continue
|
|
|
|
profile_document = documents_by_path.get(
|
|
profile_path
|
|
)
|
|
|
|
if profile_document is None:
|
|
continue
|
|
|
|
person_kind = (
|
|
person_kind_from_profile_path(
|
|
profile_path
|
|
)
|
|
)
|
|
|
|
if person_kind is None:
|
|
continue
|
|
|
|
person_id = (
|
|
f"person:{profile_path}"
|
|
)
|
|
|
|
profile_title = str(
|
|
profile_document.get(
|
|
"title",
|
|
profile_path,
|
|
)
|
|
)
|
|
|
|
profile_chunks = (
|
|
chunks_by_document.get(
|
|
profile_path,
|
|
[],
|
|
)
|
|
)
|
|
|
|
if person_id not in people:
|
|
people[person_id] = {
|
|
"id": person_id,
|
|
"name": profile_title,
|
|
"person_kind": person_kind,
|
|
"start_year": (
|
|
extract_start_year(
|
|
profile_path,
|
|
profile_chunks,
|
|
)
|
|
if person_kind
|
|
== "student"
|
|
else None
|
|
),
|
|
"source_document": (
|
|
profile_path
|
|
),
|
|
"graph_scope": (
|
|
GRAPH_SCOPE
|
|
),
|
|
}
|
|
|
|
person_documents.add(
|
|
(
|
|
person_id,
|
|
path,
|
|
)
|
|
)
|
|
|
|
if (
|
|
person_kind != "student"
|
|
or path != profile_path
|
|
):
|
|
continue
|
|
|
|
document_chunks = (
|
|
chunks_by_document.get(
|
|
profile_path,
|
|
[],
|
|
)
|
|
)
|
|
|
|
for work in extract_works(
|
|
document,
|
|
document_chunks,
|
|
):
|
|
common_data = {
|
|
"name": (
|
|
work["title"]
|
|
or work["heading"]
|
|
),
|
|
"title": work[
|
|
"title"
|
|
],
|
|
"heading": work[
|
|
"heading"
|
|
],
|
|
"year": work[
|
|
"year"
|
|
],
|
|
"source_document": path,
|
|
"source_chunk_ids": work[
|
|
"source_chunk_ids"
|
|
],
|
|
"graph_scope": (
|
|
GRAPH_SCOPE
|
|
),
|
|
}
|
|
|
|
if work["work_type"] in THESIS_TYPES:
|
|
thesis_id = (
|
|
"thesis:"
|
|
+ path
|
|
+ ":"
|
|
+ normalize_key(
|
|
work["heading"]
|
|
)
|
|
)
|
|
|
|
theses[thesis_id] = {
|
|
"id": thesis_id,
|
|
"thesis_type": work[
|
|
"work_type"
|
|
],
|
|
"thesis_type_label": (
|
|
work[
|
|
"work_type_label"
|
|
]
|
|
),
|
|
**common_data,
|
|
}
|
|
|
|
person_theses.add(
|
|
(
|
|
person_id,
|
|
thesis_id,
|
|
)
|
|
)
|
|
|
|
thesis_documents.add(
|
|
(
|
|
thesis_id,
|
|
path,
|
|
)
|
|
)
|
|
|
|
continue
|
|
|
|
work_id = (
|
|
"work:"
|
|
+ path
|
|
+ ":"
|
|
+ normalize_key(
|
|
work["heading"]
|
|
)
|
|
)
|
|
|
|
works[work_id] = {
|
|
"id": work_id,
|
|
"work_type": work[
|
|
"work_type"
|
|
],
|
|
"work_type_label": work[
|
|
"work_type_label"
|
|
],
|
|
**common_data,
|
|
}
|
|
|
|
person_works.add(
|
|
(
|
|
person_id,
|
|
work_id,
|
|
)
|
|
)
|
|
|
|
work_documents.add(
|
|
(
|
|
work_id,
|
|
path,
|
|
)
|
|
)
|
|
|
|
def relation_rows(
|
|
pairs: set[tuple[str, str]],
|
|
source_name: str,
|
|
target_name: str,
|
|
) -> list[dict[str, str]]:
|
|
return [
|
|
{
|
|
source_name: source,
|
|
target_name: target,
|
|
}
|
|
for source, target in sorted(
|
|
pairs
|
|
)
|
|
]
|
|
|
|
return {
|
|
"people": sorted(
|
|
people.values(),
|
|
key=lambda row: row["id"],
|
|
),
|
|
"document_authors": sorted(
|
|
document_authors.values(),
|
|
key=lambda row: row["id"],
|
|
),
|
|
"documents": sorted(
|
|
document_nodes.values(),
|
|
key=lambda row: row[
|
|
"path"
|
|
],
|
|
),
|
|
"categories": sorted(
|
|
categories.values(),
|
|
key=lambda row: row[
|
|
"name"
|
|
],
|
|
),
|
|
"topics": sorted(
|
|
topics.values(),
|
|
key=lambda row: row["id"],
|
|
),
|
|
"keywords": sorted(
|
|
keywords.values(),
|
|
key=lambda row: row["id"],
|
|
),
|
|
"theses": sorted(
|
|
theses.values(),
|
|
key=lambda row: row["id"],
|
|
),
|
|
"works": sorted(
|
|
works.values(),
|
|
key=lambda row: row["id"],
|
|
),
|
|
"person_documents": relation_rows(
|
|
person_documents,
|
|
"person_id",
|
|
"document_path",
|
|
),
|
|
"document_author_documents": relation_rows(
|
|
document_author_documents,
|
|
"document_author_id",
|
|
"document_path",
|
|
),
|
|
"document_categories": relation_rows(
|
|
document_categories,
|
|
"document_path",
|
|
"category_name",
|
|
),
|
|
"document_keywords": relation_rows(
|
|
document_keywords,
|
|
"document_path",
|
|
"keyword_id",
|
|
),
|
|
"document_describes_topics": (
|
|
relation_rows(
|
|
document_describes_topics,
|
|
"document_path",
|
|
"topic_id",
|
|
)
|
|
),
|
|
"person_theses": relation_rows(
|
|
person_theses,
|
|
"person_id",
|
|
"thesis_id",
|
|
),
|
|
"person_works": relation_rows(
|
|
person_works,
|
|
"person_id",
|
|
"work_id",
|
|
),
|
|
"thesis_documents": relation_rows(
|
|
thesis_documents,
|
|
"thesis_id",
|
|
"document_path",
|
|
),
|
|
"work_documents": relation_rows(
|
|
work_documents,
|
|
"work_id",
|
|
"document_path",
|
|
),
|
|
}
|