forked from KEMT/zpwiki
Nahrát soubory do „pages/students/2016/jakub_maruniak/dp2021/annotation“
This commit is contained in:
parent
9de29ed2b0
commit
99f36d3050
@ -0,0 +1,152 @@
|
||||
"""Scripts used for training and evaluation of NER models
|
||||
Usage example:
|
||||
$ python custom_train.py train ./model ./train.jsonl ./eval.jsonl -o ./output_dir -n 15
|
||||
Requirements:
|
||||
spacy>=2.2.3
|
||||
https://github.com/explosion/projects/tree/master/ner-drugs
|
||||
"""
|
||||
import spacy
|
||||
from spacy.cli.train import _load_pretrained_tok2vec
|
||||
from timeit import default_timer as timer
|
||||
from pathlib import Path
|
||||
import srsly
|
||||
from wasabi import msg
|
||||
import random
|
||||
import plac
|
||||
import sys
|
||||
import tqdm
|
||||
|
||||
|
||||
def format_data(data):
|
||||
result = []
|
||||
labels = set()
|
||||
for eg in data:
|
||||
if eg["answer"] != "accept":
|
||||
continue
|
||||
ents = [(s["start"], s["end"], s["label"]) for s in eg.get("spans", [])]
|
||||
labels.update([ent[2] for ent in ents])
|
||||
result.append((eg["text"], {"entities": ents}))
|
||||
return result, labels
|
||||
|
||||
|
||||
@plac.annotations(
|
||||
model=("The base model to load or blank:lang", "positional", None, str),
|
||||
train_path=("The training data (Prodigy JSONL)", "positional", None, str),
|
||||
eval_path=("The evaluation data (Prodigy JSONL)", "positional", None, str),
|
||||
n_iter=("Number of iterations", "option", "n", int),
|
||||
output=("Optional output directory", "option", "o", str),
|
||||
tok2vec=("Pretrained tok2vec weights to initialize model", "option", "t2v", str),
|
||||
)
|
||||
def train_model(
|
||||
model, train_path, eval_path, n_iter=10, output=None, tok2vec=None,
|
||||
):
|
||||
"""
|
||||
Train a model from Prodigy annotations and optionally save out the best
|
||||
model to disk.
|
||||
"""
|
||||
spacy.util.fix_random_seed(0)
|
||||
with msg.loading(f"Loading '{model}'..."):
|
||||
if model.startswith("blank:"):
|
||||
nlp = spacy.blank(model.replace("blank:", ""))
|
||||
else:
|
||||
nlp = spacy.load(model)
|
||||
msg.good(f"Loaded model '{model}'")
|
||||
train_data, labels = format_data(srsly.read_jsonl(train_path))
|
||||
eval_data, _ = format_data(srsly.read_jsonl(eval_path))
|
||||
ner = nlp.create_pipe("ner")
|
||||
for label in labels:
|
||||
ner.add_label(label)
|
||||
nlp.add_pipe(ner)
|
||||
t2v_cfg = {
|
||||
"embed_rows": 10000,
|
||||
"token_vector_width": 128,
|
||||
"conv_depth": 8,
|
||||
"nr_feature_tokens": 3,
|
||||
}
|
||||
optimizer = nlp.begin_training(component_cfg={"ner": t2v_cfg} if tok2vec else {})
|
||||
if tok2vec:
|
||||
_load_pretrained_tok2vec(nlp, Path(tok2vec))
|
||||
batch_size = spacy.util.compounding(1.0, 32.0, 1.001)
|
||||
best_acc = 0
|
||||
best_model = None
|
||||
row_widths = (2, 8, 8, 8, 8)
|
||||
msg.row(("#", "L", "P", "R", "F"), widths=row_widths)
|
||||
for i in range(n_iter):
|
||||
random.shuffle(train_data)
|
||||
losses = {}
|
||||
data = tqdm.tqdm(train_data, leave=False)
|
||||
for batch in spacy.util.minibatch(data, size=batch_size):
|
||||
texts, annots = zip(*batch)
|
||||
nlp.update(texts, annots, drop=0.2, losses=losses)
|
||||
with nlp.use_params(optimizer.averages):
|
||||
sc = nlp.evaluate(eval_data)
|
||||
if sc.ents_f > best_acc:
|
||||
best_acc = sc.ents_f
|
||||
if output:
|
||||
best_model = nlp.to_bytes()
|
||||
acc = (f"{sc.ents_p:.3f}", f"{sc.ents_r:.3f}", f"{sc.ents_f:.3f}")
|
||||
msg.row((i + 1, f"{losses['ner']:.2f}", *acc), widths=row_widths)
|
||||
msg.text(f"Best F-Score: {best_acc:.3f}")
|
||||
if output and best_model:
|
||||
with msg.loading("Saving model..."):
|
||||
nlp.from_bytes(best_model).to_disk(output)
|
||||
msg.good("Saved model", output)
|
||||
|
||||
|
||||
@plac.annotations(
|
||||
model=("The model to evaluate", "positional", None, str),
|
||||
eval_path=("The evaluation data (Prodigy JSONL)", "positional", None, str),
|
||||
)
|
||||
def evaluate_model(model, eval_path):
|
||||
"""
|
||||
Evaluate a trained model on Prodigy annotations and print the accuracy.
|
||||
"""
|
||||
with msg.loading(f"Loading model '{model}'..."):
|
||||
nlp = spacy.load(model)
|
||||
data, _ = format_data(srsly.read_jsonl(eval_path))
|
||||
sc = nlp.evaluate(data)
|
||||
result = [
|
||||
("Precision", f"{sc.ents_p:.3f}"),
|
||||
("Recall", f"{sc.ents_r:.3f}"),
|
||||
("F-Score", f"{sc.ents_f:.3f}"),
|
||||
]
|
||||
msg.table(result)
|
||||
|
||||
|
||||
@plac.annotations(
|
||||
model=("The model to evaluate", "positional", None, str),
|
||||
data=("Raw data as JSONL", "positional", None, str),
|
||||
)
|
||||
def wps(model, data):
|
||||
"""
|
||||
Measure the processing speed in words per second. It's recommended to
|
||||
use a larger corpus of raw text here (e.g. a few million words).
|
||||
"""
|
||||
with msg.loading(f"Loading model '{model}'..."):
|
||||
nlp = spacy.load(model)
|
||||
texts = (eg["text"] for eg in srsly.read_jsonl(data))
|
||||
n_docs = 0
|
||||
n_words = 0
|
||||
start_time = timer()
|
||||
for doc in nlp.pipe(texts):
|
||||
n_docs += 1
|
||||
n_words += len(doc)
|
||||
end_time = timer()
|
||||
wps = int(n_words / (end_time - start_time))
|
||||
result = [
|
||||
("Docs", f"{n_docs:,}"),
|
||||
("Words", f"{n_words:,}"),
|
||||
("Words/s", f"{wps:,}"),
|
||||
]
|
||||
msg.table(result, widths=(7, 12), aligns=("l", "r"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
opts = {"train": train_model, "evaluate": evaluate_model, "wps": wps}
|
||||
cmd = sys.argv.pop(1)
|
||||
if cmd not in opts:
|
||||
msg.fail(f"Unknown command: {cmd}", f"Available: {', '.join(opts)}", exits=1)
|
||||
try:
|
||||
plac.call(opts[cmd])
|
||||
except KeyboardInterrupt:
|
||||
msg.warn("Stopped.", exits=1)
|
160
pages/students/2016/jakub_maruniak/dp2021/annotation/scripts.py
Normal file
160
pages/students/2016/jakub_maruniak/dp2021/annotation/scripts.py
Normal file
@ -0,0 +1,160 @@
|
||||
"""
|
||||
Usage example:
|
||||
$ python scripts.py delete_annot jakub.maruniak ./dataset.jsonl ./new_dataset.jsonl
|
||||
|
||||
To see available commands:
|
||||
$ python scripts.py help
|
||||
|
||||
To see available arguments:
|
||||
$ python scripts.py [command] --help
|
||||
"""
|
||||
import spacy
|
||||
import srsly
|
||||
from wasabi import msg
|
||||
import plac
|
||||
import sys
|
||||
import re
|
||||
import itertools
|
||||
|
||||
@plac.annotations(
|
||||
dataset_path=("Path to dataset (Prodigy JSONL format)", "positional", None, str),
|
||||
)
|
||||
def count(
|
||||
dataset_path
|
||||
):
|
||||
"""
|
||||
Print statistics about Prodigy JSONL dataset.
|
||||
Prints number of accepted, rejected and ignored articles.
|
||||
Prints number of annotations of each entity type.
|
||||
Prints how much annotations were made by each annotator.
|
||||
"""
|
||||
# load data
|
||||
# filename = 'ner/skner/sknerv4spans.jsonl'
|
||||
file = open(sys.argv[1], 'rt', encoding='utf-8')
|
||||
text = file.read()
|
||||
|
||||
# count articles
|
||||
countAccept = text.count('accept')
|
||||
countReject = text.count('reject')
|
||||
countSkip = text.count('ignore')
|
||||
countSpans = text.count('tokens')
|
||||
# count entities
|
||||
countPER = text.count('PER')
|
||||
countLOC = text.count('LOC')
|
||||
countORG = text.count('ORG')
|
||||
countMISC = text.count('MISC')
|
||||
|
||||
underline = '\033[04m'
|
||||
reset = '\033[0m'
|
||||
red = '\033[31m'
|
||||
green='\033[32m'
|
||||
gray='\033[37m'
|
||||
|
||||
# table v1
|
||||
#from lib import TableIt
|
||||
#table1 = [
|
||||
# ['Prijatých', countAccept],
|
||||
# ['Zamietnutých', countReject],
|
||||
# ['Preskočených', countSkip],
|
||||
# ['------------', '------------'],
|
||||
# ['Spolu', countSpans]
|
||||
#]
|
||||
#
|
||||
#table = [
|
||||
# ['Entita', 'Počet'],
|
||||
# ['PER', countPER],
|
||||
# ['LOC', countLOC],
|
||||
# ['ORG', countORG],
|
||||
# ['MISC', countMISC]
|
||||
#]
|
||||
#print('\nPočet anotovaných článkov:')
|
||||
#TableIt.printTable(table1)
|
||||
#print('\nPočet jednotlivých entít:')
|
||||
#TableIt.printTable(table, useFieldNames=True, color=(26, 156, 171))
|
||||
|
||||
# table v2
|
||||
print(underline + '\nPočet anotovaných článkov:' + reset)
|
||||
print(green + "%-15s %-20s" %("Prijatých", countAccept) + reset)
|
||||
print(red + "%-15s %-15s" %("Zamietnutých", countReject) + reset)
|
||||
print(gray + "%-15s %-15s" %("Preskočených", countSkip) + reset)
|
||||
print("%-15s" %("---------------------"))
|
||||
print("%-15s %-15s" %("Spolu", countSpans))
|
||||
|
||||
print(underline + '\nPočet jednotlivých entít:' + reset)
|
||||
print("%-10s %-10s" %("Entita:", "Počet:"))
|
||||
print("%-10s" %("----------------"))
|
||||
print("%-10s %-10s" %("PER", countPER))
|
||||
print("%-10s %-10s" %("LOC", countLOC))
|
||||
print("%-10s %-10s" %("ORG", countORG))
|
||||
print("%-10s %-10s" %("MISC", countMISC))
|
||||
|
||||
# kto anotoval koľko?
|
||||
frequency = {}
|
||||
#Open the sample text file in read mode.
|
||||
#document_text = open('sample.txt', 'r')
|
||||
#convert the string of the document in lowercase and assign it to text_string variable.
|
||||
#text = document_text.read().lower()
|
||||
regex1 = '"_session_id":(.*?),'
|
||||
pattern = re.findall(regex1, text)
|
||||
for word in pattern:
|
||||
count = frequency.get(word,0)
|
||||
frequency[word] = count + 1
|
||||
frequency_list = frequency.keys()
|
||||
print(underline + '\nKto anotoval koľko článkov?' + reset)
|
||||
for words in frequency_list:
|
||||
print(words, frequency[words])
|
||||
|
||||
|
||||
@plac.annotations(
|
||||
annotator=("Keep annotations from this annotator (email address or nickname)", "positional", None, str),
|
||||
dataset_path=("Path to dataset (Prodigy JSONL format)", "positional", None, str),
|
||||
new_dataset_path=("Path to save new dataset(Prodigy JSONL format)", "positional", None, str),
|
||||
)
|
||||
def delete_annot(
|
||||
annotator, dataset_path, new_dataset_path
|
||||
):
|
||||
"""
|
||||
Load Prodigy JSONL dataset,
|
||||
and keep annotations only from one annotator.
|
||||
"""
|
||||
file1 = open(sys.argv[2], 'r', encoding='utf-8')
|
||||
file2 = open(sys.argv[3],'w', encoding='utf-8')
|
||||
|
||||
for line in file1.readlines():
|
||||
x = re.findall(sys.argv[1], line)
|
||||
if x:
|
||||
print(line)
|
||||
file2.write(line)
|
||||
|
||||
file1.close()
|
||||
file2.close()
|
||||
|
||||
def modelinfo(
|
||||
):
|
||||
"""
|
||||
Print information about trained model (Precision, Recall and F-Score)
|
||||
"""
|
||||
with open('build/train/nerposparser/model-best/meta.json', 'rt') as f:
|
||||
for line in itertools.islice(f, 31, 54):
|
||||
print(line, end =" ")
|
||||
|
||||
def helpme(
|
||||
):
|
||||
print("Available commands:",
|
||||
"\ncount - Print statistics about Prodigy JSONL dataset",
|
||||
"\ndelete_annot - Create dataset with annotations from only specific annotator",
|
||||
"\nmodelinfo - Prints informations about trained model (Precision, Recall and F-Score)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
opts = {"count": count,
|
||||
"delete_annot": delete_annot,
|
||||
"modelinfo": modelinfo,
|
||||
"help": helpme}
|
||||
cmd = sys.argv.pop(1)
|
||||
if cmd not in opts:
|
||||
msg.fail(f"Unknown command: {cmd}", f"Available: {', '.join(opts)}", exits=1)
|
||||
try:
|
||||
plac.call(opts[cmd])
|
||||
except KeyboardInterrupt:
|
||||
msg.warn("Stopped.", exits=1)
|
@ -0,0 +1,24 @@
|
||||
set -e
|
||||
OUTDIR=build/train/output
|
||||
TRAINDIR=build/train
|
||||
|
||||
# Delete old training results
|
||||
rm -r $TRAINDIR
|
||||
|
||||
mkdir -p $TRAINDIR
|
||||
mkdir -p $OUTDIR
|
||||
mkdir -p dist
|
||||
# Delete old training results
|
||||
rm -rf $OUTDIR/*
|
||||
# Train dependency and POS
|
||||
spacy train sk $OUTDIR ./build/input/slovak-treebank ./build/input/ud-artificial-gapping --n-iter 15 -p tagger,parser
|
||||
rm -rf $TRAINDIR/posparser
|
||||
mv $OUTDIR/model-best $TRAINDIR/posparser
|
||||
# Train NER
|
||||
# custom script for training, but it takes too long... input is JSONL file (db from Prodigy)
|
||||
# python custom_train.py train ./build/train/posparser ./train.jsonl ./eval.jsonl -o ./build/train/nerposparser -n 15
|
||||
spacy train sk $TRAINDIR/nerposparser ./ner/experiments/34sknerfull.json ./ner/experiments/34wikiartfull.json --n-iter 15 -p ner
|
||||
# Package model
|
||||
spacy package $TRAINDIR/nerposparser dist --meta-path ./meta.json --force
|
||||
cd dist/sk_sk1-0.2.0
|
||||
python ./setup.py sdist --dist-dir ../
|
@ -0,0 +1,80 @@
|
||||
"""
|
||||
Visualize the data with Streamlit and spaCy.
|
||||
https://github.com/explosion/projects/blob/master/ner-drugs/streamlit_visualizer.py
|
||||
|
||||
Usage example:
|
||||
$ streamlit run visualizer.py visualize ./dataset.jsonl
|
||||
"""
|
||||
import streamlit as st
|
||||
from spacy import displacy
|
||||
import srsly
|
||||
import sys
|
||||
import plac
|
||||
from wasabi import msg
|
||||
|
||||
@plac.annotations(
|
||||
dataset_path=("Path to dataset (Prodigy JSONL format)", "positional", None, str),
|
||||
)
|
||||
def visualize(
|
||||
dataset_path
|
||||
):
|
||||
FILES = [sys.argv[1]]
|
||||
MISC = "MISC"
|
||||
|
||||
HTML_WRAPPER = "<div style='border-bottom: 1px solid #ccc; padding: 20px 0'>{}</div>"
|
||||
HTML_WRAPPER1 = "<div style='border-bottom: 2px solid #000; padding: 0 0 20px 0'>{}</div>"
|
||||
SETTINGS = {"style": "ent", "manual": True, "options": {"colors": {MISC: "#d1bcff"}}}
|
||||
|
||||
@st.cache(allow_output_mutation=True)
|
||||
def load_data(filepath):
|
||||
return list(srsly.read_jsonl(filepath))
|
||||
|
||||
st.sidebar.title("Data visualizer")
|
||||
st.sidebar.markdown(
|
||||
"Visualize the annotations using [displaCy](https://spacy.io/usage/visualizers) "
|
||||
"and view stats about the datasets."
|
||||
)
|
||||
data_file = st.sidebar.selectbox("Dataset", FILES)
|
||||
data = load_data(data_file)
|
||||
n_no_ents = 0
|
||||
n_total_ents = 0
|
||||
accepted = 0
|
||||
rejected = 0
|
||||
|
||||
st.header(f"Dataset: {data_file} ({len(data)})")
|
||||
st.markdown(HTML_WRAPPER1.format("Visualize only accepted examples and their annotations."), unsafe_allow_html=True)
|
||||
for eg in data:
|
||||
if eg["answer"] == "accept":
|
||||
accepted += 1
|
||||
if eg["answer"] != "accept":
|
||||
rejected += 1
|
||||
continue
|
||||
row = {"text": eg["text"], "ents": eg.get("spans", [])}
|
||||
answer = {"answer": eg.get("answer", [])}
|
||||
n_total_ents += len(row["ents"])
|
||||
if not row["ents"]:
|
||||
n_no_ents += 1
|
||||
html = displacy.render(row, **SETTINGS).replace("\n\n", "\n")
|
||||
st.markdown(HTML_WRAPPER.format(html), unsafe_allow_html=True)
|
||||
|
||||
st.sidebar.markdown(
|
||||
f"""
|
||||
| `{data_file}` | |
|
||||
| --- | ---: |
|
||||
| Total examples | {len(data):,} |
|
||||
| Accepted examples | {accepted:,} |
|
||||
| Rejected examples | {rejected:,} |
|
||||
| Total entities | {n_total_ents:,} |
|
||||
| Examples with no entities | {n_no_ents:,} |
|
||||
""", unsafe_allow_html=True
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
opts = {"visualize": visualize}
|
||||
cmd = sys.argv.pop(1)
|
||||
if cmd not in opts:
|
||||
msg.fail(f"Unknown command: {cmd}", f"Available: {', '.join(opts)}", exits=1)
|
||||
try:
|
||||
plac.call(opts[cmd])
|
||||
except KeyboardInterrupt:
|
||||
msg.warn("Stopped.", exits=1)
|
Loading…
Reference in New Issue
Block a user