14 lines
526 B
Python
14 lines
526 B
Python
|
from tokenizers import Tokenizer
|
||
|
|
from tokenizers.models import BPE
|
||
|
|
from tokenizers.trainers import BpeTrainer
|
||
|
|
from tokenizers.pre_tokenizers import Whitespace
|
||
|
|
|
||
|
|
# training the tokenizer
|
||
|
|
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
|
||
|
|
trainer = BpeTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
|
||
|
|
tokenizer.pre_tokenizer = Whitespace()
|
||
|
|
files = [f"data/wikitext-103-raw/wiki.{split}.raw" for split in ["test", "train", "valid"]]
|
||
|
|
tokenizer.train(files, trainer)
|
||
|
|
tokenizer.save("data/bpe-tokenizer-wiki.json")
|
||
|
|
|