wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Removed deprecated default_rules

Browse Source
This commit is contained in:
Sebastian
2018-11-17 14:48:06 +00:00
parent 7fe059c9a2
commit d796610452
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastai_contrib/utils.py
+3 -2
View File
@@ -11,7 +11,7 @@ import re
import csv
from functools import reduce
from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab, default_rules
from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab
from fastai.torch_core import *
import shutil
@@ -52,6 +52,7 @@ class SentencepieceTokenizer(BaseTokenizer):
def add_special_cases(self, toks:Collection[str]):
pass
def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=None,
vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7,
pad_idx:int=PAD_TOKEN_ID):
@@ -62,7 +63,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
path = pathlib.Path(path)
os.makedirs(path / 'models', exist_ok=True)
rules = rules if rules else default_rules
rules = rules if rules else None
cache_name = 'tmp'