From d79661045257bb75f810b631d0c065fa56e32294 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 17 Nov 2018 14:48:06 +0000 Subject: [PATCH] Removed deprecated default_rules --- fastai_contrib/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 9004132..688e28d 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -11,7 +11,7 @@ import re import csv from functools import reduce -from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab, default_rules +from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab from fastai.torch_core import * import shutil @@ -52,6 +52,7 @@ class SentencepieceTokenizer(BaseTokenizer): def add_special_cases(self, toks:Collection[str]): pass + def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=None, vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, pad_idx:int=PAD_TOKEN_ID): @@ -62,7 +63,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N path = pathlib.Path(path) os.makedirs(path / 'models', exist_ok=True) - rules = rules if rules else default_rules + rules = rules if rules else None cache_name = 'tmp'