From 40a29903228b53db28d41deee0a87c117b3e8f95 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Tue, 20 Nov 2018 09:06:03 -0300 Subject: [PATCH] Add suggested changes to rules system --- fastai_contrib/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 101bce3..4507c7c 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -67,7 +67,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N cache_name = 'tmp' os.makedirs(path / cache_name, exist_ok=True) os.makedirs(path / 'models', exist_ok=True) - rules = rules if rules else None + rules = rules if rules is not None else [] # load the text frmo the train tokens file @@ -97,7 +97,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb'))) # We cannot use lambdas or local methods here, since `tok_func` needs to be # pickle-able in order to be called in subprocesses when multithread tokenizing - tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), rules=rules) + tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), pre_rules=rules, post_rules=[]) clear_cache_directory(path, cache_name)