diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 0cd855e..3017ffc 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -97,7 +97,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb'))) # We cannot use lambdas or local methods here, since `tok_func` needs to be # pickle-able in order to be called in subprocesses when multithread tokenizing - tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang=str(path / 'models'), pre_rules=rules, post_rules=[]) + tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang=str(path / 'models'), rules=rules) clear_cache_directory(path, cache_name) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index fe2d4b7..64b2da2 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -72,7 +72,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab) - data_lm = TextLMDataBunch.from_csv(dir_path, **sp) + data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp) itos = data_lm.train_ds.vocab.itos stoi = data_lm.train_ds.vocab.stoi else: