diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 90636a8..a4b68e9 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -116,7 +116,7 @@ class LMHyperParams: def lm_type(self): return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses): + def tokenizer_to_fastai_args(self, trn_data_loading_func, add_moses): tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer if self.tokenizer is Tokenizers.SUBWORD: if self.base_lm_path: # ensure we are using the same sentence piece model @@ -211,7 +211,7 @@ class LMHyperParams: for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' - args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) + args = self.tokenizer_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) try: data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 0f9221f..825d1f0 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -125,7 +125,7 @@ class CLSHyperParams(LMHyperParams): trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] cls_cache = '.' - args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) + args = self.tokenizer_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) try: if force: raise FileNotFoundError("Forcing reloading of caches")