wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-10 12:12:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

tokenzier->tokenizer typo

Browse Source
This commit is contained in:
Tomasz Pietruszka
2019-01-13 17:27:29 +01:00
parent 593c5661bf
commit a1e66c39d4
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/pretrain_lm.py
+2 -2
View File
@@ -116,7 +116,7 @@ class LMHyperParams:
def lm_type(self):
return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM
def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses):
def tokenizer_to_fastai_args(self, trn_data_loading_func, add_moses):
tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer
if self.tokenizer is Tokenizers.SUBWORD:
if self.base_lm_path: # ensure we are using the same sentence piece model
@@ -211,7 +211,7 @@ class LMHyperParams:
for path_ in [trn_path, val_path, tst_path]:
assert path_.exists(), f'Error: {path_} does not exist.'
args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False)
args = self.tokenizer_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False)
try:
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs)
print("Tokenized data loaded")
ulmfit/train_clas.py
+1 -1
View File
@@ -125,7 +125,7 @@ class CLSHyperParams(LMHyperParams):
trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:]
cls_cache = '.'
args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True)
args = self.tokenizer_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True)
try:
if force: raise FileNotFoundError("Forcing reloading of caches")