Fix loading tokenized data set in train cls

This commit is contained in:
Piotr Czapla
2018-12-04 01:28:03 +01:00
parent a499bf9a20
commit 039624870a
2 changed files with 6 additions and 1 deletions
+5
View File
@@ -68,6 +68,11 @@ def test_ulmfit_works_with_relative_paths():
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
exp2.train_cls(num_lm_epochs=1, unfreeze=False, bs=4,)
# should work for the second time as well
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
def test_ulmfit_default_end_to_end():
""" Test ulmfit with (default) Moses tokenizer on small wikipedia dataset.
+1 -1
View File
@@ -137,7 +137,7 @@ class CLSHyperParams(LMHyperParams):
args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation
try:
data_cls = TextClasDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type)
data_cls = TextClasDataBunch.load(self.cache_dir, '.')
print("Tokenized data loaded")
except FileNotFoundError:
print("Running tokenization")