mirror of
https://github.com/wassname/multifit.git
synced 2026-08-28 12:52:26 +08:00
Fix loading tokenized data set in train cls
This commit is contained in:
@@ -68,6 +68,11 @@ def test_ulmfit_works_with_relative_paths():
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=1, unfreeze=False, bs=4,)
|
||||
|
||||
# should work for the second time as well
|
||||
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
|
||||
|
||||
|
||||
def test_ulmfit_default_end_to_end():
|
||||
""" Test ulmfit with (default) Moses tokenizer on small wikipedia dataset.
|
||||
|
||||
@@ -137,7 +137,7 @@ class CLSHyperParams(LMHyperParams):
|
||||
|
||||
args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation
|
||||
try:
|
||||
data_cls = TextClasDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type)
|
||||
data_cls = TextClasDataBunch.load(self.cache_dir, '.')
|
||||
print("Tokenized data loaded")
|
||||
except FileNotFoundError:
|
||||
print("Running tokenization")
|
||||
|
||||
Reference in New Issue
Block a user