From 039624870a17336f38bf3eddccc3af8317335ca1 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 01:28:03 +0100 Subject: [PATCH] Fix loading tokenized data set in train cls --- tests/test_end_to_end.py | 5 +++++ ulmfit/train_clas.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 85d8cd7..a5de301 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -68,6 +68,11 @@ def test_ulmfit_works_with_relative_paths(): exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=1, unfreeze=False, bs=4,) + # should work for the second time as well + + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) + def test_ulmfit_default_end_to_end(): """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 0782995..4bada19 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -137,7 +137,7 @@ class CLSHyperParams(LMHyperParams): args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation try: - data_cls = TextClasDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type) + data_cls = TextClasDataBunch.load(self.cache_dir, '.') print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization")