From 908c3d7e8aeb4d6ef9a1575a4d48b0830ef79152 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 16:41:07 +0100 Subject: [PATCH] bug fix --- ulmfit/train_clas.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 652eba3..91d75d1 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -126,24 +126,26 @@ class CLSHyperParams(LMHyperParams): data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type) print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") except FileNotFoundError: - print("Running tokenization, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") + print(f"Running tokenization...") # wikitext is pretokenized with Moses data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=pd.concat([trn_df,tst_df]), valid_df=val_df, test_df=tst_df, - lm_type=self.lm_type, **args) + lm_type=self.lm_type, max_vocab=self.max_vocab, **args) data_lm.save('lm') + print(f" cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}") args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation try: data_cls = TextClasDataBunch.load(self.cache_dir, '.') print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") except FileNotFoundError: - print("Running tokenization, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") + print(f"Running tokenization...") data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, - valid_df=val_df, test_df=tst_df, + valid_df=val_df, test_df=tst_df, max_vocab=self.max_vocab, **args) + print(f" cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") data_cls.save('.') print('Size of vocabulary:', len(data_lm.vocab.itos)) print('First 20 words in vocab:', data_lm.vocab.itos[:20])