From f25deb3049bd6d452839e5b22fa31c1ee87cde44 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 01:43:01 +0100 Subject: [PATCH] Use trn + tst for LM training --- ulmfit/train_clas.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 6ee1b59..652eba3 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -124,13 +124,13 @@ class CLSHyperParams(LMHyperParams): try: data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type) - print(f"Tokenized data loaded, trn.trn {len(data_trn.train_ds)}, trn.val {len(data_trn.valid_ds)}") + print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") except FileNotFoundError: - print("Running tokenization") + print("Running tokenization, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") # wikitext is pretokenized with Moses - data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=trn_df, + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=pd.concat([trn_df,tst_df]), valid_df=val_df, test_df=tst_df, lm_type=self.lm_type, **args) data_lm.save('lm') @@ -140,7 +140,7 @@ class CLSHyperParams(LMHyperParams): data_cls = TextClasDataBunch.load(self.cache_dir, '.') print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") except FileNotFoundError: - print("Running tokenization") + print("Running tokenization, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, valid_df=val_df, test_df=tst_df, **args)