wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

bug fix

Browse Source
This commit is contained in:
Piotr Czapla
2018-12-04 16:41:07 +01:00
parent f25deb3049
commit 908c3d7e8a
1 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/train_clas.py
+6 -4
View File
@@ -126,24 +126,26 @@ class CLSHyperParams(LMHyperParams):
data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type)
print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}")
except FileNotFoundError:
print("Running tokenization, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}")
print(f"Running tokenization...")
# wikitext is pretokenized with Moses
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=pd.concat([trn_df,tst_df]),
valid_df=val_df, test_df=tst_df,
lm_type=self.lm_type, **args)
lm_type=self.lm_type, max_vocab=self.max_vocab, **args)
data_lm.save('lm')
print(f" cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}")
args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation
try:
data_cls = TextClasDataBunch.load(self.cache_dir, '.')
print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}")
except FileNotFoundError:
print("Running tokenization, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}")
print(f"Running tokenization...")
data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df,
valid_df=val_df, test_df=tst_df,
valid_df=val_df, test_df=tst_df, max_vocab=self.max_vocab,
**args)
print(f" cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}")
data_cls.save('.')
print('Size of vocabulary:', len(data_lm.vocab.itos))
print('First 20 words in vocab:', data_lm.vocab.itos[:20])