wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use trn + tst for LM training

Browse Source
This commit is contained in:
Piotr Czapla
2018-12-04 01:43:01 +01:00
parent 9826f6881c
commit f25deb3049
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/train_clas.py
+4 -4
View File
@@ -124,13 +124,13 @@ class CLSHyperParams(LMHyperParams):
try:
data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type)
print(f"Tokenized data loaded, trn.trn {len(data_trn.train_ds)}, trn.val {len(data_trn.valid_ds)}")
print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}")
except FileNotFoundError:
print("Running tokenization")
print("Running tokenization, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}")
# wikitext is pretokenized with Moses
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=trn_df,
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=pd.concat([trn_df,tst_df]),
valid_df=val_df, test_df=tst_df,
lm_type=self.lm_type, **args)
data_lm.save('lm')
@@ -140,7 +140,7 @@ class CLSHyperParams(LMHyperParams):
data_cls = TextClasDataBunch.load(self.cache_dir, '.')
print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}")
except FileNotFoundError:
print("Running tokenization")
print("Running tokenization, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}")
data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df,
valid_df=val_df, test_df=tst_df,
**args)