wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add saving itos.pkl so that the LM can be used to finetuning

Browse Source
This commit is contained in:
Piotr Czapla
2019-02-17 23:18:42 +01:00
parent c3276da062
commit ce6cc607ae
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/pretrain_lm.py
+4 -1
View File
@@ -263,7 +263,7 @@ class LMHyperParams:
def lm_databunch(self, name, *args, **kwargs):
return self.databunch(name, bunch_class=TextLMDataBunch, *args, **kwargs)
def databunch(self, name, bunch_class, train_df, valid_df, bs, force, **args):
def databunch(self, name, bunch_class, train_df, valid_df, bs, force=False, **args):
bunch_path = self.cache_dir / name
if force and bunch_path.exist():
print("Forcefully recreating the databunch, removing previously stored data")
@@ -288,6 +288,9 @@ class LMHyperParams:
bs=bs,
**args)
data.save(name)
with open(self.cache_dir/"itos.pkl", 'wb') as f:
pickle.dump(data.vocab.itos, f)
print(f"Data {name}, trn: {len(data.train_ds)}, val: {len(data.valid_ds)}")
return data