wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-10 12:12:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34 from n-waves/use-configs

Browse Source 
Use configs
This commit is contained in:
Marcin Kardas
2019-02-13 22:58:47 +01:00
committed by GitHub
parent b14a393671 0e2211dfaf
commit f95a16cee1
3 changed files with 44 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/__main__.py
+1 -1
View File
@@ -53,4 +53,4 @@ class ULMFiT:
print(list(sorted(results.items())))
if __name__ == '__main__':
fire.Fire(ULMFiT())
fire.Fire(ULMFiT())
ulmfit/pretrain_lm.py
+24 -14
View File
@@ -69,7 +69,7 @@ class LMHyperParams:
# these hyperparameters are for training on ~100M tokens (e.g. WikiText-103)
# for training on smaller datasets, more dropout is necessary
dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train
dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
clip: float = 0.12
bptt: int = 70
# alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()
@@ -98,7 +98,6 @@ class LMHyperParams:
print('Max vocab:', self.max_vocab)
print('Cache dir:', self.cache_dir)
print('Model dir:', self.model_dir)
self.dps = np.array(self.dps)
if self.nh is None: self.nh = 1550 if self.qrnn else 1150
if self.name is None: self.name = self.lang
@@ -195,20 +194,29 @@ class LMHyperParams:
print(learn.path)
self.save_info()
return learn
# do we need to return `learn'? it adds noise to Fire output
#return learn
def create_lm_learner(self, data_lm, dps=None, **kwargs):
fastai.text.learner.default_dropout['language'] = dps or self.dps
lm_learner = bilm_learner if self.bidir else language_model_learner
trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt,
pretrained_fnames=self.pretrained_fnames,
pretrained_model=self.pretrained_model,
alpha=self.rnn_alpha, beta=self.rnn_beta)
assert self.bidir == False, "bidirectional model is not yet supported"
config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, bidir=self.bidir,
tie_weights=True, out_bias=True)
config.update(dps or self.dps)
trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta)
trn_args.update(kwargs)
print ("Training args: ", trn_args, "dps: ", dps or self.dps)
learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID,
bias=True, qrnn=self.qrnn, model_dir=self.model_dir.relative_to(data_lm.path), **trn_args)
learn = language_model_learner(data_lm, AWD_LSTM, config=config, model_dir=self.model_dir.relative_to(data_lm.path), pretrained=False, **trn_args)
if self.pretrained_model is not None:
print("Loading pretrained model")
model_path = untar_data(self.pretrained_model, data=False)
fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']]
learn.load_pretrained(*fnames)
learn.freeze()
if self.pretrained_fnames is not None:
print("Loading pretrained model")
fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(self.pretrained_fnames, ['pth', 'pkl'])]
learn.load_pretrained(*fnames)
learn.freeze()
# compared to standard Adam, we set beta_1 to 0.8
learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy]
@@ -231,13 +239,15 @@ class LMHyperParams:
args = self.tokenizer_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False)
try:
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs)
data_lm = TextLMDataBunch.load(self.cache_dir, '.',
bs=bs)
print("Tokenized data loaded")
except FileNotFoundError:
print("Running tokenization")
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path),
valid_df=read_wiki_articles(val_path),
classes=None, lm_type=self.lm_type, max_vocab=self.max_vocab,
classes=None,
max_vocab=self.max_vocab,
bs=bs, text_cols='texts', **args)
data_lm.save('.')
ulmfit/train_clas.py
+19 -13
View File
@@ -24,6 +24,7 @@ from pathlib import Path
from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST
class CLSHyperParams(LMHyperParams):
# dir_path -> data/imdb/
use_test_for_validation=False
@@ -38,14 +39,14 @@ class CLSHyperParams(LMHyperParams):
def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists()
def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5,
use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0):
use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70):
assert use_test_for_validation == False, "use_test_for_validation=True is not supported"
self.model_dir.mkdir(exist_ok=True, parents=True)
data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise)
if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm)
learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls)
learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len)
try:
learn.load('cls_last')
print("Loading last classifier")
@@ -91,16 +92,21 @@ class CLSHyperParams(LMHyperParams):
return list(map(float, results))
def create_cls_learner(self, data_clas, dps=None, **kwargs):
fastai.text.learner.default_dropout['language'] = dps or self.dps
trn_args=dict(bptt=self.bptt, clip=self.clip,)
assert self.bidir == False, "bidirectional model is not yet supported"
config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, bidir=self.bidir)
config.update(dps or self.dps)
trn_args=dict(bptt=self.bptt, clip=self.clip)
trn_args.update(kwargs)
classifier_learner = text_classifier_learner
if self.bidir:
classifier_learner = bilm_text_classifier_learner
trn_args['bicls_head'] = self.bicls_head
learn = classifier_learner(data_clas, pad_token=PAD_TOKEN_ID,
path=self.model_dir.parent, model_dir=self.model_dir.name,
qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, **trn_args)
learn = text_classifier_learner(data_clas, AWD_LSTM, config=config,
pretrained=False, path=self.model_dir.parent, model_dir=self.model_dir.name, **trn_args)
if self.pretrained_model is not None:
print("Loading pretrained model")
model_path = untar_data(self.pretrained_model, data=False)
fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']]
learn.load_pretrained(*fnames, strict=False)
learn.freeze()
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"),
partial(SaveModelCallback, every='improvement', name='cls_best')]
return learn
@@ -155,12 +161,12 @@ class CLSHyperParams(LMHyperParams):
args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses)
try:
if force: raise FileNotFoundError("Forcing reloading of caches")
data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type, bs=bs)
data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', bs=bs)
print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}")
except FileNotFoundError:
print(f"Running tokenization...")
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=lm_trn_df, valid_df=lm_val_df,
max_vocab=self.max_vocab, bs=bs, lm_type=self.lm_type, **args)
max_vocab=self.max_vocab, bs=bs, **args)
print(f"Saving tokenized: cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}")
data_lm.save('lm')