wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-11 12:20:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add callbacks to save history and best weights remove bs & drop_mult

Browse Source
This commit is contained in:
Piotr Czapla
2018-12-12 00:31:16 +01:00
parent fac3ce343b
commit f9394b9af1
3 changed files with 56 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_end_to_end.py
+22 -23
View File
@@ -13,13 +13,12 @@ It is a mixture of a pytest unit test and woven together to compose an end to en
import fastai.core
fastai.core.defaults.cpus = 1
cuda_id=0
def copy_head(src_fn, dst_fn, n=1000):
with src_fn.open("r") as s, dst_fn.open("w") as d:
for i in range(n):
d.write(s.readline())
def get_test_data():
data = get_data_folder()
wt = data / "wiki" / "wikitext-2"
@@ -35,9 +34,9 @@ def get_test_data():
sz=1
# we use the same text to see if models can overfit
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=10*sz)
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=6*sz)
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=6*sz)
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=1000*sz)
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz)
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz)
copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz)
copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz)
copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz)
@@ -59,10 +58,10 @@ def test_ulmfit_works_with_relative_paths():
lang='en',
qrnn=True,
max_vocab=1000,
bs=2,
name=lm_name)
name=lm_name,
cuda_id=cuda_id)
exp.train_lm(num_epochs=1)
exp.train_lm(num_epochs=1, bs=2)
#assert exp.results['accuracy'] > 0.02
@@ -86,10 +85,10 @@ def test_ulmfit_default_end_to_end():
lang='en',
qrnn=True,
max_vocab=1000,
bs=2,
name=lm_name)
name=lm_name,
cuda_id=cuda_id)
exp.train_lm(num_epochs=1)
exp.train_lm(num_epochs=1, bs=2)
#assert exp.results['accuracy'] > 0.02
@@ -101,7 +100,7 @@ def test_ulmfit_fastai_end_to_end():
"""
test_data, wt2 = get_test_data()
lm_name = 'end-to-end-test-fastai'
cuda_id = 0
exp = ulmfit.pretrain_lm.LMHyperParams(
dataset_path=wt2,
lang='en',
@@ -109,10 +108,9 @@ def test_ulmfit_fastai_end_to_end():
qrnn=True,
tokenizer='f',
max_vocab=100,
bs=2,
name=lm_name,
)
exp.train_lm(num_epochs=1)
exp.train_lm(num_epochs=1, bs=2)
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
@@ -121,7 +119,7 @@ def test_ulmfit_fastai_bidir_end_to_end():
"""
test_data, wt2 = get_test_data()
lm_name = 'end-to-end-test-fastai'
cuda_id = 0
exp = ulmfit.pretrain_lm.LMHyperParams(
dataset_path=wt2,
lang='en',
@@ -130,10 +128,9 @@ def test_ulmfit_fastai_bidir_end_to_end():
bidir=True,
tokenizer='f',
max_vocab=100,
bs=2,
name=lm_name,
)
exp.train_lm(num_epochs=1)
exp.train_lm(num_epochs=1, bs=2)
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir))
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
@@ -142,7 +139,7 @@ def test_ulmfit_moses_fa_bidir_end_to_end():
"""
test_data, wt2 = get_test_data()
lm_name = 'end-to-end-test-fastai'
cuda_id = 0
exp = ulmfit.pretrain_lm.LMHyperParams(
dataset_path=wt2,
lang='en',
@@ -151,19 +148,22 @@ def test_ulmfit_moses_fa_bidir_end_to_end():
bidir=True,
tokenizer='vf',
max_vocab=100,
bs=2,
name=lm_name,
)
exp.train_lm(num_epochs=1)
exp.train_lm(num_epochs=1, bs=2)
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
# def test_classification_model_work_with_different_dropmul():
# learn = self.create_cls_learner(data_clas, drop_mult=0.1)
# learn = self.create_cls_learner(data_clas, drop_mult=0.0)
def test_ulmfit_sentencepiece_end_to_end():
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
"""
test_data, wt2 = get_test_data()
lm_name = 'end-to-end-test-spm'
cuda_id = 0
exp = ulmfit.pretrain_lm.LMHyperParams(
dataset_path=wt2,
lang='en',
@@ -171,10 +171,9 @@ def test_ulmfit_sentencepiece_end_to_end():
qrnn=True,
tokenizer=ulmfit.pretrain_lm.Tokenizers.SUBWORD,
max_vocab=100,
bs=2,
name=lm_name,
)
exp.train_lm(num_epochs=1)
exp.train_lm(num_epochs=1, bs=2)
# not supported yet
# exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
# exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
ulmfit/pretrain_lm.py
+21 -20
View File
@@ -10,6 +10,7 @@ import fastai
import fire
from fastai import *
from fastai.callbacks import CSVLogger, SaveModelCallback
from fastai.text import *
import torch
from fastai_contrib.utils import read_file, read_whitespace_file, \
@@ -90,11 +91,9 @@ class LMHyperParams:
# these hyperparameters are for training on ~100M tokens (e.g. WikiText-103)
# for training on smaller datasets, more dropout is necessary
drop_mult = 0.1
dps = (0.25, 0.1, 0.2, 0.02, 0.15)
dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train
clip: float = 0.12
bptt: int = 70
bs: int = 70
lang: str = 'en'
name: str = None
@@ -114,7 +113,6 @@ class LMHyperParams:
self.model_dir = self.cache_dir / self.model_name
self.model_dir.mkdir(exist_ok=True, parents=True)
print('Batch size:', self.bs)
print('Max vocab:', self.max_vocab)
print('Cache dir:', self.cache_dir)
print('Model dir:', self.model_dir)
@@ -147,16 +145,16 @@ class LMHyperParams:
with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
print("Saving info", self.model_dir / 'info.json')
def train_lm(self, num_epochs=20, data_lm=None, true_wd=False, drop_mult=0.1, lr=5e-3):
data_lm = self.load_wiki_data() if data_lm is None else data_lm
def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3):
data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
learn = self.create_lm_learner(data_lm, drop_mult=drop_mult)
learn.true_wd = true_wd
try:
learn.load("lm_best_with_opt")
print("Continuing training")
except FileNotFoundError:
pass
# try:
# learn.load("lm_best_with_opt")
# print("Continuing training")
# except FileNotFoundError:
# pass
if num_epochs > 0:
if self.pretrained_fnames or self.pretrained_model:
print("Training lm from: ", self.pretrained_fnames or self.pretrained_model)
@@ -187,7 +185,7 @@ class LMHyperParams:
fastai.text.learner.default_dropout['language'] = dps or self.dps
lm_learner = bilm_learner if self.bidir else language_model_learner
trn_args = dict(drop_mult=self.drop_mult, tie_weights=True, clip=self.clip, bptt=self.bptt,
trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt,
pretrained_fnames=self.pretrained_fnames,
pretrained_model=self.pretrained_model)
trn_args.update(kwargs)
@@ -197,9 +195,11 @@ class LMHyperParams:
# compared to standard Adam, we set beta_1 to 0.8
learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy]
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"),
partial(SaveModelCallback, every='epoch', name='lm')]
return learn
def load_wiki_data(self):
def load_wiki_data(self, bs=70):
trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens'
val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens'
tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens'
@@ -215,7 +215,7 @@ class LMHyperParams:
sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab)
data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type)
data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=bs, bptt=self.bptt, lm_type=self.lm_type)
elif self.tokenizer is Tokenizers.MOSES:
# read the already whitespace separated data without any preprocessing
trn_tok = read_whitespace_file(trn_path)
@@ -243,12 +243,12 @@ class LMHyperParams:
# data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos))
data_lm = TextLMDataBunch.from_ids(path=self.dataset_path, vocab=vocab, train_ids=trn_ids,
valid_ids=val_ids, bs=self.bs, bptt=self.bptt,
valid_ids=val_ids, bs=bs, bptt=self.bptt,
lm_type=self.lm_type)
elif self.tokenizer is Tokenizers.MOSES_FA:
try:
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs)
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs)
print("Tokenized data loaded")
except FileNotFoundError:
print("Running tokenization")
@@ -258,18 +258,18 @@ class LMHyperParams:
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path),
valid_df=read_wiki_articles(val_path), tokenizer=pretokenized,
classes=None, lm_type=self.lm_type,
max_vocab=self.max_vocab, bs=self.bs, text_cols='texts')
max_vocab=self.max_vocab, bs=bs, text_cols='texts')
data_lm.save('.')
elif self.tokenizer is Tokenizers.FASTAI:
try:
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs)
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs)
print("Tokenized data loaded")
except FileNotFoundError:
print("Running tokenization")
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path),
valid_df=read_wiki_articles(val_path),
classes=None, lm_type=self.lm_type,
max_vocab=self.max_vocab,bs=self.bs, text_cols='texts')
max_vocab=self.max_vocab, bs=bs, text_cols='texts')
data_lm.save('.')
else:
raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}")
@@ -285,7 +285,8 @@ class LMHyperParams:
with open(base_lm_path/'info.json', 'r') as f: d = json.load(f)
d['dataset_path'] = dataset_path
d['base_lm_path'] = base_lm_path
d.pop('bs', None)
d.pop('drop_mult', None)
subword = d.pop('subword', False)
tokenizer = d.pop('tokenizer', None)
if tokenizer is not None:
ulmfit/train_clas.py
+13 -2
View File
@@ -9,6 +9,7 @@ import numpy as np
import pickle
from fastai import *
from fastai.callbacks import CSVLogger, SaveModelCallback
from fastai.text import *
import torch
@@ -57,7 +58,7 @@ class CLSHyperParams(LMHyperParams):
def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5,
use_test_for_validation=False):
data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=use_test_for_validation)
if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm)
learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls)
try:
@@ -90,11 +91,19 @@ class CLSHyperParams(LMHyperParams):
learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
print(f"Saving models at {learn.path / learn.model_dir}")
learn.save('cls_last', with_opt=False)
self.validate_cls('cls_last')
self.validate_cls('cls_best')
return learn
def validate_cls(self, save_name='cls_last', bs=40):
data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=True)
learn = self.create_cls_learner(data_clas, drop_mult=0.1)
learn.load(save_name)
print(f"Loss and accuracy using ({save_name}):", learn.validate())
def create_cls_learner(self, data_clas, dps=None, **kwargs):
fastai.text.learner.default_dropout['language'] = dps or self.dps
trn_args=dict(drop_mult=self.drop_mult, bptt=self.bptt, clip=self.clip,)
trn_args=dict(bptt=self.bptt, clip=self.clip,)
trn_args.update(kwargs)
classifier_learner = text_classifier_learner
if self.bidir:
@@ -103,6 +112,8 @@ class CLSHyperParams(LMHyperParams):
learn = classifier_learner(data_clas, pad_token=PAD_TOKEN_ID,
path=self.model_dir.parent, model_dir=self.model_dir.name,
qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, **trn_args)
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"),
partial(SaveModelCallback, every='improvement', name='cls_best')]
return learn
def load_cls_data(self, bs, **kwargs):