From f9394b9af132b75af000c97b29e65f6385685037 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 12 Dec 2018 00:31:16 +0100 Subject: [PATCH] Add callbacks to save history and best weights remove bs & drop_mult --- tests/test_end_to_end.py | 45 ++++++++++++++++++++-------------------- ulmfit/pretrain_lm.py | 41 ++++++++++++++++++------------------ ulmfit/train_clas.py | 15 ++++++++++++-- 3 files changed, 56 insertions(+), 45 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 014b6f1..6f26c24 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -13,13 +13,12 @@ It is a mixture of a pytest unit test and woven together to compose an end to en import fastai.core fastai.core.defaults.cpus = 1 - +cuda_id=0 def copy_head(src_fn, dst_fn, n=1000): with src_fn.open("r") as s, dst_fn.open("w") as d: for i in range(n): d.write(s.readline()) - def get_test_data(): data = get_data_folder() wt = data / "wiki" / "wikitext-2" @@ -35,9 +34,9 @@ def get_test_data(): sz=1 # we use the same text to see if models can overfit - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=10*sz) - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=6*sz) - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=6*sz) + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=1000*sz) + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz) + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz) copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz) copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz) copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz) @@ -59,10 +58,10 @@ def test_ulmfit_works_with_relative_paths(): lang='en', qrnn=True, max_vocab=1000, - bs=2, - name=lm_name) + name=lm_name, + cuda_id=cuda_id) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) #assert exp.results['accuracy'] > 0.02 @@ -86,10 +85,10 @@ def test_ulmfit_default_end_to_end(): lang='en', qrnn=True, max_vocab=1000, - bs=2, - name=lm_name) + name=lm_name, + cuda_id=cuda_id) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) #assert exp.results['accuracy'] > 0.02 @@ -101,7 +100,7 @@ def test_ulmfit_fastai_end_to_end(): """ test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' - cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', @@ -109,10 +108,9 @@ def test_ulmfit_fastai_end_to_end(): qrnn=True, tokenizer='f', max_vocab=100, - bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) @@ -121,7 +119,7 @@ def test_ulmfit_fastai_bidir_end_to_end(): """ test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' - cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', @@ -130,10 +128,9 @@ def test_ulmfit_fastai_bidir_end_to_end(): bidir=True, tokenizer='f', max_vocab=100, - bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir)) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) @@ -142,7 +139,7 @@ def test_ulmfit_moses_fa_bidir_end_to_end(): """ test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' - cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', @@ -151,19 +148,22 @@ def test_ulmfit_moses_fa_bidir_end_to_end(): bidir=True, tokenizer='vf', max_vocab=100, - bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) +# def test_classification_model_work_with_different_dropmul(): +# learn = self.create_cls_learner(data_clas, drop_mult=0.1) +# learn = self.create_cls_learner(data_clas, drop_mult=0.0) + def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-spm' - cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', @@ -171,10 +171,9 @@ def test_ulmfit_sentencepiece_end_to_end(): qrnn=True, tokenizer=ulmfit.pretrain_lm.Tokenizers.SUBWORD, max_vocab=100, - bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) # not supported yet # exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) # exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 278ec43..3b499b8 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -10,6 +10,7 @@ import fastai import fire from fastai import * +from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ @@ -90,11 +91,9 @@ class LMHyperParams: # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) # for training on smaller datasets, more dropout is necessary - drop_mult = 0.1 - dps = (0.25, 0.1, 0.2, 0.02, 0.15) + dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 - bs: int = 70 lang: str = 'en' name: str = None @@ -114,7 +113,6 @@ class LMHyperParams: self.model_dir = self.cache_dir / self.model_name self.model_dir.mkdir(exist_ok=True, parents=True) - print('Batch size:', self.bs) print('Max vocab:', self.max_vocab) print('Cache dir:', self.cache_dir) print('Model dir:', self.model_dir) @@ -147,16 +145,16 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, true_wd=False, drop_mult=0.1, lr=5e-3): - data_lm = self.load_wiki_data() if data_lm is None else data_lm + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3): + data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) learn.true_wd = true_wd - try: - learn.load("lm_best_with_opt") - print("Continuing training") - except FileNotFoundError: - pass + # try: + # learn.load("lm_best_with_opt") + # print("Continuing training") + # except FileNotFoundError: + # pass if num_epochs > 0: if self.pretrained_fnames or self.pretrained_model: print("Training lm from: ", self.pretrained_fnames or self.pretrained_model) @@ -187,7 +185,7 @@ class LMHyperParams: fastai.text.learner.default_dropout['language'] = dps or self.dps lm_learner = bilm_learner if self.bidir else language_model_learner - trn_args = dict(drop_mult=self.drop_mult, tie_weights=True, clip=self.clip, bptt=self.bptt, + trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt, pretrained_fnames=self.pretrained_fnames, pretrained_model=self.pretrained_model) trn_args.update(kwargs) @@ -197,9 +195,11 @@ class LMHyperParams: # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), + partial(SaveModelCallback, every='epoch', name='lm')] return learn - def load_wiki_data(self): + def load_wiki_data(self, bs=70): trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' @@ -215,7 +215,7 @@ class LMHyperParams: sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab) - data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type) + data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=bs, bptt=self.bptt, lm_type=self.lm_type) elif self.tokenizer is Tokenizers.MOSES: # read the already whitespace separated data without any preprocessing trn_tok = read_whitespace_file(trn_path) @@ -243,12 +243,12 @@ class LMHyperParams: # data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos)) data_lm = TextLMDataBunch.from_ids(path=self.dataset_path, vocab=vocab, train_ids=trn_ids, - valid_ids=val_ids, bs=self.bs, bptt=self.bptt, + valid_ids=val_ids, bs=bs, bptt=self.bptt, lm_type=self.lm_type) elif self.tokenizer is Tokenizers.MOSES_FA: try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs) + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") @@ -258,18 +258,18 @@ class LMHyperParams: data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), valid_df=read_wiki_articles(val_path), tokenizer=pretokenized, classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab, bs=self.bs, text_cols='texts') + max_vocab=self.max_vocab, bs=bs, text_cols='texts') data_lm.save('.') elif self.tokenizer is Tokenizers.FASTAI: try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs) + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), valid_df=read_wiki_articles(val_path), classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab,bs=self.bs, text_cols='texts') + max_vocab=self.max_vocab, bs=bs, text_cols='texts') data_lm.save('.') else: raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") @@ -285,7 +285,8 @@ class LMHyperParams: with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) d['dataset_path'] = dataset_path d['base_lm_path'] = base_lm_path - + d.pop('bs', None) + d.pop('drop_mult', None) subword = d.pop('subword', False) tokenizer = d.pop('tokenizer', None) if tokenizer is not None: diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 93c0561..627cb09 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -9,6 +9,7 @@ import numpy as np import pickle from fastai import * +from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * import torch @@ -57,7 +58,7 @@ class CLSHyperParams(LMHyperParams): def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5, use_test_for_validation=False): data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=use_test_for_validation) - + if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm) learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls) try: @@ -90,11 +91,19 @@ class CLSHyperParams(LMHyperParams): learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) + self.validate_cls('cls_last') + self.validate_cls('cls_best') return learn + def validate_cls(self, save_name='cls_last', bs=40): + data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=True) + learn = self.create_cls_learner(data_clas, drop_mult=0.1) + learn.load(save_name) + print(f"Loss and accuracy using ({save_name}):", learn.validate()) + def create_cls_learner(self, data_clas, dps=None, **kwargs): fastai.text.learner.default_dropout['language'] = dps or self.dps - trn_args=dict(drop_mult=self.drop_mult, bptt=self.bptt, clip=self.clip,) + trn_args=dict(bptt=self.bptt, clip=self.clip,) trn_args.update(kwargs) classifier_learner = text_classifier_learner if self.bidir: @@ -103,6 +112,8 @@ class CLSHyperParams(LMHyperParams): learn = classifier_learner(data_clas, pad_token=PAD_TOKEN_ID, path=self.model_dir.parent, model_dir=self.model_dir.name, qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, **trn_args) + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), + partial(SaveModelCallback, every='improvement', name='cls_best')] return learn def load_cls_data(self, bs, **kwargs):