From 015f04ec087f85dbb89667ce3eb63be9d0281edc Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 22 Feb 2019 12:02:18 +0100 Subject: [PATCH] Training with noise & label smoothing --- tests/test_end_to_end.py | 20 ++++++++++++++++++++ ulmfit/__main__.py | 37 ++++++++++++++++++++++++++++--------- ulmfit/pretrain_lm.py | 23 +++++++++++++++++++---- ulmfit/train_clas.py | 35 +++++++++++++++++++++++------------ 4 files changed, 90 insertions(+), 25 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 266bc3f..73ded8a 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -115,6 +115,26 @@ def test_ulmfit_fastai_end_to_end(): exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) +def test_ulmfit_fastai_end_to_end_label_smoothing(): + """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. + """ + test_data, wt2 = get_test_data() + lm_name = 'end-to-end-test-fastai' + + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2, + lang='en', + cuda_id=cuda_id, + qrnn=False, + tokenizer='f', + max_vocab=100, + name=lm_name, + ) + exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1) + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 ) + + def test_ulmfit_fastai_bidir_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 9af2ee4..13a3922 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -1,12 +1,13 @@ import gc import os import pprint +import tarfile import shutil from collections import OrderedDict from functools import wraps - +import pandas as pd import fire -from .pretrain_lm import LMHyperParams, np +from .pretrain_lm import LMHyperParams from .train_clas import CLSHyperParams from pathlib import Path from string import Template @@ -49,19 +50,37 @@ class ULMFiT: return FireView(train=params.train_cls, validate_cls=params.validate_cls) - def eval_noise_resistance(self, lang="de"): - results = {} + def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m"): + def first_or_default(l, default=None): + l = list(l) + if l: + return l[0] + return default + results= [] for noise in range(0, 80, 5): print("Noise: ", noise) - d = self.eval(glob=f"mldoc/{lang}-1/models/sp15k/qrnn_nl4.m", - name=f"nl4_{noise}", + d = self.eval(glob=f"mldoc/{lang}-1/models/{model}", + name=f"nl4_{prefix_name}{noise}", noise=noise/100, + dataset_template='${lang}-'+str(size), num_cls_epochs=8, bs=18, lr_sched="1cycle") - results.update(d) - np.save('results.npy', results) - print(results) + val = first_or_default(d.values(), default=-1) + results.append((noise/100, val)) + df = pd.DataFrame(results, columns=["noise", "accuracy"]) + df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv") + print(df) + + def tar(self, model_path): + params = CLSHyperParams.from_json(model_path) + tar_name = f"models/{params.lang}-{params.tokenizer_prefix}-{params.model_name}.tar" + print("Storing model in", tar_name) + with tarfile.open(tar_name, mode="w") as tar: + for g in map(params.model_dir.glob, ['*_last.*', 'info.json', 'info.json', '../spm.*', '../itos.*',]): + for f in g: + print("Adding", f, f.relative_to("data")) + tar.add(f, f.relative_to("data")) def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params): results = OrderedDict() diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 655116c..5b70ef6 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -171,10 +171,10 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3): + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0): self.model_dir.mkdir(exist_ok=True, parents=True) data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm - learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) + learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps) learn.true_wd = true_wd if num_epochs > 0: @@ -204,7 +204,7 @@ class LMHyperParams: # do we need to return `learn'? it adds noise to Fire output #return learn - def create_lm_learner(self, data_lm, dps=None, **kwargs): + def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, tie_weights=True, out_bias=True) @@ -230,6 +230,8 @@ class LMHyperParams: learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"), # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues ] + if label_smoothing_eps > 0.0: + learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps) return learn def load_train_text(self): @@ -316,13 +318,26 @@ class LMHyperParams: d.update(kwargs) return cls(**d) @classmethod - def from_json(cls, model_path, **kwargs): + def from_json(cls, model_path:Path, **kwargs): model_path = Path(model_path).resolve() + name = re.search(r"[a-z]+_(.+).m", model_path.name).group(1) with open(model_path / 'info.json', 'r') as f: d = json.load(f) d.update(kwargs) + d['name'] = name + dataset_path = path_strip(model_path, "data", "models").parent + d['dataset_path'] = str(dataset_path) + d['lang'] = infer_lang_from_dataset(dataset_path.name) return cls(**d) +def infer_lang_from_dataset(name:str): + return name.split("-")[0] + +def path_strip(path, from_folder, to_folder): + to_p = [p for p in path.parents if p.name == to_folder][0] + from_p = [p for p in path.parents if p.name == from_folder][0] + return to_p.relative_to(from_p.parent) + def validate_lm(self): if not self.exp.subword and self.exp.max_vocab is None: raise NotImplementedError("figure out how to validate and save results") diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index e23688e..278a629 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -64,7 +64,8 @@ class CLSHyperParams(LMHyperParams): learn.fit_one_cycle(num_cls_epochs-4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, - use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered'): + use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered', + label_smoothing_eps=0.0): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" self.model_dir.mkdir(exist_ok=True, parents=True) @@ -73,8 +74,8 @@ class CLSHyperParams(LMHyperParams): data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise) - if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm) - learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len) + if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) + learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps) try: learn.load('cls_last') print("Loading last classifier") @@ -84,6 +85,8 @@ class CLSHyperParams(LMHyperParams): if hasattr(self, 'lr_schedule_'+lr_sched): learn.true_wd = True getattr(self, 'lr_schedule_'+lr_sched)(learn, num_cls_epochs) + else: + raise ValueError(f"Wrong lr_sched: {lr_sched}") print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) @@ -102,7 +105,7 @@ class CLSHyperParams(LMHyperParams): print(f"Loss and accuracy using ({save_name}):", results) return list(map(float, results)) - def create_cls_learner(self, data_clas, dps=None, **kwargs): + def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn) config.update(dps or self.dps) @@ -121,6 +124,8 @@ class CLSHyperParams(LMHyperParams): learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), #partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues ] + if label_smoothing_eps > 0.0: + learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps) return learn def load_cls_data(self, bs, **kwargs): @@ -178,6 +183,17 @@ class CLSHyperParams(LMHyperParams): kwargs.update(dict(trn_df=trn_df, val_df=val_df, tst_df=tst_df, unsup_df=unsup_df)) return kwargs + def add_noise(self, trn_df, noise): + count = len(trn_df) + labels = trn_df[0].unique() + assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers" + modulo = labels.max() + 1 + idx_to_distrub = np.random.permutation(count)[:int(count * noise)] + trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo - 1, size=len(idx_to_distrub)) + + trn_df.loc[idx_to_distrub][0]) % modulo + print(f"Added noise to {len(idx_to_distrub)} examples, only {(count - len(idx_to_distrub)) / count} have correct labels") + return trn_df + def databunches(self, bs, trn_df, val_df, tst_df, unsup_df, add_trn_to_lm=True, use_moses=False, force=False, limit=None, noise=0.0): lm_trn_df = pd.concat([unsup_df, val_df, tst_df] + ([trn_df] if add_trn_to_lm else [])) val_len = max(int(len(lm_trn_df) * 0.1), 2) @@ -192,14 +208,9 @@ class CLSHyperParams(LMHyperParams): cls_name=f'{cls_name}limit{limit}' if noise > 0.0: - count = len(trn_df) - labels = trn_df[0].unique() - assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers" - modulo = labels.max()+1 - idx_to_distrub = np.random.permutation(count)[:int(count * noise)] - trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo-1, size=len(idx_to_distrub)) + trn_df.loc[idx_to_distrub][0]) % modulo - print(f"Added noise to {len(idx_to_distrub)} examples, only {(count-len(idx_to_distrub))/count} have correct labels") - cls_name = f'{cls_name}noise{noise}' + trn_df = self.add_noise(trn_df, noise) + val_df = self.add_noise(val_df, noise) + cls_name = f'{cls_name}noise{noise}tv' args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) data_lm = self.lm_databunch('lm', train_df=lm_trn_df, valid_df=lm_val_df, bs=bs, force=force, **args)