Training with noise & label smoothing

tests/test_end_to_end.py

@@ -115,6 +115,26 @@ def test_ulmfit_fastai_end_to_end():
     exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
     exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
 
+def test_ulmfit_fastai_end_to_end_label_smoothing():
+    """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
+    """
+    test_data, wt2 = get_test_data()
+    lm_name = 'end-to-end-test-fastai'
+
+    exp = ulmfit.pretrain_lm.LMHyperParams(
+        dataset_path=wt2,
+        lang='en',
+        cuda_id=cuda_id,
+        qrnn=False,
+        tokenizer='f',
+        max_vocab=100,
+        name=lm_name,
+    )
+    exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1)
+    exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
+    exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 )
+
+
 def test_ulmfit_fastai_bidir_end_to_end():
     """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
     """

ulmfit/__main__.py

@@ -1,12 +1,13 @@
 import gc
 import os
 import pprint
+import tarfile
 import shutil
 from collections import OrderedDict
 from functools import wraps
-
+import pandas as pd
 import fire
-from .pretrain_lm import LMHyperParams, np
+from .pretrain_lm import LMHyperParams
 from .train_clas import CLSHyperParams
 from pathlib import Path
 from string import Template
@@ -49,19 +50,37 @@ class ULMFiT:
         return FireView(train=params.train_cls, validate_cls=params.validate_cls)
 
 
-    def eval_noise_resistance(self, lang="de"):
-        results = {}
+    def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m"):
+        def first_or_default(l, default=None):
+            l = list(l)
+            if l:
+                return l[0]
+            return default
+        results= []
         for noise in range(0, 80, 5):
             print("Noise: ", noise)
-            d = self.eval(glob=f"mldoc/{lang}-1/models/sp15k/qrnn_nl4.m",
-                          name=f"nl4_{noise}",
+            d = self.eval(glob=f"mldoc/{lang}-1/models/{model}",
+                          name=f"nl4_{prefix_name}{noise}",
                           noise=noise/100,
+                          dataset_template='${lang}-'+str(size),
                           num_cls_epochs=8,
                           bs=18,
                           lr_sched="1cycle")
-            results.update(d)
-            np.save('results.npy', results)
-        print(results)
+            val = first_or_default(d.values(), default=-1)
+            results.append((noise/100, val))
+        df = pd.DataFrame(results, columns=["noise", "accuracy"])
+        df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv")
+        print(df)
+
+    def tar(self, model_path):
+        params = CLSHyperParams.from_json(model_path)
+        tar_name = f"models/{params.lang}-{params.tokenizer_prefix}-{params.model_name}.tar"
+        print("Storing model in", tar_name)
+        with tarfile.open(tar_name, mode="w") as tar:
+            for g in map(params.model_dir.glob, ['*_last.*', 'info.json', 'info.json', '../spm.*', '../itos.*',]):
+                for f in g:
+                    print("Adding", f, f.relative_to("data"))
+                    tar.add(f, f.relative_to("data"))
 
     def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params):
         results = OrderedDict()

ulmfit/pretrain_lm.py

@@ -171,10 +171,10 @@ class LMHyperParams:
         with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
         print("Saving info", self.model_dir / 'info.json')
 
-    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3):
+    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0):
         self.model_dir.mkdir(exist_ok=True, parents=True)
         data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
-        learn = self.create_lm_learner(data_lm, drop_mult=drop_mult)
+        learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps)
 
         learn.true_wd = true_wd
         if num_epochs > 0:
@@ -204,7 +204,7 @@ class LMHyperParams:
         # do we need to return `learn'? it adds noise to Fire output
         #return learn
 
-    def create_lm_learner(self, data_lm, dps=None, **kwargs):
+    def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"
         config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn,
                           tie_weights=True, out_bias=True)
@@ -230,6 +230,8 @@ class LMHyperParams:
         learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"),
                                # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues
                                ]
+        if label_smoothing_eps > 0.0:
+            learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps)
         return learn
 
     def load_train_text(self):
@@ -316,13 +318,26 @@ class LMHyperParams:
         d.update(kwargs)
         return cls(**d)
     @classmethod
-    def from_json(cls, model_path, **kwargs):
+    def from_json(cls, model_path:Path, **kwargs):
         model_path = Path(model_path).resolve()
+        name = re.search(r"[a-z]+_(.+).m", model_path.name).group(1)
         with open(model_path / 'info.json', 'r') as f:
             d = json.load(f)
         d.update(kwargs)
+        d['name'] = name
+        dataset_path = path_strip(model_path, "data", "models").parent
+        d['dataset_path'] = str(dataset_path)
+        d['lang'] = infer_lang_from_dataset(dataset_path.name)
         return cls(**d)
 
+def infer_lang_from_dataset(name:str):
+    return name.split("-")[0]
+
+def path_strip(path, from_folder, to_folder):
+    to_p = [p for p in path.parents if p.name == to_folder][0]
+    from_p = [p for p in path.parents if p.name == from_folder][0]
+    return to_p.relative_to(from_p.parent)
+
 def validate_lm(self):
     if not self.exp.subword and self.exp.max_vocab is None:
         raise NotImplementedError("figure out how to validate and save results")

ulmfit/train_clas.py

@@ -64,7 +64,8 @@ class CLSHyperParams(LMHyperParams):
                 learn.fit_one_cycle(num_cls_epochs-4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
 
     def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5,
-                   use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered'):
+                  use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered',
+                  label_smoothing_eps=0.0):
         assert use_test_for_validation == False, "use_test_for_validation=True is not supported"
         self.model_dir.mkdir(exist_ok=True, parents=True)
 
@@ -73,8 +74,8 @@ class CLSHyperParams(LMHyperParams):
 
         data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise)
 
-        if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm)
-        learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len)
+        if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps)
+        learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps)
         try:
             learn.load('cls_last')
             print("Loading last classifier")
@@ -84,6 +85,8 @@ class CLSHyperParams(LMHyperParams):
         if hasattr(self, 'lr_schedule_'+lr_sched):
             learn.true_wd = True
             getattr(self, 'lr_schedule_'+lr_sched)(learn, num_cls_epochs)
+        else:
+            raise ValueError(f"Wrong lr_sched: {lr_sched}")
 
         print(f"Saving models at {learn.path / learn.model_dir}")
         learn.save('cls_last', with_opt=False)
@@ -102,7 +105,7 @@ class CLSHyperParams(LMHyperParams):
         print(f"Loss and accuracy using ({save_name}):", results)
         return list(map(float, results))
 
-    def create_cls_learner(self, data_clas, dps=None, **kwargs):
+    def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"
         config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn)
         config.update(dps or self.dps)
@@ -121,6 +124,8 @@ class CLSHyperParams(LMHyperParams):
         learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"),
                                #partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues
                                ]
+        if label_smoothing_eps > 0.0:
+            learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps)
         return learn
 
     def load_cls_data(self, bs, **kwargs):
@@ -178,6 +183,17 @@ class CLSHyperParams(LMHyperParams):
         kwargs.update(dict(trn_df=trn_df, val_df=val_df, tst_df=tst_df, unsup_df=unsup_df))
         return kwargs
 
+    def add_noise(self, trn_df, noise):
+        count = len(trn_df)
+        labels = trn_df[0].unique()
+        assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers"
+        modulo = labels.max() + 1
+        idx_to_distrub = np.random.permutation(count)[:int(count * noise)]
+        trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo - 1, size=len(idx_to_distrub)) +
+                                           trn_df.loc[idx_to_distrub][0]) % modulo
+        print(f"Added noise to {len(idx_to_distrub)} examples, only {(count - len(idx_to_distrub)) / count} have correct labels")
+        return trn_df
+
     def databunches(self, bs, trn_df, val_df, tst_df, unsup_df, add_trn_to_lm=True, use_moses=False, force=False, limit=None, noise=0.0):
         lm_trn_df = pd.concat([unsup_df, val_df, tst_df] + ([trn_df] if add_trn_to_lm else []))
         val_len = max(int(len(lm_trn_df) * 0.1), 2)
@@ -192,14 +208,9 @@ class CLSHyperParams(LMHyperParams):
             cls_name=f'{cls_name}limit{limit}'
 
         if noise > 0.0:
-            count = len(trn_df)
-            labels = trn_df[0].unique()
-            assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers"
-            modulo = labels.max()+1
-            idx_to_distrub = np.random.permutation(count)[:int(count * noise)]
-            trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo-1, size=len(idx_to_distrub)) + trn_df.loc[idx_to_distrub][0]) % modulo
-            print(f"Added noise to {len(idx_to_distrub)} examples, only {(count-len(idx_to_distrub))/count} have correct labels")
-            cls_name = f'{cls_name}noise{noise}'
+            trn_df = self.add_noise(trn_df, noise)
+            val_df = self.add_noise(val_df, noise)
+            cls_name = f'{cls_name}noise{noise}tv'
 
         args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses)
         data_lm = self.lm_databunch('lm', train_df=lm_trn_df, valid_df=lm_val_df, bs=bs, force=force, **args)