Correct noise generation training + convenience functions

ulmfit/__main__.py

@@ -6,7 +6,7 @@ from collections import OrderedDict
 from functools import wraps
 
 import fire
-from .pretrain_lm import LMHyperParams
+from .pretrain_lm import LMHyperParams, np
 from .train_clas import CLSHyperParams
 from pathlib import Path
 from string import Template
@@ -25,7 +25,9 @@ def get_lang_from_dataset_path(ds):
 def get_dataset_path(p, dataset_template):
     ds = [x for x in p.parents if x.name == "models"][0].parent
     lang = get_lang_from_dataset_path(ds)
-    for ds_path in ds.parent.glob(Template(dataset_template).substitute(lang=lang, ds_name=ds.name)):
+    pattern = Template(dataset_template).substitute(lang=lang, ds_name=ds.name)
+    print(pattern)
+    for ds_path in ds.parent.glob(pattern):
         yield lang, ds_path
 
 class ULMFiT:
@@ -41,22 +43,47 @@ class ULMFiT:
         params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
         return FireView(train=params.train_cls, validate_cls=params.validate_cls)
 
+    @wraps(CLSHyperParams)
+    def load_cls(self, model_path, **changes):
+        params = CLSHyperParams.from_json(model_path, **changes)
+        return FireView(train=params.train_cls, validate_cls=params.validate_cls)
+
+
+    def eval_noise_resistance(self, lang="de"):
+        results = {}
+        for noise in range(0, 80, 5):
+            print("Noise: ", noise)
+            d = self.eval(glob=f"mldoc/{lang}-1/models/sp15k/qrnn_nl4.m",
+                          name=f"nl4_{noise}",
+                          noise=noise/100,
+                          num_cls_epochs=8,
+                          bs=18,
+                          lr_sched="1cycle")
+            results.update(d)
+            np.save('results.npy', results)
+        print(results)
+
     def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params):
         results = OrderedDict()
         for base_model in sorted(Path("data").glob(glob)):
+            print("Processing", base_model)
             for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
-                params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
-                key = str(params.model_dir.relative_to(Path.cwd()))
-                if (params.model_dir/"cls_last.pth").exists():
-                    print("Evaluating previously trained model")
-                    results[key] = params.validate_cls()[1]
-                else:
-                    print("Training")
-                    results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1]
-                del params
+                try:
+                    params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
+                    key = str(params.model_dir.relative_to(Path.cwd()))
+                    if (params.model_dir/"cls_last.pth").exists():
+                        print("Evaluating previously trained model")
+                        results[key] = params.validate_cls()[1]
+                    else:
+                        print("Training")
+                        results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1]
+                    del params
+                except Exception as e:
+                    print("Error", e)
                 gc.collect()
 
         pprint.pprint(results)
+        return results
 
     def remove_lm_saves(self):
         for lm_save in Path("data").glob("**/lm_*.pth"):

ulmfit/pretrain_lm.py

@@ -297,8 +297,8 @@ class LMHyperParams:
 
     @classmethod
     def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams':
-        base_lm_path = Path(base_lm_path).resolve()
         dataset_path = Path(dataset_path).resolve()
+        base_lm_path = Path(base_lm_path).resolve()
         with open(base_lm_path/'info.json', 'r') as f: d = json.load(f)
         d['dataset_path'] = dataset_path
         d['base_lm_path'] = base_lm_path
@@ -315,6 +315,13 @@ class LMHyperParams:
 
         d.update(kwargs)
         return cls(**d)
+    @classmethod
+    def from_json(cls, model_path, **kwargs):
+        model_path = Path(model_path).resolve()
+        with open(model_path / 'info.json', 'r') as f:
+            d = json.load(f)
+        d.update(kwargs)
+        return cls(**d)
 
 def validate_lm(self):
     if not self.exp.subword and self.exp.max_vocab is None:

ulmfit/train_clas.py

@@ -197,7 +197,7 @@ class CLSHyperParams(LMHyperParams):
             assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers"
             modulo = labels.max()+1
             idx_to_distrub = np.random.permutation(count)[:int(count * noise)]
-            trn_df.loc[idx_to_distrub, [0]] = (trn_df.loc[idx_to_distrub, [0]] + 1) % modulo
+            trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo-1, size=len(idx_to_distrub)) + trn_df.loc[idx_to_distrub][0]) % modulo
             print(f"Added noise to {len(idx_to_distrub)} examples, only {(count-len(idx_to_distrub))/count} have correct labels")
             cls_name = f'{cls_name}noise{noise}'