Imporved validate_cls & eval to pick the best model based on val accuracy

ulmfit/__main__.py

@@ -7,7 +7,7 @@ from collections import OrderedDict
 from functools import wraps
 import pandas as pd
 import fire
-from .pretrain_lm import LMHyperParams
+from .pretrain_lm import LMHyperParams, json_save, json_load, np
 from .train_clas import CLSHyperParams
 from pathlib import Path
 from string import Template
@@ -40,8 +40,11 @@ class ULMFiT:
 
     lm2 = LMHyperParams
     @wraps(CLSHyperParams)
-    def cls(self, dataset_path, base_lm_path, **changes):
-        params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
+    def cls(self, dataset_path, base_lm_path=None, **changes):
+        if base_lm_path is not None:
+            params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
+        else:
+            params = CLSHyperParams(dataset_path=dataset_path, **changes)
         return FireView(train=params.train_cls, validate_cls=params.validate_cls)
 
     @wraps(CLSHyperParams)
@@ -63,8 +66,9 @@ class ULMFiT:
                           bs=bs,
                           lr_sched=lr_sched,
                           label_smoothing_eps=label_smoothing_eps,
+                          return_df=True,
                           **kwargs)
-            val = next(iter(d.values()), -1)
+            val = d['tst_accuracy'][0]
             results.append((noise/100, val))
         df = pd.DataFrame(results, columns=["noise", "accuracy"])
         df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv")
@@ -84,27 +88,61 @@ class ULMFiT:
                     print("Adding", f, dest)
                     tar.add(f, dest)
 
-    def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params):
-        results = OrderedDict()
+    def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${ds_name}', name=None,
+             num_lm_epochs=0, cuda_id=0, train=True, to_csv=None, return_df=False, label_smoothing_eps=0.0,
+             **trn_params):
+        results = []
+
+
+        def extract_agg(group):
+            best = group.loc[group["val_accuracy"].idxmax()]["tst_accuracy"]
+            best_name = group.loc[group["val_accuracy"].idxmax()]["n"]
+            return pd.Series({'best': best* 100,
+                              'max': group['tst_accuracy'].max()* 100,
+                              'avg': group['tst_accuracy'].mean()* 100})
+        def pivot_to_lang(df):
+            df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]*)/models')
+            df['n'] = df['name'].str.extract(r'models/[^/]*/([^/]*).m')
+            best = df.groupby('ds').apply(extract_agg)
+            best = best.round(2)
+            return best.T
         for base_model in sorted(Path("data").glob(glob)):
             print("Processing", base_model)
             for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
                 try:
-                    params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
+                    _name = name
+                    if name is None:
+                        _name = base_model.name.replace(".m","").replace("lstm_","").replace("qrnn_","")
+                    params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, cuda_id=cuda_id)
                     key = str(params.model_dir.relative_to(Path.cwd()))
-                    if (params.model_dir/"cls_best.pth").exists():
+                    if (params.model_dir / "results.npy").exists():
+                        d = np.load(params.model_dir / "results.npy")
+                        d = d.tolist() # magiacally convert to dict
+                    elif (params.model_dir/"cls_best.pth").exists():
                         print("Evaluating previously trained model")
-                        results[key] = params.validate_cls()[1]
-                    else:
+                        d = params.validate_cls(label_smoothing_eps=label_smoothing_eps)
+                    elif train:
                         print("Training")
-                        results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1]
+                        d = params.train_cls(num_lm_epochs=num_lm_epochs, label_smoothing_eps=label_smoothing_eps, **trn_params)
+                    else:
+                        print("Skipping", (params.model_dir/"cls_best.pth"))
+                        d  = None
+                    if d is not None:
+                        d['name']=key
+                        np.save(params.model_dir / "results.npy", d)
+                        results.append(d)
                     del params
                 except Exception as e:
                     print("Error", e)
                 gc.collect()
-
-        pprint.pprint(results)
-        return results
+        df = pd.DataFrame.from_records(results)
+        print(df)
+        print(pivot_to_lang(df))
+        if to_csv is not None:
+            print(f"Saving result to: {to_csv}")
+            df.to_csv(to_csv)
+        if return_df:
+            return df
 
     def remove_lm_saves(self):
         for lm_save in Path("data").glob("**/lm_*.pth"):

ulmfit/pretrain_lm.py

@@ -52,6 +52,14 @@ def read_wiki_articles(filename):
     print(f"Wiki text was split to {len(articles)} articles")
     return pd.DataFrame({'texts': np.array(articles, dtype=np.object)})
 
+def json_save(f, d):
+    with Path(f).open("w") as fp:
+        json.dump(d, fp)
+
+def json_load(f):
+    with open(f, 'r') as f:
+        return json.load(f)
+
 @dataclass
 class LMHyperParams:
     dataset_path: str # data_dir
@@ -168,7 +176,7 @@ class LMHyperParams:
         vals.pop('name', None)
         vals.pop('lang', None)
         vals['tokenizer'] = self.tokenizer.value
-        with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
+        json_save(self.model_dir/'info.json', vals)
         print("Saving info", self.model_dir / 'info.json')
 
     def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0):
@@ -249,7 +257,7 @@ class LMHyperParams:
 
         args = self.tokenizer_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False)
 
-        data_lm = self.lm_databunch("lm",
+        data_lm = self.lm_databunch(f"lm{self.bptt if self.bptt != 70 else ''}",
                           train_df=read_wiki_articles(trn_path),
                           valid_df=read_wiki_articles(val_path),
                           classes=None,
@@ -302,7 +310,7 @@ class LMHyperParams:
     def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams':
         dataset_path = Path(dataset_path).resolve()
         base_lm_path = Path(base_lm_path).resolve()
-        with open(base_lm_path/'info.json', 'r') as f: d = json.load(f)
+        d = json_load(base_lm_path/'info.json')
         d['dataset_path'] = dataset_path
         d['base_lm_path'] = base_lm_path
         d.pop('bs', None)

ulmfit/train_clas.py

@@ -10,7 +10,7 @@ from fastai_contrib.utils import PAD_TOKEN_ID
 
 import fire
 
-from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST
+from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST, json_save
 
 
 class CLSHyperParams(LMHyperParams):
@@ -101,18 +101,22 @@ class CLSHyperParams(LMHyperParams):
         learn.save('cls_last', with_opt=False)
         learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being
         del learn
-        return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
+        return self.validate_cls('cls_best', bs=bs, data_cls=data_clas, data_tst=data_tst, learn=None)
 
-    def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None):
+    def validate_cls(self, save_name='cls_best', bs=40, data_cls=None, data_tst=None, learn=None, label_smoothing_eps=0.0):
         if data_tst is None:
-            _, _, data_tst = self.load_cls_data(bs)
+            data_cls, _, data_tst = self.load_cls_data(bs)
         if learn is None:
-            learn = self.create_cls_learner(data_tst, drop_mult=0.3)
+            learn = self.create_cls_learner(data_tst, drop_mult=0.3, label_smoothing_eps=label_smoothing_eps)
             learn.unfreeze()
         learn.load(save_name)
-        results = learn.validate(data_tst.valid_dl)
-        print(f"Loss and accuracy using ({save_name}):", results)
-        return list(map(float, results))
+        val_res=[-1, -1]
+        if data_cls:
+            val_res = learn.validate(data_cls.valid_dl)
+        tst_res = learn.validate(data_tst.valid_dl)
+        print(f"Loss and accuracy using ({save_name}):", tst_res, val_res)
+        results = {'val_loss': val_res[0], 'val_accuracy': float(val_res[1]), 'tst_loss':tst_res[0], 'tst_accuracy': float(tst_res[1]) }
+        return results
 
     def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"