Add more seeds

split-cls.py

@@ -0,0 +1,18 @@
+import pandas as pd, numpy as np
+import fire
+from pathlib import Path
+from sys import stderr
+from sklearn.model_selection import train_test_split
+
+def to_csv(df, path):
+    df.to_csv(path, header=None, index=None)
+
+def split(data_dir):
+    data_dir = Path(data_dir)
+    train = pd.read_csv(data_dir / "pl.unsup.csv", header=None)
+    trn, val = train_test_split(train, test_size=0.1, random_state=12345, stratify=train[0])
+
+    to_csv(trn, data_dir / "pl.train.csv")
+    to_csv(val, data_dir / "pl.dev.csv")
+
+if __name__ == "__main__": fire.Fire(split)

ulmfit/pretrain_lm.py

@@ -71,6 +71,9 @@ class LMHyperParams:
     emb_sz:int = 400
     nh: int = None
     nl: int = 3
+    out_bias: bool = True
+
+    lmseed: int = None
 
     # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103)
     # for training on smaller datasets, more dropout is necessary
@@ -123,7 +126,10 @@ class LMHyperParams:
     def model_prefix(self): return self.model_direction + ('qrnn' if self.qrnn else 'lstm')
 
     @property
-    def model_name(self): return f"{self.model_prefix}_{self.name}.m"
+    def model_name(self): return f"{self.model_prefix}_{self.name}{self.model_suffix}.m"
+
+    @property
+    def model_suffix(self): return '' if self.lmseed is None else f'_lmseed-{self.lmseed}'
 
     @property
     def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None
@@ -167,6 +173,14 @@ class LMHyperParams:
                 f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}")
         return args
 
+    def set_seed(self, seed, name):
+        if seed is not None:
+            print(f"Setting {name} seed to {seed}")
+            torch.manual_seed(seed)
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.benchmark = False
+            np.random.seed(seed)
+
     def save_info(self):
         from dataclasses import asdict
         vals = {k: (str(v) if isinstance(v, Path) else v) for k,v in asdict(self).items()}
@@ -176,17 +190,15 @@ class LMHyperParams:
         with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
         print("Saving info", self.model_dir / 'info.json')
 
-    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True, seed=None):
-        if seed is not None:
-            print(f"Setting seed to {seed}")
-            torch.manual_seed(seed)
-            torch.backends.cudnn.deterministic = True
-            torch.backends.cudnn.benchmark = False
-            np.random.seed(seed)
+    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0):
+        if self.ftseed is None:
+            self.set_seed(self.lmseed, "LM")
+        else:
+            self.set_seed(self.ftseed, "fine-tune")
 
         self.model_dir.mkdir(exist_ok=True, parents=True)
         data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
-        learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps, out_bias=out_bias)
+        learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps)
         print("Bptt", data_lm.bptt)
         learn.true_wd = true_wd
         if num_epochs > 0:
@@ -216,10 +228,10 @@ class LMHyperParams:
         # do we need to return `learn'? it adds noise to Fire output
         #return learn
 
-    def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, out_bias=True, **kwargs):
+    def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"
         config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn,
-                          tie_weights=True, out_bias=out_bias)
+                          tie_weights=True, out_bias=self.out_bias)
         config.update(dps or self.dps)
         trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta)
         trn_args.update(kwargs)

ulmfit/train_clas.py

@@ -14,14 +14,13 @@ from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST
 
 from sklearn.metrics import f1_score as f1s, precision_score, recall_score
 
-def f1_score(preds, targs):
-    preds = torch.max(preds, dim=1)[1].cpu().numpy()
-    targs = targs.cpu().numpy()
-    return torch.tensor(f1s(targs, preds))
-
+@dataclass
 class CLSHyperParams(LMHyperParams):
     # dir_path -> data/imdb/
-    use_test_for_validation=False
+    use_test_for_validation: bool=False
+    ftseed: int = None
+    clsweightseed: int = None
+    clstrainseed: int = None
 
     bicls_head:str = 'BiPoolingLinearClassifier'
 
@@ -29,6 +28,17 @@ class CLSHyperParams(LMHyperParams):
         super().__post_init__(*args, **kwargs)
         self.dataset_dir=self.dataset_path
 
+    @property
+    def model_suffix(self):
+        s1 = '' if self.lmseed is None else f'lmseed-{self.lmseed}'
+        s2 = '' if self.ftseed is None else f'ftseed-{self.ftseed}'
+        s3 = '' if self.clsweightseed is None else f'clsweightseed-{self.clsweightseed}'
+        s4 = '' if self.clstrainseed is None else f'clstrainseed-{self.clstrainseed}'
+        s = '-'.join([x for x in [s1, s2, s3, s4] if x != ''])
+        if s != '':
+            return '_'+s
+        return ''
+
     @property
     def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists()
 
@@ -69,9 +79,28 @@ class CLSHyperParams(LMHyperParams):
             if num_cls_epochs > 5:
                 learn.fit_one_cycle(num_cls_epochs-4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
 
+    def get_metrics(self):
+        f1_score = FBeta(beta=1.0)
+        precision = Precision()
+        recall = Recall()
+        metrics = [f1_score, precision, recall]
+
+        # TODO: fix this in fast.ai
+        for metric in metrics: metric.on_train_begin()
+        metrics.append(accuracy)
+        return metrics
+
+    def output_metrics(self, results):
+        print(f"F1 score bin: {results[1].item()}")
+        print(f"Loss: {results[0]}")
+        print(f"Precision: {results[2].item()}")
+        print(f"Recall: {results[3].item()}")
+        print(f"Accuracy: {results[4].item()}")
+
+
     def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5,
                   use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered',
-                  label_smoothing_eps=0.0, random_init=False, seed=None, dump_preds=None):
+                  label_smoothing_eps=0.0, random_init=False, dump_preds=None):
         assert use_test_for_validation == False, "use_test_for_validation=True is not supported"
         self.model_dir.mkdir(exist_ok=True, parents=True)
 
@@ -82,13 +111,16 @@ class CLSHyperParams(LMHyperParams):
 
         if self.need_fine_tune_lm and not random_init:
             if not (self.model_dir/(ENC_BEST+".pth")).exists():
-                self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps, seed=seed)
+                self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps)
             else:
                 print("Language model already exist, skipping finetuning")
         loss_func = CrossEntropyFlat(weight=torch.FloatTensor([0.5,30]).cuda())
+
+        self.set_seed(self.clsweightseed, "classifier weights")
+
         learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len,
                                         label_smoothing_eps=label_smoothing_eps, random_init=random_init,
-                                        metrics=[FBeta(beta=1.0), f1_score, accuracy],
+                                        metrics=self.get_metrics(),
                                         loss_func=loss_func)
 
         if not random_init:
@@ -100,12 +132,8 @@ class CLSHyperParams(LMHyperParams):
         else:
             print("Starting classifier from random weights")
 
-        if seed is not None:
-            print(f"Setting seed to {seed}")
-            torch.manual_seed(seed)
-            torch.backends.cudnn.deterministic = True
-            torch.backends.cudnn.benchmark = False
-            np.random.seed(seed)
+
+        self.set_seed(self.clstrainseed, "classifier train")
 
         if hasattr(self, 'lr_schedule_'+lr_sched):
             learn.true_wd = True
@@ -119,13 +147,11 @@ class CLSHyperParams(LMHyperParams):
         del learn
         return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
 
-    def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None, dump_preds=None):
+    def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None, dump_preds=None, mode="test"):
         if data_tst is None:
-            _, _, data_tst = self.load_cls_data(bs)
+            data_clas , _, data_tst = self.load_cls_data(bs)
         if learn is None:
-            fbeta = FBeta(beta=1.0)
-            fbeta.on_train_begin()
-            learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[fbeta, f1_score, accuracy])
+            learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=self.get_metrics())
             learn.unfreeze()
         learn.load(save_name)
         probs, targets = learn.get_preds(ordered=True)
@@ -133,9 +159,11 @@ class CLSHyperParams(LMHyperParams):
         if dump_preds:
             with open(dump_preds, 'w') as f:
                 f.write('\n'.join([str(x) for x in preds]))
-        results = learn.validate(data_tst.valid_dl)
-        print(f"F1 score bin: {results[1].item()}")
-        print(f"Loss, f1_score, almost f1_score and accuracy using ({save_name}):", results)
+        results = learn.validate(data_tst.valid_dl if mode == "test" else data_clas.valid_dl)
+        print(f"Model: {self.name}")
+        print(f"Validation on: {mode}")
+        self.output_metrics(results)
+
         return list(map(float, results))
 
     def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs):