Add generating of pseudo labels

ulmfit/__main__.py

@@ -88,21 +88,50 @@ class ULMFiT:
                     print("Adding", f, dest)
                     tar.add(f, dest)
 
+    def generate_pseudo_labels(self, glob="mldoc/*-1-laser-en1/models/sp15k/qrnn_nl4.m", bs=20, dest_dataset_template='${ds_name}-ps'):
+        for base_model in sorted(Path("data").glob(glob)):
+            print("Processing", base_model)
+
+            dataset_path = [x for x in base_model.parents if x.name == "models"][0].parent
+            lang = get_lang_from_dataset_path(dataset_path)
+            dest_dataset_path = dataset_path.parent/Template(dest_dataset_template).substitute(ds_name=dataset_path.name)
+            try:
+                _name = base_model.name.replace(".m", "").replace("lstm_", "").replace("qrnn_", "")
+                params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, cuda_id=0)
+                key = str(params.model_dir.relative_to(Path.cwd()))
+                if (params.model_dir / "results.npy").exists():
+                    d = np.load(params.model_dir / "results.npy")
+                    d = d.tolist()  # magiacally convert to dict
+                elif (params.model_dir / "cls_best.pth").exists():
+                    print("Evaluating previously trained model")
+                    d = params.validate_cls(label_smoothing_eps=0.1)
+                else:
+                    print("The model is not trained ignoring")
+                    continue
+                print("Generating pseduolabels", dest_dataset_path)
+                params.generate_pseudo_labels(dest_dataset_path, bs=bs)
+                del params
+            except Exception as e:
+                print("Error", e)
+                raise e
+            gc.collect()
+
+    def ls(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m"):
+        for i, name in enumerate(sorted(Path("data").glob(glob))):
+            print(i, name, "cls:", (name/"cls_best.pth").exists())
+
     def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${ds_name}', name=None,
              num_lm_epochs=0, cuda_id=0, train=True, to_csv=None, return_df=False, label_smoothing_eps=0.0,
              **trn_params):
         results = []
 
-
         def extract_agg(group):
             best = group.loc[group["val_accuracy"].idxmax()]["tst_accuracy"]
-            best_name = group.loc[group["val_accuracy"].idxmax()]["n"]
             return pd.Series({'best': best* 100,
                               'max': group['tst_accuracy'].max()* 100,
                               'avg': group['tst_accuracy'].mean()* 100})
         def pivot_to_lang(df):
-            df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]*)/models')
-            df['n'] = df['name'].str.extract(r'models/[^/]*/([^/]*).m')
+            df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]{1,12})[^/]*/models')
             best = df.groupby('ds').apply(extract_agg)
             best = best.round(2)
             return best.T

ulmfit/train_clas.py

@@ -118,6 +118,42 @@ class CLSHyperParams(LMHyperParams):
         results = {'val_loss': val_res[0], 'val_accuracy': float(val_res[1]), 'tst_loss':tst_res[0], 'tst_accuracy': float(tst_res[1]) }
         return results
 
+    def generate_pseudo_labels(self, dest_folder,  save_name='cls_best', bs=40, data_cls=None,  learn=None, label_smoothing_eps=0.0):
+        if data_cls is None:
+            data_cls, _, _ = self.load_cls_data(bs)
+        if learn is None:
+            learn = self.create_cls_learner(data_cls, drop_mult=0.3, label_smoothing_eps=label_smoothing_eps)
+            learn.unfreeze()
+        learn.load(save_name)
+
+        def make_data_set(ds_type, name):
+            probs, lbls = learn.get_preds(ds_type=ds_type, ordered=True)
+            preds = torch.argmax(probs, 1)
+            preds = to_np(preds)
+            fn = self.dataset_path / f"{self.lang}.{name}.csv"
+            if fn.exists():
+                df = pd.read_csv(fn, header=None)
+                df = df.iloc[(len(df) - len(preds)):]  # account for the training files where first 10% elements were taken as validation
+            else:
+                df = pd.read_csv(self.dataset_path / f"{self.lang}.dev.csv", header=None)
+                df = df.iloc[:len(preds)]  # if using training only get first n for validatation
+
+            accuracy = (df[0] == preds).sum() / len(preds)
+            print(f"Generating {name} dataset of size {len(preds)}, the accuracy is {accuracy}")
+            df['preds'] = preds
+            print(df.head())
+            del df['preds']
+            assert accuracy > 0.7, f"Accuracy is smaller than 0.7 {accuracy}"
+
+            df[0] = preds
+            dest_folder.mkdir(parents=True, exist_ok=True)
+            df.to_csv(dest_folder / f"{self.lang}.{name}.csv", index=None, header=None)
+
+        make_data_set(DatasetType.Train, "train")
+        make_data_set(DatasetType.Valid, "dev")
+        shutil.copy(self.dataset_path / f"{self.lang}.test.csv", dest_folder)
+        shutil.copy(self.dataset_path / f"{self.lang}.unsup.csv", dest_folder)
+
     def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"
         config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn)