From 24f1d741f0fa8a2a6663f82226c7de306d4d077f Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 26 Mar 2019 21:27:56 +0100 Subject: [PATCH] Add generating of pseudo labels --- ulmfit/__main__.py | 37 +++++++++++++++++++++++++++++++++---- ulmfit/train_clas.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index cec29f7..01e109d 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -88,21 +88,50 @@ class ULMFiT: print("Adding", f, dest) tar.add(f, dest) + def generate_pseudo_labels(self, glob="mldoc/*-1-laser-en1/models/sp15k/qrnn_nl4.m", bs=20, dest_dataset_template='${ds_name}-ps'): + for base_model in sorted(Path("data").glob(glob)): + print("Processing", base_model) + + dataset_path = [x for x in base_model.parents if x.name == "models"][0].parent + lang = get_lang_from_dataset_path(dataset_path) + dest_dataset_path = dataset_path.parent/Template(dest_dataset_template).substitute(ds_name=dataset_path.name) + try: + _name = base_model.name.replace(".m", "").replace("lstm_", "").replace("qrnn_", "") + params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, cuda_id=0) + key = str(params.model_dir.relative_to(Path.cwd())) + if (params.model_dir / "results.npy").exists(): + d = np.load(params.model_dir / "results.npy") + d = d.tolist() # magiacally convert to dict + elif (params.model_dir / "cls_best.pth").exists(): + print("Evaluating previously trained model") + d = params.validate_cls(label_smoothing_eps=0.1) + else: + print("The model is not trained ignoring") + continue + print("Generating pseduolabels", dest_dataset_path) + params.generate_pseudo_labels(dest_dataset_path, bs=bs) + del params + except Exception as e: + print("Error", e) + raise e + gc.collect() + + def ls(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m"): + for i, name in enumerate(sorted(Path("data").glob(glob))): + print(i, name, "cls:", (name/"cls_best.pth").exists()) + def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${ds_name}', name=None, num_lm_epochs=0, cuda_id=0, train=True, to_csv=None, return_df=False, label_smoothing_eps=0.0, **trn_params): results = [] - def extract_agg(group): best = group.loc[group["val_accuracy"].idxmax()]["tst_accuracy"] - best_name = group.loc[group["val_accuracy"].idxmax()]["n"] return pd.Series({'best': best* 100, 'max': group['tst_accuracy'].max()* 100, 'avg': group['tst_accuracy'].mean()* 100}) def pivot_to_lang(df): - df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]*)/models') - df['n'] = df['name'].str.extract(r'models/[^/]*/([^/]*).m') + df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]{1,12})[^/]*/models') best = df.groupby('ds').apply(extract_agg) best = best.round(2) return best.T diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 965df41..b0c6a5f 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -118,6 +118,42 @@ class CLSHyperParams(LMHyperParams): results = {'val_loss': val_res[0], 'val_accuracy': float(val_res[1]), 'tst_loss':tst_res[0], 'tst_accuracy': float(tst_res[1]) } return results + def generate_pseudo_labels(self, dest_folder, save_name='cls_best', bs=40, data_cls=None, learn=None, label_smoothing_eps=0.0): + if data_cls is None: + data_cls, _, _ = self.load_cls_data(bs) + if learn is None: + learn = self.create_cls_learner(data_cls, drop_mult=0.3, label_smoothing_eps=label_smoothing_eps) + learn.unfreeze() + learn.load(save_name) + + def make_data_set(ds_type, name): + probs, lbls = learn.get_preds(ds_type=ds_type, ordered=True) + preds = torch.argmax(probs, 1) + preds = to_np(preds) + fn = self.dataset_path / f"{self.lang}.{name}.csv" + if fn.exists(): + df = pd.read_csv(fn, header=None) + df = df.iloc[(len(df) - len(preds)):] # account for the training files where first 10% elements were taken as validation + else: + df = pd.read_csv(self.dataset_path / f"{self.lang}.dev.csv", header=None) + df = df.iloc[:len(preds)] # if using training only get first n for validatation + + accuracy = (df[0] == preds).sum() / len(preds) + print(f"Generating {name} dataset of size {len(preds)}, the accuracy is {accuracy}") + df['preds'] = preds + print(df.head()) + del df['preds'] + assert accuracy > 0.7, f"Accuracy is smaller than 0.7 {accuracy}" + + df[0] = preds + dest_folder.mkdir(parents=True, exist_ok=True) + df.to_csv(dest_folder / f"{self.lang}.{name}.csv", index=None, header=None) + + make_data_set(DatasetType.Train, "train") + make_data_set(DatasetType.Valid, "dev") + shutil.copy(self.dataset_path / f"{self.lang}.test.csv", dest_folder) + shutil.copy(self.dataset_path / f"{self.lang}.unsup.csv", dest_folder) + def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn)