From 6dc4a5f10280d51eae0189e231306d586673f63e Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 14 May 2019 11:53:16 +0200 Subject: [PATCH] Add Kappa and Mathew score calcualtion + ls command to main --- ulmfit/__main__.py | 76 ++++++++++++++++++++++++-------------------- ulmfit/train_clas.py | 39 +++++++++++++++++------ 2 files changed, 71 insertions(+), 44 deletions(-) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 93e8b8a..6bf1d48 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -30,7 +30,6 @@ def get_dataset_path(p, dataset_template): ds = [x for x in p.parents if x.name == "models"][0].parent lang = get_lang_from_dataset_path(ds) pattern = Template(dataset_template).substitute(lang=lang, ds_name=ds.name) - print(f"Searching for {pattern}, {ds.parent}") for ds_path in ds.parent.glob(pattern): yield lang, ds_path @@ -140,6 +139,17 @@ class ULMFiT: lr_sched=lr_sched, **kwargs) + def ls(self, glob, dataset_template='${ds_name}'): + data_dir = Path("data").absolute() + glob = str(glob) + if "data" not in glob and not glob.startswith("/"): + glob = "data/" + glob + results = [] + for base_model in sorted(data_dir.parent.glob(glob)): + for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)): + results.append((base_model, lang, dataset_path)) + return results + def eval(self, glob="data/mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${ds_name}', name=None, num_lm_epochs=0, train=True, to_csv=None, return_df=False, label_smoothing_eps=0.0, lmseed=None, ftseed=None, clsweightseed=None, clstrainseed=None, save_name="cls_best", @@ -159,39 +169,37 @@ class ULMFiT: glob=str(glob) if "data" not in glob and not glob.startswith("/"): glob = "data/"+glob - for base_model in sorted(data_dir.parent.glob(glob)): - print("Processing", base_model) - for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)): - try: - _name = name - if name is None: - _name = folder_name_to_model_name(base_model.name) - params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, **model_args) - last_model_dir = params.model_dir.relative_to(data_dir.parent) - if (params.model_dir/"cls_best.pth").exists(): - print("Evaluating previously trained model") - d_tst = params.validate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test") - d_val = params.validate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid") - d={} - d.update(d_val) - d.update(d_tst) - elif train: - print("Training") - d = params.train_cls(num_lm_epochs=num_lm_epochs, label_smoothing_eps=label_smoothing_eps, **trn_params) - else: - print("Skipping", (params.model_dir/"cls_best.pth")) - d = None - if d is not None: - d['model_dir_parent'] = params.model_dir.relative_to(data_dir.parent).parent - d['model_name'] = params.model_name - np.save(params.model_dir / "results.npy", d) - results.append(d) - del params - except Exception as e: - print("Error", e) - if not skip_on_error: - raise e - gc.collect() + for base_model, lang, dataset_path in self.ls(glob, dataset_template): + try: + _name = name + if name is None: + _name = folder_name_to_model_name(base_model.name) + params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, **model_args) + last_model_dir = params.model_dir.relative_to(data_dir.parent) + if (params.model_dir/"cls_best.pth").exists(): + print("Evaluating previously trained model") + d_tst = params.validate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test") + d_val = params.validate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid") + d={} + d.update(d_val) + d.update(d_tst) + elif train: + print("Training") + d = params.train_cls(num_lm_epochs=num_lm_epochs, label_smoothing_eps=label_smoothing_eps, **trn_params) + else: + print("Skipping", (params.model_dir/"cls_best.pth")) + d = None + if d is not None: + d['model_dir_parent'] = params.model_dir.relative_to(data_dir.parent).parent + d['model_name'] = params.model_name + np.save(params.model_dir / "results.npy", d) + results.append(d) + del params + except Exception as e: + print("Error", e) + if not skip_on_error: + raise e + gc.collect() df = pd.DataFrame.from_records(results) print(df) if to_csv is not None: diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 9ee3dd5..58f5a64 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -66,6 +66,19 @@ class CLSHyperParams(LMHyperParams): learn.unfreeze() learn.fit_one_cycle(num_cls_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) + def lr_schedule_reverse_2cycle(self, learn, num_cls_epochs): + print("Reverse 2cycle ") + learn.unfreeze() + for g in learn.layer_groups[-1:]: + for l in g: + if not learn.train_bn or not isinstance(l, bn_types): requires_grad(l, False) + learn.create_opt(defaults.lr) + print("training LM") + learn.fit_one_cycle(num_cls_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) + learn.unfreeze() + print("training ALL") + learn.fit_one_cycle(num_cls_epochs, slice(1e-3 / (2.6 ** 4), 2e-3), moms=(0.8, 0.7)) + def lr_schedule_false_wd(self, learn, num_cls_epochs): learn.true_wd = False print("Starting classifier training") @@ -83,8 +96,9 @@ class CLSHyperParams(LMHyperParams): f1_score = FBeta(beta=1.0) precision = Precision() recall = Recall() - metrics = [f1_score, precision, recall] - + kappa_lin = KappaScore('linear') + matthews_correff = MatthewsCorreff() + metrics = [f1_score, precision, recall, kappa_lin, matthews_correff] # TODO: fix this in fast.ai if init: for metric in metrics: metric.on_train_begin() @@ -96,17 +110,19 @@ class CLSHyperParams(LMHyperParams): print(f"Loss: {results[0]}") print(f"Precision: {results[2].item()}") print(f"Recall: {results[3].item()}") - print(f"Accuracy: {results[4].item()}") + print(f"Accuracy: {results[6].item()}") d = {f"{mode} F1 score bin": results[1].item(), f"{mode} Loss": results[0], f"{mode} Precision": results[2].item(), f"{mode} Recall": results[3].item(), - f"{mode} Accuracy": results[4].item()} + f"{mode} Kappa Linear": results[4].item(), + f"{mode} Matthews Correff": results[5].item(), + f"{mode} Accuracy": results[6].item()} return {k:float(str(v)) for k,v in d.items()} # float(str(x)) to avoid float32 -> float64 conversion isssues def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered', - label_smoothing_eps=0.0, random_init=False, dump_preds=None): + label_smoothing_eps=0.0, random_init=False, dump_preds=None, early_stopping=True): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" self.model_dir.mkdir(exist_ok=True, parents=True) @@ -127,7 +143,7 @@ class CLSHyperParams(LMHyperParams): learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps, random_init=random_init, metrics=self.get_metrics(), - loss_func=loss_func) + loss_func=loss_func, early_stopping=early_stopping) if not random_init: try: @@ -189,7 +205,7 @@ class CLSHyperParams(LMHyperParams): return labeled_results - def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs): + def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, early_stopping=True, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn) config.update(dps or self.dps) @@ -205,9 +221,12 @@ class CLSHyperParams(LMHyperParams): learn.load_pretrained(*fnames, strict=False) learn.freeze() - learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), - partial(SaveModelCallback, every='improvement', name='cls_best_tmp', monitor="f_beta") - ] + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history")] + if early_stopping: + learn.callback_fns += [partial(SaveModelCallback, every='improvement', + name='cls_best_tmp', + monitor="f_beta")] + if label_smoothing_eps > 0.0: learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps) return learn