From 837925ff5307da970373bc974124797512381a51 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sun, 3 Mar 2019 13:29:49 +0100 Subject: [PATCH] Imporved validate_cls & eval to pick the best model based on val accuracy --- ulmfit/__main__.py | 66 ++++++++++++++++++++++++++++++++++--------- ulmfit/pretrain_lm.py | 14 +++++++-- ulmfit/train_clas.py | 20 +++++++------ 3 files changed, 75 insertions(+), 25 deletions(-) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 38b7f76..cec29f7 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -7,7 +7,7 @@ from collections import OrderedDict from functools import wraps import pandas as pd import fire -from .pretrain_lm import LMHyperParams +from .pretrain_lm import LMHyperParams, json_save, json_load, np from .train_clas import CLSHyperParams from pathlib import Path from string import Template @@ -40,8 +40,11 @@ class ULMFiT: lm2 = LMHyperParams @wraps(CLSHyperParams) - def cls(self, dataset_path, base_lm_path, **changes): - params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes) + def cls(self, dataset_path, base_lm_path=None, **changes): + if base_lm_path is not None: + params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes) + else: + params = CLSHyperParams(dataset_path=dataset_path, **changes) return FireView(train=params.train_cls, validate_cls=params.validate_cls) @wraps(CLSHyperParams) @@ -63,8 +66,9 @@ class ULMFiT: bs=bs, lr_sched=lr_sched, label_smoothing_eps=label_smoothing_eps, + return_df=True, **kwargs) - val = next(iter(d.values()), -1) + val = d['tst_accuracy'][0] results.append((noise/100, val)) df = pd.DataFrame(results, columns=["noise", "accuracy"]) df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv") @@ -84,27 +88,61 @@ class ULMFiT: print("Adding", f, dest) tar.add(f, dest) - def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params): - results = OrderedDict() + def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${ds_name}', name=None, + num_lm_epochs=0, cuda_id=0, train=True, to_csv=None, return_df=False, label_smoothing_eps=0.0, + **trn_params): + results = [] + + + def extract_agg(group): + best = group.loc[group["val_accuracy"].idxmax()]["tst_accuracy"] + best_name = group.loc[group["val_accuracy"].idxmax()]["n"] + return pd.Series({'best': best* 100, + 'max': group['tst_accuracy'].max()* 100, + 'avg': group['tst_accuracy'].mean()* 100}) + def pivot_to_lang(df): + df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]*)/models') + df['n'] = df['name'].str.extract(r'models/[^/]*/([^/]*).m') + best = df.groupby('ds').apply(extract_agg) + best = best.round(2) + return best.T for base_model in sorted(Path("data").glob(glob)): print("Processing", base_model) for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)): try: - params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) + _name = name + if name is None: + _name = base_model.name.replace(".m","").replace("lstm_","").replace("qrnn_","") + params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, cuda_id=cuda_id) key = str(params.model_dir.relative_to(Path.cwd())) - if (params.model_dir/"cls_best.pth").exists(): + if (params.model_dir / "results.npy").exists(): + d = np.load(params.model_dir / "results.npy") + d = d.tolist() # magiacally convert to dict + elif (params.model_dir/"cls_best.pth").exists(): print("Evaluating previously trained model") - results[key] = params.validate_cls()[1] - else: + d = params.validate_cls(label_smoothing_eps=label_smoothing_eps) + elif train: print("Training") - results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1] + d = params.train_cls(num_lm_epochs=num_lm_epochs, label_smoothing_eps=label_smoothing_eps, **trn_params) + else: + print("Skipping", (params.model_dir/"cls_best.pth")) + d = None + if d is not None: + d['name']=key + np.save(params.model_dir / "results.npy", d) + results.append(d) del params except Exception as e: print("Error", e) gc.collect() - - pprint.pprint(results) - return results + df = pd.DataFrame.from_records(results) + print(df) + print(pivot_to_lang(df)) + if to_csv is not None: + print(f"Saving result to: {to_csv}") + df.to_csv(to_csv) + if return_df: + return df def remove_lm_saves(self): for lm_save in Path("data").glob("**/lm_*.pth"): diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 2a6f59f..94efd8a 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -52,6 +52,14 @@ def read_wiki_articles(filename): print(f"Wiki text was split to {len(articles)} articles") return pd.DataFrame({'texts': np.array(articles, dtype=np.object)}) +def json_save(f, d): + with Path(f).open("w") as fp: + json.dump(d, fp) + +def json_load(f): + with open(f, 'r') as f: + return json.load(f) + @dataclass class LMHyperParams: dataset_path: str # data_dir @@ -168,7 +176,7 @@ class LMHyperParams: vals.pop('name', None) vals.pop('lang', None) vals['tokenizer'] = self.tokenizer.value - with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) + json_save(self.model_dir/'info.json', vals) print("Saving info", self.model_dir / 'info.json') def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0): @@ -249,7 +257,7 @@ class LMHyperParams: args = self.tokenizer_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False) - data_lm = self.lm_databunch("lm", + data_lm = self.lm_databunch(f"lm{self.bptt if self.bptt != 70 else ''}", train_df=read_wiki_articles(trn_path), valid_df=read_wiki_articles(val_path), classes=None, @@ -302,7 +310,7 @@ class LMHyperParams: def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams': dataset_path = Path(dataset_path).resolve() base_lm_path = Path(base_lm_path).resolve() - with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) + d = json_load(base_lm_path/'info.json') d['dataset_path'] = dataset_path d['base_lm_path'] = base_lm_path d.pop('bs', None) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index b989989..965df41 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -10,7 +10,7 @@ from fastai_contrib.utils import PAD_TOKEN_ID import fire -from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST +from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST, json_save class CLSHyperParams(LMHyperParams): @@ -101,18 +101,22 @@ class CLSHyperParams(LMHyperParams): learn.save('cls_last', with_opt=False) learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being del learn - return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None) + return self.validate_cls('cls_best', bs=bs, data_cls=data_clas, data_tst=data_tst, learn=None) - def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None): + def validate_cls(self, save_name='cls_best', bs=40, data_cls=None, data_tst=None, learn=None, label_smoothing_eps=0.0): if data_tst is None: - _, _, data_tst = self.load_cls_data(bs) + data_cls, _, data_tst = self.load_cls_data(bs) if learn is None: - learn = self.create_cls_learner(data_tst, drop_mult=0.3) + learn = self.create_cls_learner(data_tst, drop_mult=0.3, label_smoothing_eps=label_smoothing_eps) learn.unfreeze() learn.load(save_name) - results = learn.validate(data_tst.valid_dl) - print(f"Loss and accuracy using ({save_name}):", results) - return list(map(float, results)) + val_res=[-1, -1] + if data_cls: + val_res = learn.validate(data_cls.valid_dl) + tst_res = learn.validate(data_tst.valid_dl) + print(f"Loss and accuracy using ({save_name}):", tst_res, val_res) + results = {'val_loss': val_res[0], 'val_accuracy': float(val_res[1]), 'tst_loss':tst_res[0], 'tst_accuracy': float(tst_res[1]) } + return results def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported"