From 6aff0a9e22baeea87ecd000e9e4ac3818cf41ed7 Mon Sep 17 00:00:00 2001 From: Marcin Date: Sun, 12 May 2019 21:29:32 +0200 Subject: [PATCH] Add more seeds --- split-cls.py | 18 +++++++++++ ulmfit/pretrain_lm.py | 34 +++++++++++++------- ulmfit/train_clas.py | 74 +++++++++++++++++++++++++++++-------------- 3 files changed, 92 insertions(+), 34 deletions(-) create mode 100644 split-cls.py diff --git a/split-cls.py b/split-cls.py new file mode 100644 index 0000000..c89c428 --- /dev/null +++ b/split-cls.py @@ -0,0 +1,18 @@ +import pandas as pd, numpy as np +import fire +from pathlib import Path +from sys import stderr +from sklearn.model_selection import train_test_split + +def to_csv(df, path): + df.to_csv(path, header=None, index=None) + +def split(data_dir): + data_dir = Path(data_dir) + train = pd.read_csv(data_dir / "pl.unsup.csv", header=None) + trn, val = train_test_split(train, test_size=0.1, random_state=12345, stratify=train[0]) + + to_csv(trn, data_dir / "pl.train.csv") + to_csv(val, data_dir / "pl.dev.csv") + +if __name__ == "__main__": fire.Fire(split) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 00f3bc6..b6f8458 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -71,6 +71,9 @@ class LMHyperParams: emb_sz:int = 400 nh: int = None nl: int = 3 + out_bias: bool = True + + lmseed: int = None # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) # for training on smaller datasets, more dropout is necessary @@ -123,7 +126,10 @@ class LMHyperParams: def model_prefix(self): return self.model_direction + ('qrnn' if self.qrnn else 'lstm') @property - def model_name(self): return f"{self.model_prefix}_{self.name}.m" + def model_name(self): return f"{self.model_prefix}_{self.name}{self.model_suffix}.m" + + @property + def model_suffix(self): return '' if self.lmseed is None else f'_lmseed-{self.lmseed}' @property def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None @@ -167,6 +173,14 @@ class LMHyperParams: f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") return args + def set_seed(self, seed, name): + if seed is not None: + print(f"Setting {name} seed to {seed}") + torch.manual_seed(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(seed) + def save_info(self): from dataclasses import asdict vals = {k: (str(v) if isinstance(v, Path) else v) for k,v in asdict(self).items()} @@ -176,17 +190,15 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True, seed=None): - if seed is not None: - print(f"Setting seed to {seed}") - torch.manual_seed(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - np.random.seed(seed) + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0): + if self.ftseed is None: + self.set_seed(self.lmseed, "LM") + else: + self.set_seed(self.ftseed, "fine-tune") self.model_dir.mkdir(exist_ok=True, parents=True) data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm - learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps, out_bias=out_bias) + learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps) print("Bptt", data_lm.bptt) learn.true_wd = true_wd if num_epochs > 0: @@ -216,10 +228,10 @@ class LMHyperParams: # do we need to return `learn'? it adds noise to Fire output #return learn - def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, out_bias=True, **kwargs): + def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, - tie_weights=True, out_bias=out_bias) + tie_weights=True, out_bias=self.out_bias) config.update(dps or self.dps) trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta) trn_args.update(kwargs) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 1db1106..c20a1bb 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -14,14 +14,13 @@ from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST from sklearn.metrics import f1_score as f1s, precision_score, recall_score -def f1_score(preds, targs): - preds = torch.max(preds, dim=1)[1].cpu().numpy() - targs = targs.cpu().numpy() - return torch.tensor(f1s(targs, preds)) - +@dataclass class CLSHyperParams(LMHyperParams): # dir_path -> data/imdb/ - use_test_for_validation=False + use_test_for_validation: bool=False + ftseed: int = None + clsweightseed: int = None + clstrainseed: int = None bicls_head:str = 'BiPoolingLinearClassifier' @@ -29,6 +28,17 @@ class CLSHyperParams(LMHyperParams): super().__post_init__(*args, **kwargs) self.dataset_dir=self.dataset_path + @property + def model_suffix(self): + s1 = '' if self.lmseed is None else f'lmseed-{self.lmseed}' + s2 = '' if self.ftseed is None else f'ftseed-{self.ftseed}' + s3 = '' if self.clsweightseed is None else f'clsweightseed-{self.clsweightseed}' + s4 = '' if self.clstrainseed is None else f'clstrainseed-{self.clstrainseed}' + s = '-'.join([x for x in [s1, s2, s3, s4] if x != '']) + if s != '': + return '_'+s + return '' + @property def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists() @@ -69,9 +79,28 @@ class CLSHyperParams(LMHyperParams): if num_cls_epochs > 5: learn.fit_one_cycle(num_cls_epochs-4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) + def get_metrics(self): + f1_score = FBeta(beta=1.0) + precision = Precision() + recall = Recall() + metrics = [f1_score, precision, recall] + + # TODO: fix this in fast.ai + for metric in metrics: metric.on_train_begin() + metrics.append(accuracy) + return metrics + + def output_metrics(self, results): + print(f"F1 score bin: {results[1].item()}") + print(f"Loss: {results[0]}") + print(f"Precision: {results[2].item()}") + print(f"Recall: {results[3].item()}") + print(f"Accuracy: {results[4].item()}") + + def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered', - label_smoothing_eps=0.0, random_init=False, seed=None, dump_preds=None): + label_smoothing_eps=0.0, random_init=False, dump_preds=None): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" self.model_dir.mkdir(exist_ok=True, parents=True) @@ -82,13 +111,16 @@ class CLSHyperParams(LMHyperParams): if self.need_fine_tune_lm and not random_init: if not (self.model_dir/(ENC_BEST+".pth")).exists(): - self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps, seed=seed) + self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) else: print("Language model already exist, skipping finetuning") loss_func = CrossEntropyFlat(weight=torch.FloatTensor([0.5,30]).cuda()) + + self.set_seed(self.clsweightseed, "classifier weights") + learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps, random_init=random_init, - metrics=[FBeta(beta=1.0), f1_score, accuracy], + metrics=self.get_metrics(), loss_func=loss_func) if not random_init: @@ -100,12 +132,8 @@ class CLSHyperParams(LMHyperParams): else: print("Starting classifier from random weights") - if seed is not None: - print(f"Setting seed to {seed}") - torch.manual_seed(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - np.random.seed(seed) + + self.set_seed(self.clstrainseed, "classifier train") if hasattr(self, 'lr_schedule_'+lr_sched): learn.true_wd = True @@ -119,13 +147,11 @@ class CLSHyperParams(LMHyperParams): del learn return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None) - def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None, dump_preds=None): + def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None, dump_preds=None, mode="test"): if data_tst is None: - _, _, data_tst = self.load_cls_data(bs) + data_clas , _, data_tst = self.load_cls_data(bs) if learn is None: - fbeta = FBeta(beta=1.0) - fbeta.on_train_begin() - learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[fbeta, f1_score, accuracy]) + learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=self.get_metrics()) learn.unfreeze() learn.load(save_name) probs, targets = learn.get_preds(ordered=True) @@ -133,9 +159,11 @@ class CLSHyperParams(LMHyperParams): if dump_preds: with open(dump_preds, 'w') as f: f.write('\n'.join([str(x) for x in preds])) - results = learn.validate(data_tst.valid_dl) - print(f"F1 score bin: {results[1].item()}") - print(f"Loss, f1_score, almost f1_score and accuracy using ({save_name}):", results) + results = learn.validate(data_tst.valid_dl if mode == "test" else data_clas.valid_dl) + print(f"Model: {self.name}") + print(f"Validation on: {mode}") + self.output_metrics(results) + return list(map(float, results)) def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs):