From 27a720b1056aed0393d99980a28445a5d60786db Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 1 May 2019 20:57:20 +0200 Subject: [PATCH] Make decoder bias optional These changes were made during poleval'19 competition to try to reproduce results from fastai v0.7 in v1.0. --- fastai_contrib/learner.py | 13 ++++++++++--- fastai_contrib/models.py | 4 ++-- fastai_contrib/utils.py | 4 ++-- ulmfit/pretrain_lm.py | 13 ++++++++----- ulmfit/train_clas.py | 13 +++++++++++-- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index 67e79cb..de5d26a 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -86,7 +86,13 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new: if 'model' in wgts: wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix) else: - dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight'] + #dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight'] + has_bias = prefix+'1.decoder.bias' in wgts + enc_wgts = wgts[prefix+'0.encoder.weight'] + if has_bias: + dec_bias = wgts[prefix+'1.decoder.bias'] + else: + dec_bias = enc_wgts.new_zeros((len(stoi_wgts),)) bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0) new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_() new_b = dec_bias.new_zeros((len(itos_new),)).zero_() @@ -101,7 +107,8 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new: wgts[prefix+'0.encoder.weight'] = new_w wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone() wgts[prefix+'1.decoder.weight'] = new_w.clone() - wgts[prefix+'1.decoder.bias'] = new_b + if has_bias: + wgts[prefix+'1.decoder.bias'] = new_b return wgts #endregion @@ -110,4 +117,4 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new: import fastai.text.learner fastai.text.learner.convert_weights = convert_weights -#endregion \ No newline at end of file +#endregion diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index 92ac7e4..8b5afdb 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -1,7 +1,7 @@ from fastai.torch_core import * from fastai.layers import * from fastai.text.models import * - +from fastai.text.learner import * #region New code class BiLMModel(nn.Module): @@ -163,4 +163,4 @@ def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_s model.reset() return model -#endregion \ No newline at end of file +#endregion diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index ebd9674..f527596 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -80,7 +80,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules post_rules = post_rules if post_rules is not None else defaults.text_post_rules - special_cases = defaults.text_spec_tok + special_cases = defaults.text_spec_tok + ['', '', '', '', ''] if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'): # load the text from the train tokens file text = load_text() @@ -100,7 +100,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, sp_params = [ f"--input={raw_text_path}", f"--character_coverage={char_coverage}", - f"--unk_id={len(defaults.text_spec_tok)}", + f"--unk_id={len(special_cases)}", f"--pad_id=-1", f"--bos_id=-1", f"--eos_id=-1", diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 2a6f59f..e7f8c5c 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -39,7 +39,9 @@ def istitle(line): return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0 def read_wiki_articles(filename): + return pd.read_csv(filename, sep="\t", header=None, names=["texts"]) articles = [] + with open(filename, encoding='utf8') as f: lines = f.readlines() current_article = [] @@ -70,7 +72,8 @@ class LMHyperParams: # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) # for training on smaller datasets, more dropout is necessary - dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train + # buggy dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train + dps = dict(input_p=0.25, output_p=0.1, weight_p=0.2, embed_p=0.02, hidden_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner() @@ -171,10 +174,10 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0): + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True): self.model_dir.mkdir(exist_ok=True, parents=True) data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm - learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps) + learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps, out_bias=out_bias) print("Bptt", data_lm.bptt) learn.true_wd = true_wd if num_epochs > 0: @@ -204,10 +207,10 @@ class LMHyperParams: # do we need to return `learn'? it adds noise to Fire output #return learn - def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs): + def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, out_bias=True, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, - tie_weights=True, out_bias=True) + tie_weights=True, out_bias=out_bias) config.update(dps or self.dps) trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta) trn_args.update(kwargs) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index da66aac..4396136 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -12,6 +12,12 @@ import fire from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST +from sklearn.metrics import f1_score as f1s, precision_score, recall_score + +def f1_score(preds, targs): + preds = torch.max(preds, dim=1)[1].cpu().numpy() + targs = targs.cpu().numpy() + return torch.tensor(f1s(targs, preds)) class CLSHyperParams(LMHyperParams): # dir_path -> data/imdb/ @@ -107,9 +113,12 @@ class CLSHyperParams(LMHyperParams): if data_tst is None: _, _, data_tst = self.load_cls_data(bs) if learn is None: - learn = self.create_cls_learner(data_tst, drop_mult=0.3) + learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[f1_score, accuracy]) learn.unfreeze() learn.load(save_name) + print(data_tst.one_batch()[1]) + print(data_tst.one_batch()[1]) + print(data_tst.one_batch()[1]) results = learn.validate(data_tst.valid_dl) print(f"Loss and accuracy using ({save_name}):", results) return list(map(float, results)) @@ -141,7 +150,7 @@ class CLSHyperParams(LMHyperParams): self.model_dir.mkdir(exist_ok=True, parents=True) add_trn_to_lm = True lang = self.lang - use_moses = True + use_moses = False #True if 'xnli' in str(self.dataset_dir): NotImplementedError("Support for Xnli is not implemented yet") if 'imdb' in self.dataset_dir.name: