From 593c5661bf0347f8aa65cf6286c5101c8eb7fe19 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:26:16 +0100 Subject: [PATCH 1/5] Added alpha and beta params for RNNTrainer --- ulmfit/pretrain_lm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index c9d3b47..90636a8 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -71,6 +71,9 @@ class LMHyperParams: dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 + # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner() + rnn_alpha: float = 2 # activation regularization (AR) + rnn_beta: float = 1 # temporal activation regularization (TAR) lang: str = 'en' name: str = None @@ -183,7 +186,8 @@ class LMHyperParams: trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt, pretrained_fnames=self.pretrained_fnames, - pretrained_model=self.pretrained_model) + pretrained_model=self.pretrained_model, + alpha=self.rnn_alpha, beta=self.rnn_beta) trn_args.update(kwargs) print ("Training args: ", trn_args, "dps: ", dps or self.dps) learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, From a1e66c39d4b1e27db776fa934b737f4b1c75062a Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:27:29 +0100 Subject: [PATCH 2/5] tokenzier->tokenizer typo --- ulmfit/pretrain_lm.py | 4 ++-- ulmfit/train_clas.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 90636a8..a4b68e9 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -116,7 +116,7 @@ class LMHyperParams: def lm_type(self): return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses): + def tokenizer_to_fastai_args(self, trn_data_loading_func, add_moses): tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer if self.tokenizer is Tokenizers.SUBWORD: if self.base_lm_path: # ensure we are using the same sentence piece model @@ -211,7 +211,7 @@ class LMHyperParams: for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' - args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) + args = self.tokenizer_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) try: data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 0f9221f..825d1f0 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -125,7 +125,7 @@ class CLSHyperParams(LMHyperParams): trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] cls_cache = '.' - args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) + args = self.tokenizer_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) try: if force: raise FileNotFoundError("Forcing reloading of caches") From b00410e0cb91a152506ebfc9eedbae007634c4f6 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:28:39 +0100 Subject: [PATCH 3/5] LM save with_opt fix --- ulmfit/pretrain_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index a4b68e9..7909e3b 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -172,7 +172,7 @@ class LMHyperParams: learn.unfreeze() if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7) else: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values - learn.save("lm_best_with_opt", with_opt=False) + learn.save("lm_best_with_opt", with_opt=True) learn.save_encoder(ENC_BEST) learn.save(LM_BEST, with_opt=False) print(learn.path) From 5ba8ae4c59a72813ce4338c983cd087cacb25b02 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:29:55 +0100 Subject: [PATCH 4/5] non-ascii char removed from code. Caused display bugs --- ulmfit/postprocess_wikitext.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ulmfit/postprocess_wikitext.py b/ulmfit/postprocess_wikitext.py index 7ae8774..481631c 100644 --- a/ulmfit/postprocess_wikitext.py +++ b/ulmfit/postprocess_wikitext.py @@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab): tokens = [''] + tokens line = ' '.join(tokens) f_out.write(line) - print(f'{unk_path.name}. # of tokens: {total_num_tokens}') + print(f'{unk_path.name}. # of tokens: {total_num_tokens}') temp_file_path.replace(unk_path) @@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang): unk_path = dest_path / f'{lang}.wiki.{split}.tokens' limit_vocab(unk_path, vocab) + if __name__ == '__main__': - fire.Fire(postprocess_wikitext) \ No newline at end of file + fire.Fire(postprocess_wikitext) From dd296f30883b09c566239e7bf63a8c46069fa123 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:30:35 +0100 Subject: [PATCH 5/5] prepare_wiki.sh made executable --- prepare_wiki.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 prepare_wiki.sh diff --git a/prepare_wiki.sh b/prepare_wiki.sh old mode 100644 new mode 100755