From e6407fe0c914a9626a012a3c863a4dd78875b457 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Thu, 10 Jan 2019 00:45:22 +0100 Subject: [PATCH 1/9] Added the param and model type for BwdLM --- ulmfit/pretrain_lm.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index c9d3b47..3f70319 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -56,6 +56,7 @@ class LMHyperParams: dataset_path: str # data_dir base_lm_path: str = None + backwards: str = False bidir: bool =False qrnn: bool = True max_vocab: int = 60000 @@ -77,6 +78,8 @@ class LMHyperParams: cuda_id: InitVar[int] = 0 def __post_init__(self, cuda_id): + if self.bidir and self.backwards: + raise ValueError('Both "backwards" and "bidir" options cannot be enabled at the same time') if not torch.cuda.is_available(): print('CUDA not available. Setting device=-1.') cuda_id = -1 @@ -101,7 +104,16 @@ class LMHyperParams: def tokenizer_prefix(self): return f"{self.tokenizer.value}{self.max_vocab // 1000}k" @property - def model_prefix(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm') + def model_direction(self): + if self.bidir: + return 'bi' + if self.backwards: + return 'bwd' + else: + return '' + + @property + def model_prefix(self): return self.model_direction + ('qrnn' if self.qrnn else 'lstm') @property def model_name(self): return f"{self.model_prefix}_{self.name}.m" @@ -111,7 +123,12 @@ class LMHyperParams: @property def lm_type(self): - return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM + if self.bidir: + return contrib_data.LanguageModelType.BiLM + if self.backwards: + return contrib_data.LanguageModelType.BwdLM + else: + return contrib_data.LanguageModelType.FwdLM def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses): tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer From 593c5661bf0347f8aa65cf6286c5101c8eb7fe19 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:26:16 +0100 Subject: [PATCH 2/9] Added alpha and beta params for RNNTrainer --- ulmfit/pretrain_lm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index c9d3b47..90636a8 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -71,6 +71,9 @@ class LMHyperParams: dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 + # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner() + rnn_alpha: float = 2 # activation regularization (AR) + rnn_beta: float = 1 # temporal activation regularization (TAR) lang: str = 'en' name: str = None @@ -183,7 +186,8 @@ class LMHyperParams: trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt, pretrained_fnames=self.pretrained_fnames, - pretrained_model=self.pretrained_model) + pretrained_model=self.pretrained_model, + alpha=self.rnn_alpha, beta=self.rnn_beta) trn_args.update(kwargs) print ("Training args: ", trn_args, "dps: ", dps or self.dps) learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, From a1e66c39d4b1e27db776fa934b737f4b1c75062a Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:27:29 +0100 Subject: [PATCH 3/9] tokenzier->tokenizer typo --- ulmfit/pretrain_lm.py | 4 ++-- ulmfit/train_clas.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 90636a8..a4b68e9 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -116,7 +116,7 @@ class LMHyperParams: def lm_type(self): return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses): + def tokenizer_to_fastai_args(self, trn_data_loading_func, add_moses): tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer if self.tokenizer is Tokenizers.SUBWORD: if self.base_lm_path: # ensure we are using the same sentence piece model @@ -211,7 +211,7 @@ class LMHyperParams: for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' - args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) + args = self.tokenizer_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) try: data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 0f9221f..825d1f0 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -125,7 +125,7 @@ class CLSHyperParams(LMHyperParams): trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] cls_cache = '.' - args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) + args = self.tokenizer_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) try: if force: raise FileNotFoundError("Forcing reloading of caches") From b00410e0cb91a152506ebfc9eedbae007634c4f6 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:28:39 +0100 Subject: [PATCH 4/9] LM save with_opt fix --- ulmfit/pretrain_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index a4b68e9..7909e3b 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -172,7 +172,7 @@ class LMHyperParams: learn.unfreeze() if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7) else: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values - learn.save("lm_best_with_opt", with_opt=False) + learn.save("lm_best_with_opt", with_opt=True) learn.save_encoder(ENC_BEST) learn.save(LM_BEST, with_opt=False) print(learn.path) From 5ba8ae4c59a72813ce4338c983cd087cacb25b02 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:29:55 +0100 Subject: [PATCH 5/9] non-ascii char removed from code. Caused display bugs --- ulmfit/postprocess_wikitext.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ulmfit/postprocess_wikitext.py b/ulmfit/postprocess_wikitext.py index 7ae8774..481631c 100644 --- a/ulmfit/postprocess_wikitext.py +++ b/ulmfit/postprocess_wikitext.py @@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab): tokens = [''] + tokens line = ' '.join(tokens) f_out.write(line) - print(f'{unk_path.name}. # of tokens: {total_num_tokens}') + print(f'{unk_path.name}. # of tokens: {total_num_tokens}') temp_file_path.replace(unk_path) @@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang): unk_path = dest_path / f'{lang}.wiki.{split}.tokens' limit_vocab(unk_path, vocab) + if __name__ == '__main__': - fire.Fire(postprocess_wikitext) \ No newline at end of file + fire.Fire(postprocess_wikitext) From dd296f30883b09c566239e7bf63a8c46069fa123 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:30:35 +0100 Subject: [PATCH 6/9] prepare_wiki.sh made executable --- prepare_wiki.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 prepare_wiki.sh diff --git a/prepare_wiki.sh b/prepare_wiki.sh old mode 100644 new mode 100755 From 0e864fb29140a765ee20bbe7ea8afaec9df25d88 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Tue, 22 Jan 2019 20:10:38 +0100 Subject: [PATCH 7/9] test_end_to_end now working, added instructions to README, sentencepiece dependency --- README.md | 10 ++++++++++ prepare_imdb.sh | 0 requirements.txt | 3 ++- tests/test_end_to_end.py | 3 ++- 4 files changed, 14 insertions(+), 2 deletions(-) mode change 100644 => 100755 prepare_imdb.sh diff --git a/README.md b/README.md index a04fca8..41cd8b6 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,13 @@ $ git push --set-upstream n-waves ulmfit_multilingual # to automatically push u - `bilm` -- scripts to train biLM ELMo style, Bert style - `class` -- scripts to test classifiers on multiple languages - `xnli` -- scripts to test nli + + +## Running tests + +To run the tests, the following data is necessary: + +- wikitext-2 (prepared by `./prepare_wiki-en.sh`, along with wikitext-103) +- imdb (prepared by `./prepare_imdb.sh`) + +then simply run tests, e.g. `pytest .` diff --git a/prepare_imdb.sh b/prepare_imdb.sh old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt index eb003e9..b211a96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ fire>=0.1.3 cupy>=5.0.0 scikit-learn>=0.20 -sacremoses>=0.0.5 \ No newline at end of file +sacremoses>=0.0.5 +sentencepiece diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 3d4d112..266bc3f 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -25,7 +25,8 @@ def get_test_data(): imdb = data / "imdb" test_data = data / "test" - shutil.rmtree(test_data) + if test_data.exists(): + shutil.rmtree(test_data) test_wt = test_data / 'wikitext-s' test_imdb = test_data / 'imdb' From 386fc49431162993c1e0aade2856cf6ed2ff1321 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Tue, 22 Jan 2019 20:31:30 +0100 Subject: [PATCH 8/9] Adapted test_text_data to batch dimension being the first, and only xxbos token at the start --- experiments/cls_test_wt103_1_f.ipynb | 6 +++--- tests/test_text_data.py | 23 ++++++++++------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/experiments/cls_test_wt103_1_f.ipynb b/experiments/cls_test_wt103_1_f.ipynb index 26a6e2e..ee3fd22 100644 --- a/experiments/cls_test_wt103_1_f.ipynb +++ b/experiments/cls_test_wt103_1_f.ipynb @@ -714,9 +714,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:fastaiv1]", + "display_name": "fastai-dev", "language": "python", - "name": "conda-env-fastaiv1-py" + "name": "fastai-dev" }, "language_info": { "codemirror_mode": { @@ -728,7 +728,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.0" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/tests/test_text_data.py b/tests/test_text_data.py index 6b3dabf..1b96079 100644 --- a/tests/test_text_data.py +++ b/tests/test_text_data.py @@ -22,38 +22,35 @@ def test_should_load_backwards_lm(): df = text_df(['neg','pos']) data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BwdLM, - ld_cls=contrib_data.LanguageModelLoader) + lm_type=contrib_data.LanguageModelType.BwdLM) lml = data.train_dl.dl lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) batch = lml.get_batch(lml.data, 0, 70) - assert batch[0].shape == (70, lml.bs) + assert batch[0].shape == (lml.bs, 70) assert batch[1].shape == (70*lml.bs,) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',]) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]] + np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) def test_should_load_bi_lm(): path = untar_data(URLs.IMDB_SAMPLE) df = text_df(['neg', 'pos']) data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BiLM, - ld_cls=contrib_data.LanguageModelLoader) + lm_type=contrib_data.LanguageModelType.BiLM) lml = data.train_dl.dl lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) batch = lml.get_batch(lml.data, 0, 70) - assert batch[0].shape == (70, lml.bs, 2) + assert batch[0].shape == (lml.bs, 70, 2) assert batch[1].shape == (70*lml.bs, 2) - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:, 0, 0]] - np.testing.assert_array_equal(as_text[:7], "xxfld 1 fast ai is a cool".split()) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]] + np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split()) - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0,1]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',]) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]] + np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) ###################### NEW CODE From c6cf44a7b9154b54e46eabe677d09ed03c0e12df Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Wed, 23 Jan 2019 00:05:38 +0100 Subject: [PATCH 9/9] Fixed test_bilm_classifier_loads_encoder - inconsistent settings, problematic input (too short) --- tests/test_text_train.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_text_train.py b/tests/test_text_train.py index 0103939..7c3d894 100644 --- a/tests/test_text_train.py +++ b/tests/test_text_train.py @@ -42,7 +42,7 @@ def learn(): def text_df(n_labels): data = [] - texts = ["fast ai is a cool project", "hello world"] + texts = ["fast ai is a cool project", "hello world"] * 20 for ind, text in enumerate(texts): sample = {} for label in range(n_labels): sample[label] = ind%2 @@ -58,19 +58,21 @@ def test_val_loss(learn): def test_bilm_classifier_loads_encoder(): - n_labels=2 + n_labels=1 + nl = 1 + emb_sz = 100 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp') os.makedirs(path) try: - df = text_df(n_labels=1) + df = text_df(n_labels=n_labels) lmdf = df#[["text"]] print(lmdf.head()) lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer), lm_type=contrib_data.LanguageModelType.BiLM) - learn = bilm_learner(lmdata, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) learn.save_encoder("enc") - data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"]) - classifier = bilm_text_classifier_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8) + classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) print(last_layer(classifier.model), ) classifier.load_encoder("enc") classifier.fit(1)