diff --git a/README.md b/README.md index a04fca8..41cd8b6 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,13 @@ $ git push --set-upstream n-waves ulmfit_multilingual # to automatically push u - `bilm` -- scripts to train biLM ELMo style, Bert style - `class` -- scripts to test classifiers on multiple languages - `xnli` -- scripts to test nli + + +## Running tests + +To run the tests, the following data is necessary: + +- wikitext-2 (prepared by `./prepare_wiki-en.sh`, along with wikitext-103) +- imdb (prepared by `./prepare_imdb.sh`) + +then simply run tests, e.g. `pytest .` diff --git a/experiments/cls_test_wt103_1_f.ipynb b/experiments/cls_test_wt103_1_f.ipynb index 26a6e2e..ee3fd22 100644 --- a/experiments/cls_test_wt103_1_f.ipynb +++ b/experiments/cls_test_wt103_1_f.ipynb @@ -714,9 +714,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:fastaiv1]", + "display_name": "fastai-dev", "language": "python", - "name": "conda-env-fastaiv1-py" + "name": "fastai-dev" }, "language_info": { "codemirror_mode": { @@ -728,7 +728,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.0" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/prepare_imdb.sh b/prepare_imdb.sh old mode 100644 new mode 100755 diff --git a/prepare_wiki.sh b/prepare_wiki.sh old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt index eb003e9..b211a96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ fire>=0.1.3 cupy>=5.0.0 scikit-learn>=0.20 -sacremoses>=0.0.5 \ No newline at end of file +sacremoses>=0.0.5 +sentencepiece diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 3d4d112..266bc3f 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -25,7 +25,8 @@ def get_test_data(): imdb = data / "imdb" test_data = data / "test" - shutil.rmtree(test_data) + if test_data.exists(): + shutil.rmtree(test_data) test_wt = test_data / 'wikitext-s' test_imdb = test_data / 'imdb' diff --git a/tests/test_text_data.py b/tests/test_text_data.py index 6b3dabf..1b96079 100644 --- a/tests/test_text_data.py +++ b/tests/test_text_data.py @@ -22,38 +22,35 @@ def test_should_load_backwards_lm(): df = text_df(['neg','pos']) data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BwdLM, - ld_cls=contrib_data.LanguageModelLoader) + lm_type=contrib_data.LanguageModelType.BwdLM) lml = data.train_dl.dl lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) batch = lml.get_batch(lml.data, 0, 70) - assert batch[0].shape == (70, lml.bs) + assert batch[0].shape == (lml.bs, 70) assert batch[1].shape == (70*lml.bs,) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',]) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]] + np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) def test_should_load_bi_lm(): path = untar_data(URLs.IMDB_SAMPLE) df = text_df(['neg', 'pos']) data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BiLM, - ld_cls=contrib_data.LanguageModelLoader) + lm_type=contrib_data.LanguageModelType.BiLM) lml = data.train_dl.dl lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) batch = lml.get_batch(lml.data, 0, 70) - assert batch[0].shape == (70, lml.bs, 2) + assert batch[0].shape == (lml.bs, 70, 2) assert batch[1].shape == (70*lml.bs, 2) - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:, 0, 0]] - np.testing.assert_array_equal(as_text[:7], "xxfld 1 fast ai is a cool".split()) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]] + np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split()) - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0,1]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',]) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]] + np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) ###################### NEW CODE diff --git a/tests/test_text_train.py b/tests/test_text_train.py index 0103939..7c3d894 100644 --- a/tests/test_text_train.py +++ b/tests/test_text_train.py @@ -42,7 +42,7 @@ def learn(): def text_df(n_labels): data = [] - texts = ["fast ai is a cool project", "hello world"] + texts = ["fast ai is a cool project", "hello world"] * 20 for ind, text in enumerate(texts): sample = {} for label in range(n_labels): sample[label] = ind%2 @@ -58,19 +58,21 @@ def test_val_loss(learn): def test_bilm_classifier_loads_encoder(): - n_labels=2 + n_labels=1 + nl = 1 + emb_sz = 100 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp') os.makedirs(path) try: - df = text_df(n_labels=1) + df = text_df(n_labels=n_labels) lmdf = df#[["text"]] print(lmdf.head()) lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer), lm_type=contrib_data.LanguageModelType.BiLM) - learn = bilm_learner(lmdata, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) learn.save_encoder("enc") - data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"]) - classifier = bilm_text_classifier_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8) + classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) print(last_layer(classifier.model), ) classifier.load_encoder("enc") classifier.fit(1) diff --git a/ulmfit/postprocess_wikitext.py b/ulmfit/postprocess_wikitext.py index 7ae8774..481631c 100644 --- a/ulmfit/postprocess_wikitext.py +++ b/ulmfit/postprocess_wikitext.py @@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab): tokens = [''] + tokens line = ' '.join(tokens) f_out.write(line) - print(f'{unk_path.name}. # of tokens: {total_num_tokens}') + print(f'{unk_path.name}. # of tokens: {total_num_tokens}') temp_file_path.replace(unk_path) @@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang): unk_path = dest_path / f'{lang}.wiki.{split}.tokens' limit_vocab(unk_path, vocab) + if __name__ == '__main__': - fire.Fire(postprocess_wikitext) \ No newline at end of file + fire.Fire(postprocess_wikitext) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index fef99bf..3df9213 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -56,6 +56,7 @@ class LMHyperParams: dataset_path: str # data_dir base_lm_path: str = None + backwards: str = False bidir: bool =False qrnn: bool = True max_vocab: int = 60000 @@ -71,12 +72,17 @@ class LMHyperParams: dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 + # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner() + rnn_alpha: float = 2 # activation regularization (AR) + rnn_beta: float = 1 # temporal activation regularization (TAR) lang: str = 'en' name: str = None cuda_id: InitVar[int] = 0 def __post_init__(self, cuda_id): + if self.bidir and self.backwards: + raise ValueError('Both "backwards" and "bidir" options cannot be enabled at the same time') if not torch.cuda.is_available(): print('CUDA not available. Setting device=-1.') cuda_id = -1 @@ -100,7 +106,16 @@ class LMHyperParams: def tokenizer_prefix(self): return f"{self.tokenizer.value}{self.max_vocab // 1000}k" @property - def model_prefix(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm') + def model_direction(self): + if self.bidir: + return 'bi' + if self.backwards: + return 'bwd' + else: + return '' + + @property + def model_prefix(self): return self.model_direction + ('qrnn' if self.qrnn else 'lstm') @property def model_name(self): return f"{self.model_prefix}_{self.name}.m" @@ -110,9 +125,14 @@ class LMHyperParams: @property def lm_type(self): - return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM + if self.bidir: + return contrib_data.LanguageModelType.BiLM + if self.backwards: + return contrib_data.LanguageModelType.BwdLM + else: + return contrib_data.LanguageModelType.FwdLM - def tokenzier_to_fastai_args(self, sp_data_func, use_moses): + def tokenizer_to_fastai_args(self, sp_data_func, use_moses): tok_func = MosesTokenizerFunc if use_moses else BaseTokenizer if self.tokenizer is Tokenizers.SUBWORD: if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model @@ -169,7 +189,7 @@ class LMHyperParams: learn.unfreeze() if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7) else: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values - learn.save("lm_best_with_opt", with_opt=False) + learn.save("lm_best_with_opt", with_opt=True) learn.save_encoder(ENC_BEST) learn.save(LM_BEST, with_opt=False) print(learn.path) @@ -183,7 +203,8 @@ class LMHyperParams: trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt, pretrained_fnames=self.pretrained_fnames, - pretrained_model=self.pretrained_model) + pretrained_model=self.pretrained_model, + alpha=self.rnn_alpha, beta=self.rnn_beta) trn_args.update(kwargs) print ("Training args: ", trn_args, "dps: ", dps or self.dps) learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, @@ -208,7 +229,7 @@ class LMHyperParams: for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' - args = self.tokenzier_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False) + args = self.tokenizer_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False) try: data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index ae5e7c3..5cff5d3 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -152,7 +152,7 @@ class CLSHyperParams(LMHyperParams): lm_trn_df = lm_trn_df[val_len:] lm_val_df = lm_trn_df[:val_len] - args = self.tokenzier_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) + args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) try: if force: raise FileNotFoundError("Forcing reloading of caches") data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type, bs=bs)