From 635fb1c1133ca8b4771bdf5762f9fa6d91e3d712 Mon Sep 17 00:00:00 2001 From: Nirant K Date: Mon, 19 Nov 2018 06:50:50 +0000 Subject: [PATCH 1/4] Add prepare_xnli.py --- prepare_xnli.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 prepare_xnli.py diff --git a/prepare_xnli.py b/prepare_xnli.py new file mode 100644 index 0000000..ee13c3a --- /dev/null +++ b/prepare_xnli.py @@ -0,0 +1,73 @@ +import zipfile +from pathlib import Path +from typing import Optional, Union + +import fire +from tqdm import tqdm + +from fastai.core import * +from fastai.datasets import * + +ROOT = Path("data").resolve() +XNLI_DIR = ROOT / "xnli" +if not ROOT.exists(): + ROOT.mkdir() +XNLI_DIR.mkdir(exist_ok=True) + +print(f"Saving data in {ROOT}") +MT_FILE = "XNLI-MT-1.0.zip" +XNLI_FILE = "XNLI-1.0.zip" +MT_PATH = XNLI_DIR / MT_FILE +XNLI_PATH = XNLI_DIR / XNLI_FILE +MT_URL = "https://s3.amazonaws.com/xnli/XNLI-MT-1.0.zip" +XNLI_URL = "https://s3.amazonaws.com/xnli/XNLI-1.0.zip" + + + +class TqdmUpTo(tqdm): + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) + + +def download_data(url: str, fname: Union[str, Path], dest: Optional[Union[str, Path]]): + """ + Download data if the filename does not exist already + Uses Tqdm to show download progress + """ + from urllib.request import urlretrieve + + filepath = (Path(dest) / fname).resolve() + + if not filepath.exists(): + dirname = Path(filepath.parents[0]) + print(f"Creating directory {dirname} from {filepath}") + dirname.mkdir(exist_ok=True) + + with TqdmUpTo(unit="B", unit_scale=True, miniters=1, desc=url.split("/")[-1]) as t: + urlretrieve(url, filepath, reporthook=t.update_to) + + return str(filepath.resolve().absolute()) + + +def get_and_unzip_data(url: str, fname: Union[str, Path] = None, dest: Union[str, Path] = None): + """Download `url` if it doesn't exist to `fname` and un-tgz to folder `dest`""" + if dest is None: + dest = url.split("/")[-1] + dest = Path(dest) + fname = dest / fname + if not fname.exists(): + download_data(url=url, fname=fname, dest=dest) + print(f"Extracting {fname.resolve().absolute()} \n to {dest}") + zipfile.ZipFile(fname, "r").extractall(dest) + return dest + + +def get_xnli_and_MT(dest: Union[str, Path] = XNLI_DIR): + get_and_unzip_data(url=XNLI_URL, fname=XNLI_FILE, dest=dest) + get_and_unzip_data(url=MT_URL, fname=MT_FILE, dest=dest) + + +if __name__ == "__main__": + fire.Fire(get_xnli_and_MT) From 56a9feec7d5c1659f121e8d33b39b3d84e44f8a7 Mon Sep 17 00:00:00 2001 From: Nirant Date: Mon, 19 Nov 2018 12:22:41 +0530 Subject: [PATCH 2/4] Removed dependency note, use requirements.txt --- ulmfit/README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ulmfit/README.md b/ulmfit/README.md index 355965d..17b2388 100644 --- a/ulmfit/README.md +++ b/ulmfit/README.md @@ -25,16 +25,6 @@ The extracted data should be in the folder `wiki_extr` -> language name e.g.`en` ## Create and Post Process WikiText -### Get the Dependencies - -**Python Fire**: To install Python Fire with pip, run: `pip install fire` - -To install Python Fire with conda, run: `conda install fire -c conda-forge` - -**Moses Tokenizer**: To install Moses Tokenizer: `pip install -U sacremoses` - -Use the Python script [create_wikitext.py](./create_wikitext.py) to process the extracted Wikipedia documents. - ### Create and Post-Process If you used the automated shell script from previous step, this might look something like ```bash From 69ed7b4169226bb169613d070891238e0bf9c1cd Mon Sep 17 00:00:00 2001 From: Aayush Date: Mon, 19 Nov 2018 19:11:20 +0530 Subject: [PATCH 3/4] Added preliminary test scripts for sentencepiece Other minor changes: - Function renaming: `test_pretrain_lm` -> `test_ulmfit_default_end_to_end` - Delete test models after completing each test. --- tests/test_end_to_end.py | 71 +++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 008c488..f17bbfb 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -1,3 +1,5 @@ +import os +import glob import fire import pytest @@ -10,20 +12,43 @@ from fastai_contrib.utils import * It is a mixture of a pytest unit test and woven together to compose an end to end functional test. """ +def delete_test_models(): + wt2 = data / 'wiki' / 'wikitext-2' + imdb = data / 'imdb' + + # delete test models from the pretraining step + for test_file in glob.iglob(f'{str(wt2)}/models/end-to-end-test*'): + if os.path.isfile(test_file): os.remove(test_file) + + # delete test vocab and model of sentencepiece training + for test_file in [wt2 / 'models' / 'spm.model', + wt2 / 'models' / 'spm.vocab']: + if os.path.isfile(test_file): os.remove(test_file) + + # delete test models from the finetuning/classifier training step + for test_file in glob.iglob(f'{str(imdb)}/models/end-to-end-test*'): + if os.path.isfile(test_file): os.remove(test_file) + + def check_data_exists(): data = get_data_folder() - wt2 = data / "wiki" / "wikitext-2" - imdb = data / "imdb" - ensure_paths_exists(wt2 / "en.wiki.train.tokens", - imdb / "train.csv", - message="We don't run data preparation scripts automatically as it takes ages, run prepare_wiki-en.sh & prepare_imdb.sh") + wt2 = data / 'wiki' / 'wikitext-2' + imdb = data / 'imdb' + ensure_paths_exists(wt2 / 'en.wiki.train.tokens', + imdb / 'train.csv', + message="We don't run data preparation" + " scripts automatically as it takes ages," + " run prepare_wiki-en.sh & prepare_imdb.sh") return imdb, wt2 -def test_pretrain_lm(): - imdb,wt2 = check_data_exists() - lm_name="end-to-end-test-quick" - cuda_id=0 + +def test_ulmfit_default_end_to_end(): + """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. + """ + imdb, wt2 = check_data_exists() + lm_name = 'end-to-end-test-default' + cuda_id = 0 results = ulmfit.pretrain_lm.pretrain_lm( dir_path=wt2, lang='en', @@ -48,3 +73,31 @@ def test_pretrain_lm(): bs=20, bptt=70, name=lm_name+'-imdb-clas', dataset='imdb', ds_pct=0.03) + + delete_test_models() + + +def test_ulmfit_sentencepiece_end_to_end(): + """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. + """ + imdb, wt2 = check_data_exists() + lm_name = 'end-to-end-test-spm' + cuda_id = 0 + results = ulmfit.pretrain_lm.pretrain_lm( + dir_path=wt2, + lang='en', + cuda_id=cuda_id, + qrnn=True, + subword=True, + max_vocab=1000, + bs=80, + num_epochs=1, + name=lm_name, + ) + + assert results['accuracy'] > 0.30 + + # NOTE: ds_pct is not available for sentencepiece -- tests are on the complete dataset + # sentencepiece for finetuning/classification is currently not implemented + + delete_test_models() From 5ffbe8ba5ce6146188a64e4427fa42d781921920 Mon Sep 17 00:00:00 2001 From: Aayush Date: Mon, 19 Nov 2018 19:13:55 +0530 Subject: [PATCH 4/4] add vocab_size to sentencepiece --- ulmfit/pretrain_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 0847ca6..06853a0 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -70,7 +70,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo read_file(trn_path, 'train') read_file(val_path, 'valid') - sp = get_sentencepiece(dir_path, trn_path, name) + sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab) data_lm = TextLMDataBunch.from_csv(dir_path, **sp) itos = data_lm.train_ds.vocab.itos