Merge branch 'master' of https://github.com/n-waves/ulmfit-multilingual into datasets

prepare_xnli.py

@@ -0,0 +1,73 @@
+import zipfile
+from pathlib import Path
+from typing import Optional, Union
+
+import fire
+from tqdm import tqdm
+
+from fastai.core import *
+from fastai.datasets import *
+
+ROOT = Path("data").resolve()
+XNLI_DIR = ROOT / "xnli"
+if not ROOT.exists():
+    ROOT.mkdir()
+XNLI_DIR.mkdir(exist_ok=True)
+
+print(f"Saving data in {ROOT}")
+MT_FILE = "XNLI-MT-1.0.zip"
+XNLI_FILE = "XNLI-1.0.zip"
+MT_PATH = XNLI_DIR / MT_FILE
+XNLI_PATH = XNLI_DIR / XNLI_FILE
+MT_URL = "https://s3.amazonaws.com/xnli/XNLI-MT-1.0.zip"
+XNLI_URL = "https://s3.amazonaws.com/xnli/XNLI-1.0.zip"
+
+
+
+class TqdmUpTo(tqdm):
+    def update_to(self, b=1, bsize=1, tsize=None):
+        if tsize is not None:
+            self.total = tsize
+        self.update(b * bsize - self.n)
+
+
+def download_data(url: str, fname: Union[str, Path], dest: Optional[Union[str, Path]]):
+    """
+    Download data if the filename does not exist already
+    Uses Tqdm to show download progress
+    """
+    from urllib.request import urlretrieve
+
+    filepath = (Path(dest) / fname).resolve()
+
+    if not filepath.exists():
+        dirname = Path(filepath.parents[0])
+        print(f"Creating directory {dirname} from {filepath}")
+        dirname.mkdir(exist_ok=True)
+
+        with TqdmUpTo(unit="B", unit_scale=True, miniters=1, desc=url.split("/")[-1]) as t:
+            urlretrieve(url, filepath, reporthook=t.update_to)
+
+    return str(filepath.resolve().absolute())
+
+
+def get_and_unzip_data(url: str, fname: Union[str, Path] = None, dest: Union[str, Path] = None):
+    """Download `url` if it doesn't exist to `fname` and un-tgz to folder `dest`"""
+    if dest is None:
+        dest = url.split("/")[-1]
+    dest = Path(dest)
+    fname = dest / fname
+    if not fname.exists():
+        download_data(url=url, fname=fname, dest=dest)
+    print(f"Extracting {fname.resolve().absolute()} \n to {dest}")
+    zipfile.ZipFile(fname, "r").extractall(dest)
+    return dest
+
+
+def get_xnli_and_MT(dest: Union[str, Path] = XNLI_DIR):
+    get_and_unzip_data(url=XNLI_URL, fname=XNLI_FILE, dest=dest)
+    get_and_unzip_data(url=MT_URL, fname=MT_FILE, dest=dest)
+
+
+if __name__ == "__main__":
+    fire.Fire(get_xnli_and_MT)

tests/test_end_to_end.py

@@ -1,3 +1,5 @@
+import os
+import glob
 import fire
 import pytest
 
@@ -10,20 +12,43 @@ from fastai_contrib.utils import  *
 It is a mixture of a pytest unit test and woven together to compose an end to end functional test. 
 """
 
+def delete_test_models():
+    wt2 = data / 'wiki' / 'wikitext-2'
+    imdb = data / 'imdb'
+
+    # delete test models from the pretraining step
+    for test_file in glob.iglob(f'{str(wt2)}/models/end-to-end-test*'):
+        if os.path.isfile(test_file): os.remove(test_file)
+
+    # delete test vocab and model of sentencepiece training
+    for test_file in [wt2 / 'models' / 'spm.model', 
+                      wt2 / 'models' / 'spm.vocab']:
+        if os.path.isfile(test_file): os.remove(test_file)
+
+    # delete test models from the finetuning/classifier training step
+    for test_file in glob.iglob(f'{str(imdb)}/models/end-to-end-test*'):
+        if os.path.isfile(test_file): os.remove(test_file)
+
+
 def check_data_exists():
     data = get_data_folder()
 
-    wt2 = data / "wiki" / "wikitext-2"
-    imdb = data / "imdb"
-    ensure_paths_exists(wt2 / "en.wiki.train.tokens",
-                        imdb / "train.csv",
-                        message="We don't run data preparation scripts automatically as it takes ages, run prepare_wiki-en.sh & prepare_imdb.sh")
+    wt2 = data / 'wiki' / 'wikitext-2'
+    imdb = data / 'imdb'
+    ensure_paths_exists(wt2 / 'en.wiki.train.tokens',
+                        imdb / 'train.csv',
+                        message="We don't run data preparation"
+                                " scripts automatically as it takes ages,"
+                                " run prepare_wiki-en.sh & prepare_imdb.sh")
     return imdb, wt2
 
-def test_pretrain_lm():
-    imdb,wt2 = check_data_exists()
-    lm_name="end-to-end-test-quick"
-    cuda_id=0
+
+def test_ulmfit_default_end_to_end():
+    """  Test ulmfit with (default) Moses tokenizer on small wikipedia dataset.
+    """
+    imdb, wt2 = check_data_exists()
+    lm_name = 'end-to-end-test-default'
+    cuda_id = 0
     results = ulmfit.pretrain_lm.pretrain_lm(
         dir_path=wt2,
         lang='en',
@@ -48,3 +73,31 @@ def test_pretrain_lm():
                 bs=20, bptt=70, name=lm_name+'-imdb-clas',
                 dataset='imdb',
                 ds_pct=0.03)
+
+    delete_test_models()
+
+
+def test_ulmfit_sentencepiece_end_to_end():
+    """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
+    """
+    imdb, wt2 = check_data_exists()
+    lm_name = 'end-to-end-test-spm'
+    cuda_id = 0
+    results = ulmfit.pretrain_lm.pretrain_lm(
+        dir_path=wt2,
+        lang='en',
+        cuda_id=cuda_id,
+        qrnn=True,
+        subword=True,
+        max_vocab=1000,
+        bs=80,
+        num_epochs=1,
+        name=lm_name,
+    )
+
+    assert results['accuracy'] > 0.30
+
+    # NOTE: ds_pct is not available for sentencepiece -- tests are on the complete dataset
+    #       sentencepiece for finetuning/classification is currently not implemented
+
+    delete_test_models()

ulmfit/README.md

@@ -25,16 +25,6 @@ The extracted data should be in the folder `wiki_extr` -> language name e.g.`en`
 
 ## Create and Post Process WikiText
 
-### Get the Dependencies
-
-**Python Fire**: To install Python Fire with pip, run: `pip install fire`
-
-To install Python Fire with conda, run: `conda install fire -c conda-forge`
-
-**Moses Tokenizer**: To install Moses Tokenizer: `pip install -U sacremoses`
-
-Use the Python script [create_wikitext.py](./create_wikitext.py) to process the extracted Wikipedia documents. 
-
 ### Create and Post-Process 
 If you used the automated shell script from previous step, this might look something like
 ```bash

ulmfit/pretrain_lm.py

@@ -70,7 +70,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
         read_file(trn_path, 'train')
         read_file(val_path, 'valid')
 
-        sp = get_sentencepiece(dir_path, trn_path, name)
+        sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab)
 
         data_lm = TextLMDataBunch.from_csv(dir_path, **sp)
         itos = data_lm.train_ds.vocab.itos