test_end_to_end now working, added instructions to README, sentencepiece dependency

README.md

@@ -88,3 +88,13 @@ $ git push --set-upstream n-waves ulmfit_multilingual  # to automatically push u
     - `bilm` -- scripts to train biLM ELMo style, Bert style
     - `class`  -- scripts to test classifiers on multiple languages
     - `xnli` -- scripts to test nli 
+
+
+## Running tests
+
+To run the tests, the following data is necessary:
+
+- wikitext-2 (prepared by `./prepare_wiki-en.sh`, along with wikitext-103)
+- imdb (prepared by `./prepare_imdb.sh`)
+
+then simply run tests, e.g. `pytest .`

requirements.txt

@@ -1,4 +1,5 @@
 fire>=0.1.3
 cupy>=5.0.0
 scikit-learn>=0.20
-sacremoses>=0.0.5
+sacremoses>=0.0.5
+sentencepiece

tests/test_end_to_end.py

@@ -25,7 +25,8 @@ def get_test_data():
     imdb = data / "imdb"
 
     test_data = data / "test"
-    shutil.rmtree(test_data)
+    if test_data.exists():
+        shutil.rmtree(test_data)
 
     test_wt = test_data / 'wikitext-s'
     test_imdb = test_data / 'imdb'