Add perpare_imdb script

fastai_contrib/utils.py

@@ -57,20 +57,20 @@ def prepare_imdb(file_path: str, prepare_lm = False):
     """
 
     file_path = pathlib.Path(file_path)
-    dir_path = pathlib.Path(file_path.stem).resolve()
+    dir_path = pathlib.Path(file_path.parent / 'aclImdb').resolve()
     assert tarfile.is_tarfile(file_path), "this is not a valid targz file"
 
     if not dir_path.exists():
         print(f"Extracting {file_path} to {dir_path}. This may take a long time...")
         tgz_file = tarfile.open(file_path)
-        tgz_file.extractall()
+        tgz_file.extractall(path=dir_path.parent) # the aclImdb.tgz has aclImdb dir packed 
         assert dir_path.exists()
         print(f"Extracted to {dir_path}")
 
-    CLAS_PATH = dir_path
+    CLAS_PATH = dir_path.parent
     CLAS_PATH.mkdir(exist_ok=True)
 
-    LM_PATH = dir_path /'imdb_lm'
+    LM_PATH = dir_path.parent /'imdb_lm'
     LM_PATH.mkdir(exist_ok=True)
 
     # processing the split files to create train.csv and test.csv in fastai format

prepare_imdb.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+ROOT="data"
+DATA_DIR="${ROOT}/imdb"
+mkdir -p "${DATA_DIR}"
+echo "Saving data in $DATA_DIR"
+wget -c "http://files.fast.ai/data/aclImdb.tgz" -P "${DATA_DIR}"
+
+echo "Imdb is raw text so we are tokenizing it with Moses"
+python -m fastai_contrib.utils prepare_imdb "${DATA_DIR}/aclImdb.tgz"
+