From 111fc7e4c32c1df891d152877305eb1d64c8c44f Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Thu, 15 Nov 2018 23:37:15 +0100
Subject: [PATCH] Add perpare_imdb script

---
 fastai_contrib/utils.py |  8 ++++----
 prepare_imdb.sh         | 11 +++++++++++
 2 files changed, 15 insertions(+), 4 deletions(-)
 create mode 100644 prepare_imdb.sh

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 3ea3b36..cbccac9 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -57,20 +57,20 @@ def prepare_imdb(file_path: str, prepare_lm = False):
     """
 
     file_path = pathlib.Path(file_path)
-    dir_path = pathlib.Path(file_path.stem).resolve()
+    dir_path = pathlib.Path(file_path.parent / 'aclImdb').resolve()
     assert tarfile.is_tarfile(file_path), "this is not a valid targz file"
 
     if not dir_path.exists():
         print(f"Extracting {file_path} to {dir_path}. This may take a long time...")
         tgz_file = tarfile.open(file_path)
-        tgz_file.extractall()
+        tgz_file.extractall(path=dir_path.parent) # the aclImdb.tgz has aclImdb dir packed 
         assert dir_path.exists()
         print(f"Extracted to {dir_path}")
 
-    CLAS_PATH = dir_path
+    CLAS_PATH = dir_path.parent
     CLAS_PATH.mkdir(exist_ok=True)
 
-    LM_PATH = dir_path /'imdb_lm'
+    LM_PATH = dir_path.parent /'imdb_lm'
     LM_PATH.mkdir(exist_ok=True)
 
     # processing the split files to create train.csv and test.csv in fastai format
diff --git a/prepare_imdb.sh b/prepare_imdb.sh
new file mode 100644
index 0000000..453f0a1
--- /dev/null
+++ b/prepare_imdb.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+ROOT="data"
+DATA_DIR="${ROOT}/imdb"
+mkdir -p "${DATA_DIR}"
+echo "Saving data in $DATA_DIR"
+wget -c "http://files.fast.ai/data/aclImdb.tgz" -P "${DATA_DIR}"
+
+echo "Imdb is raw text so we are tokenizing it with Moses"
+python -m fastai_contrib.utils prepare_imdb "${DATA_DIR}/aclImdb.tgz"
+