From 66341801c0ee5f1249cf46e400907a0fb756fae8 Mon Sep 17 00:00:00 2001
From: Cahya Wirawan <Cahya.Wirawan@gmail.com>
Date: Wed, 11 Sep 2019 12:49:58 +0200
Subject: [PATCH] added question of minimal tokens/article in to
 prepare_wiki.sh changed 'fname' to 'file' in load_data function to match the
 current fastai version

---
 prepare_wiki.sh       | 9 ++++++++-
 ulmfit/pretrain_lm.py | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/prepare_wiki.sh b/prepare_wiki.sh
index a43a868..d579275 100755
--- a/prepare_wiki.sh
+++ b/prepare_wiki.sh
@@ -13,6 +13,13 @@ else
 fi
 echo "Chosen language: ""$LANG"
 
+if [ "$2" == "" ] ; then
+    read -p "Enter the minimal tokens per articles [100]: " tokens_min
+    TOKENS_MIN=${tokens_min:-100}
+else
+    TOKENS_MIN="$2"
+fi
+
 DUMP_DIR="${ROOT}/wiki_dumps"
 EXTR_DIR="${ROOT}/wiki_extr"
 WIKI_DIR="${ROOT}/wiki"
@@ -46,7 +53,7 @@ else
   echo "${EXTR_PATH} already exists. Skipping extraction."
 fi
 
-python -m ulmfit.create_wikitext -i "${EXTR_PATH}"  -l "${LANG}" -o "${WIKI_DIR}"
+python -m ulmfit.create_wikitext -i "${EXTR_PATH}"  -l "${LANG}" -o "${WIKI_DIR}" -t "${TOKENS_MIN}"
 
 python -m ulmfit.postprocess_wikitext "${WIKI_DIR}/${LANG}-2" $LANG
 python -m ulmfit.postprocess_wikitext "${WIKI_DIR}/${LANG}-100" $LANG
diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 6b0b760..5bd55cf 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -268,7 +268,7 @@ class LMHyperParams:
         if (bunch_path / 'itos.pkl').exists():
             data = bunch_class.load(self.cache_dir, name, bs=bs)
         elif bunch_path.exists():
-            data = load_data(self.cache_dir, fname=name, bs=bs)
+            data = load_data(self.cache_dir, file=name, bs=bs)
         else:
             print(f"Running tokenization {name}...")
             data = bunch_class.from_df(path=self.cache_dir,