wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-11 12:20:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #49 from cahya-wirawan/tokens_min

Browse Source 
a bug fix and adding minimal tokens/article to prepare_wiki.sh
This commit is contained in:
Piotr Czapla
2019-10-08 21:46:33 +02:00
committed by GitHub
parent 8f2b7edee4 66341801c0
commit 96a870b9b9
2 changed files with 9 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
prepare_wiki.sh
+8 -1
View File
@@ -13,6 +13,13 @@ else
fi
echo "Chosen language: ""$LANG"
if [ "$2" == "" ] ; then
read -p "Enter the minimal tokens per articles [100]: " tokens_min
TOKENS_MIN=${tokens_min:-100}
else
TOKENS_MIN="$2"
fi
DUMP_DIR="${ROOT}/wiki_dumps"
EXTR_DIR="${ROOT}/wiki_extr"
WIKI_DIR="${ROOT}/wiki"
@@ -46,7 +53,7 @@ else
echo "${EXTR_PATH} already exists. Skipping extraction."
fi
python -m ulmfit.create_wikitext -i "${EXTR_PATH}" -l "${LANG}" -o "${WIKI_DIR}"
python -m ulmfit.create_wikitext -i "${EXTR_PATH}" -l "${LANG}" -o "${WIKI_DIR}" -t "${TOKENS_MIN}"
python -m ulmfit.postprocess_wikitext "${WIKI_DIR}/${LANG}-2" $LANG
python -m ulmfit.postprocess_wikitext "${WIKI_DIR}/${LANG}-100" $LANG
ulmfit/pretrain_lm.py
+1 -1
View File
@@ -268,7 +268,7 @@ class LMHyperParams:
if (bunch_path / 'itos.pkl').exists():
data = bunch_class.load(self.cache_dir, name, bs=bs)
elif bunch_path.exists():
data = load_data(self.cache_dir, fname=name, bs=bs)
data = load_data(self.cache_dir, file=name, bs=bs)
else:
print(f"Running tokenization {name}...")
data = bunch_class.from_df(path=self.cache_dir,