diff --git a/get_preprocessed_wikis.sh b/get_preprocessed_wikis.sh new file mode 100755 index 0000000..b5c1339 --- /dev/null +++ b/get_preprocessed_wikis.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -x +ZIPNAME="preprocessed_wiki_8langs.zip" +OUTDIR="data/wiki" +wget -nc 'https://www.dropbox.com/sh/srfwvur6orq0cre/AAAQc36bcD17C1KM1mneXN7fa/data/wiki?dl=1' -O "${ZIPNAME}" +mkdir -p "${OUTDIR}" +unzip "${ZIPNAME}" -d "${OUTDIR}" + +for archive in "${OUTDIR}"/??-100.tar.gz; do tar xvf "${archive}" -C "${OUTDIR}" && rm "${archive}"; done