From 6fda7f2cdaad9677a4ccb44539dc3c86f6cfe8bf Mon Sep 17 00:00:00 2001
From: Marcin <marcin.kardas@gmail.com>
Date: Thu, 14 Feb 2019 16:45:51 +0100
Subject: [PATCH] Download preprocessed wikis

---
 get_preprocessed_wikis.sh | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100755 get_preprocessed_wikis.sh

diff --git a/get_preprocessed_wikis.sh b/get_preprocessed_wikis.sh
new file mode 100755
index 0000000..b5c1339
--- /dev/null
+++ b/get_preprocessed_wikis.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -x
+ZIPNAME="preprocessed_wiki_8langs.zip"
+OUTDIR="data/wiki"
+wget -nc 'https://www.dropbox.com/sh/srfwvur6orq0cre/AAAQc36bcD17C1KM1mneXN7fa/data/wiki?dl=1' -O "${ZIPNAME}"
+mkdir -p "${OUTDIR}"
+unzip "${ZIPNAME}" -d "${OUTDIR}"
+
+for archive in "${OUTDIR}"/??-100.tar.gz; do tar xvf "${archive}" -C "${OUTDIR}" && rm "${archive}"; done