From 62e6777952716e2ba1d100ca6700180b27433548 Mon Sep 17 00:00:00 2001
From: Nirant K <nirant@soroco.com>
Date: Wed, 14 Nov 2018 15:53:59 +0000
Subject: [PATCH] Add download and unzip script

---
 ulmfit/xnli/prepare_xnli.sh | 48 +++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 ulmfit/xnli/prepare_xnli.sh

diff --git a/ulmfit/xnli/prepare_xnli.sh b/ulmfit/xnli/prepare_xnli.sh
new file mode 100644
index 0000000..31e1f0b
--- /dev/null
+++ b/ulmfit/xnli/prepare_xnli.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Script to download a Wikipedia dump
+
+# Script is partially based on https://github.com/facebookresearch/fastText/blob/master/get-wikimedia.sh
+ROOT="data"
+DUMP_DIR="${ROOT}/xnli_dnld"
+EXTR_DIR="${ROOT}/xnli_extr"
+WIKI_DIR="${ROOT}/xnli"
+EXTR="wikiextractor"
+mkdir -p "${ROOT}"
+mkdir -p "${DUMP_DIR}"
+mkdir -p "${WIKI_DIR}"
+
+echo "Saving data in ""$ROOT"
+read -r -p "Choose a language (e.g. en, fr, etc.): " choice
+LANG="$choice"
+echo "Chosen language: ""$LANG"
+MT_FILE="XNLI-MT-1.0.zip"
+XNLI_FILE="XNLI-1.0.zip" 
+MT_PATH="${DUMP_DIR}/${MT_FILE}"
+XNLI_PATH="${DUMP_DIR}/${XNLI_FILE}"
+
+if [ ! -f "${MT_PATH}" ]; then
+  read -r -p "Continue to download (WARNING: This might be big and can take a long time!) (y/n)? " choice
+  case "$choice" in
+    y|Y ) echo "Starting download...";;
+    n|N ) echo "Exiting";exit 1;;
+    * ) echo "Invalid answer";exit 1;;
+  esac
+  wget -c "https://s3.amazonaws.com/xnli/XNLI-MT-1.0.zip" -P "${DUMP_DIR}"
+  wget -c "https://s3.amazonaws.com/xnli/XNLI-1.0.zip" -P "${DUMP_DIR}"
+
+else
+  echo "${MT_PATH} already exists. Skipping download."
+fi
+
+if [ ! -d "${EXTR_DIR}" ]; then
+  read -r -p "Continue to extract XNLI (WARNING: This might take a long time!) (y/n)? " choice
+  case "$choice" in
+    y|Y ) echo "Extracting ${MT_PATH} to ${EXTR_DIR}...";;
+    n|N ) echo "Exiting";exit 1;;
+    * ) echo "Invalid answer";exit 1;;
+  esac
+  unzip "${MT_PATH}" -d "${EXTR_DIR}"
+  unzip "${XNLI_PATH}" -d "${EXTR_DIR}"
+else
+  echo "${EXTR_DIR} already exists. Skipping extraction."
+fi