From d5e6849cd8ec6964cb103980db76a5665d97bd74 Mon Sep 17 00:00:00 2001
From: Sebastian <ruder.sebastian@gmail.com>
Date: Thu, 15 Nov 2018 14:33:21 +0000
Subject: [PATCH] Shortened XNLI download script, removed unnecessary parts;
 extracts now to data folder

---
 prepare_xnli.sh             | 24 +++++++++++++++++++
 ulmfit/xnli/prepare_xnli.sh | 48 -------------------------------------
 2 files changed, 24 insertions(+), 48 deletions(-)
 create mode 100644 prepare_xnli.sh
 delete mode 100644 ulmfit/xnli/prepare_xnli.sh

diff --git a/prepare_xnli.sh b/prepare_xnli.sh
new file mode 100644
index 0000000..fa67589
--- /dev/null
+++ b/prepare_xnli.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Script to download a Wikipedia dump
+
+# Script is partially based on https://github.com/facebookresearch/fastText/blob/master/get-wikimedia.sh
+ROOT="data"
+XNLI_DIR="${ROOT}/xnli"
+mkdir -p "${ROOT}"
+mkdir -p "${XNLI_DIR}"
+
+echo "Saving data in ""$ROOT"
+MT_FILE="XNLI-MT-1.0.zip"
+XNLI_FILE="XNLI-1.0.zip" 
+MT_PATH="${XNLI_DIR}/${MT_FILE}"
+XNLI_PATH="${XNLI_DIR}/${XNLI_FILE}"
+
+if [ ! -f "${MT_PATH}" ]; then
+  wget -c "https://s3.amazonaws.com/xnli/XNLI-MT-1.0.zip" -P "${XNLI_DIR}"
+  wget -c "https://s3.amazonaws.com/xnli/XNLI-1.0.zip" -P "${XNLI_DIR}"
+else
+  echo "${MT_PATH} already exists. Skipping download."
+fi
+
+unzip "${MT_PATH}" -d "${XNLI_DIR}"
+unzip "${XNLI_PATH}" -d "${XNLI_DIR}"
diff --git a/ulmfit/xnli/prepare_xnli.sh b/ulmfit/xnli/prepare_xnli.sh
deleted file mode 100644
index 31e1f0b..0000000
--- a/ulmfit/xnli/prepare_xnli.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Script to download a Wikipedia dump
-
-# Script is partially based on https://github.com/facebookresearch/fastText/blob/master/get-wikimedia.sh
-ROOT="data"
-DUMP_DIR="${ROOT}/xnli_dnld"
-EXTR_DIR="${ROOT}/xnli_extr"
-WIKI_DIR="${ROOT}/xnli"
-EXTR="wikiextractor"
-mkdir -p "${ROOT}"
-mkdir -p "${DUMP_DIR}"
-mkdir -p "${WIKI_DIR}"
-
-echo "Saving data in ""$ROOT"
-read -r -p "Choose a language (e.g. en, fr, etc.): " choice
-LANG="$choice"
-echo "Chosen language: ""$LANG"
-MT_FILE="XNLI-MT-1.0.zip"
-XNLI_FILE="XNLI-1.0.zip" 
-MT_PATH="${DUMP_DIR}/${MT_FILE}"
-XNLI_PATH="${DUMP_DIR}/${XNLI_FILE}"
-
-if [ ! -f "${MT_PATH}" ]; then
-  read -r -p "Continue to download (WARNING: This might be big and can take a long time!) (y/n)? " choice
-  case "$choice" in
-    y|Y ) echo "Starting download...";;
-    n|N ) echo "Exiting";exit 1;;
-    * ) echo "Invalid answer";exit 1;;
-  esac
-  wget -c "https://s3.amazonaws.com/xnli/XNLI-MT-1.0.zip" -P "${DUMP_DIR}"
-  wget -c "https://s3.amazonaws.com/xnli/XNLI-1.0.zip" -P "${DUMP_DIR}"
-
-else
-  echo "${MT_PATH} already exists. Skipping download."
-fi
-
-if [ ! -d "${EXTR_DIR}" ]; then
-  read -r -p "Continue to extract XNLI (WARNING: This might take a long time!) (y/n)? " choice
-  case "$choice" in
-    y|Y ) echo "Extracting ${MT_PATH} to ${EXTR_DIR}...";;
-    n|N ) echo "Exiting";exit 1;;
-    * ) echo "Invalid answer";exit 1;;
-  esac
-  unzip "${MT_PATH}" -d "${EXTR_DIR}"
-  unzip "${XNLI_PATH}" -d "${EXTR_DIR}"
-else
-  echo "${EXTR_DIR} already exists. Skipping extraction."
-fi