mirror of
https://github.com/wassname/multifit.git
synced 2026-08-30 11:27:22 +08:00
27 lines
773 B
Bash
27 lines
773 B
Bash
#!/usr/bin/env bash
|
||
# Script to download a Wikipedia dump
|
||
|
||
# Script is partially based on https://github.com/facebookresearch/fastText/blob/master/get-wikimedia.sh
|
||
ROOT="data"
|
||
XNLI_DIR="${ROOT}/xnli"
|
||
mkdir -p "${ROOT}"
|
||
mkdir -p "${XNLI_DIR}"
|
||
|
||
echo "Saving data in ""$ROOT"
|
||
MT_FILE="XNLI-MT-1.0.zip"
|
||
XNLI_FILE="XNLI-1.0.zip"
|
||
MT_PATH="${XNLI_DIR}/${MT_FILE}"
|
||
XNLI_PATH="${XNLI_DIR}/${XNLI_FILE}"
|
||
|
||
if [ ! -f "${MT_PATH}" ]; then
|
||
wget -c "https://s3.amazonaws.com/xnli/XNLI-MT-1.0.zip" -P "${XNLI_DIR}"
|
||
wget -c "https://s3.amazonaws.com/xnli/XNLI-1.0.zip" -P "${XNLI_DIR}"
|
||
else
|
||
echo "${MT_PATH} already exists. Skipping download."
|
||
fi
|
||
|
||
unzip "${MT_PATH}" -d "${XNLI_DIR}"
|
||
unzip "${XNLI_PATH}" -d "${XNLI_DIR}"
|
||
|
||
echo "Please note xnli en is already tokenized with Moses"
|