From 08d71dd45e76ece56afcb7d9e10ccd1af7e1b4cc Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 15 Nov 2018 23:40:50 +0100 Subject: [PATCH] Add script that fetches and prepres WT-2 WT-103 --- prepare_wiki-en.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 prepare_wiki-en.sh diff --git a/prepare_wiki-en.sh b/prepare_wiki-en.sh new file mode 100755 index 0000000..9e5add9 --- /dev/null +++ b/prepare_wiki-en.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +ROOT="data" +DATA_DIR="${ROOT}/wiki/" +mkdir -p "${DATA_DIR}" +echo "Saving data in ""$DATA_DIR" +wget -c "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip" -P "${DATA_DIR}" +wget -c "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip" -P "${DATA_DIR}" + +unzip "${DATA_DIR}/wikitext-2-v1.zip" -d "${DATA_DIR}" +unzip "${DATA_DIR}/wikitext-103-v1.zip" -d "${DATA_DIR}" + +for f in ${DATA_DIR}/wikitext-*/wiki.*.tokens; do + nf=$(dirname $f)/en.$(basename $f) + echo "Renaming $f to $nf" + mv $f $(dirname $f)/en.$(basename $f) +done + +echo "Please note wikitext en is already tokenized with Moses" \ No newline at end of file