wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-06 16:51:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add script that fetches and prepres WT-2 WT-103

Browse Source
This commit is contained in:
Piotr Czapla
2018-11-15 23:40:50 +01:00
parent 9a60ef2fbd
commit 08d71dd45e
1 changed files with 19 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
prepare_wiki-en.sh
Executable
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
ROOT="data"
DATA_DIR="${ROOT}/wiki/"
mkdir -p "${DATA_DIR}"
echo "Saving data in ""$DATA_DIR"
wget -c "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip" -P "${DATA_DIR}"
wget -c "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip" -P "${DATA_DIR}"
unzip "${DATA_DIR}/wikitext-2-v1.zip" -d "${DATA_DIR}"
unzip "${DATA_DIR}/wikitext-103-v1.zip" -d "${DATA_DIR}"
for f in ${DATA_DIR}/wikitext-*/wiki.*.tokens; do
nf=$(dirname $f)/en.$(basename $f)
echo "Renaming $f to $nf"
mv $f $(dirname $f)/en.$(basename $f)
done
echo "Please note wikitext en is already tokenized with Moses"