Download preprocessed wikis

2026-09-09 11:27:26 +08:00 · 2019-02-14 16:45:51 +01:00
parent fdac9f7ccd
commit 6fda7f2cda
1 changed files with 9 additions and 0 deletions
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -x
+ZIPNAME="preprocessed_wiki_8langs.zip"
+OUTDIR="data/wiki"
+wget -nc 'https://www.dropbox.com/sh/srfwvur6orq0cre/AAAQc36bcD17C1KM1mneXN7fa/data/wiki?dl=1' -O "${ZIPNAME}"
+mkdir -p "${OUTDIR}"
+unzip "${ZIPNAME}" -d "${OUTDIR}"
+
+for archive in "${OUTDIR}"/??-100.tar.gz; do tar xvf "${archive}" -C "${OUTDIR}" && rm "${archive}"; done