Use a torrent file to spread the load. (#44)

* Use a torrent file to spread the load.

* Stop seeding after the download.

Co-Authored-By: steamport <steamport@tutanota.com>

* Use absolute paths to change directories
This commit is contained in:
Aleksej Komarov
2019-12-07 23:29:27 -07:00
committed by Nick Walton
co-authored by steamport
parent 1c029b9642
commit 02cd136a92
+16 -17
View File
@@ -1,26 +1,25 @@
#!/bin/bash
cd "$(dirname "${0}")"
BASE_DIR="$(pwd)"
MODELS_DIRECTORY=generator/gpt2/models
MODEL_VERSION=model_v5
MODEL_NAME=model-550
DOWNLOAD_URL=http://130.211.31.182:80
MODEL_TORRENT_URL="https://github.com/nickwalton/AIDungeon/files/3935881/model_v5.torrent.zip"
MODEL_TORRENT_BASENAME="$(basename "${MODEL_TORRENT_URL}")"
if [ -d "${MODELS_DIRECTORY}/${MODEL_VERSION}" ]; then
if [[ -d "${MODELS_DIRECTORY}/${MODEL_VERSION}" ]]; then
echo "AIDungeon2 is already installed"
else
echo "Downloading AIDungeon2 Model... (this may take a few minutes)"
cd ${MODELS_DIRECTORY}
mkdir ${MODEL_VERSION}
cd ${MODEL_VERSION}
apt-get install aria2 > /dev/null
aria2c -x 16 -s 32 "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.data-00000-of-00001"
wget "${DOWNLOAD_URL}/${MODEL_VERSION}/checkpoint" > /dev/null
wget "${DOWNLOAD_URL}/${MODEL_VERSION}/encoder.json" > /dev/null
wget "${DOWNLOAD_URL}/${MODEL_VERSION}/hparams.json" > /dev/null
wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.index" > /dev/null
wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.meta" > /dev/null
wget "${DOWNLOAD_URL}/${MODEL_VERSION}/vocab.bpe" > /dev/null
echo "Downloading AIDungeon2 Model... (this may take a few minutes)"
mkdir -p "${MODELS_DIRECTORY}"
cd "${MODELS_DIRECTORY}"
mkdir "${MODEL_VERSION}"
apt-get install aria2 unzip > /dev/null
wget "${MODEL_TORRENT_URL}"
unzip "${MODEL_TORRENT_BASENAME}"
aria2c -x 16 -s 32 --seed-time=0 "${MODEL_TORRENT_BASENAME%.*}"
echo "Download Complete!"
cd ../../../..
cd "${BASE_DIR}"
pip install -r requirements.txt > /dev/null
fi