Bug solve on attention module and a new Notebook to experiment spectrogram reconstruction

PlayGround.ipynb

@@ -0,0 +1,285 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Populating the interactive namespace from numpy and matplotlib\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "import io\n",
+    "import torch \n",
+    "import numpy as np\n",
+    "from collections import OrderedDict\n",
+    "\n",
+    "%pylab inline\n",
+    "rcParams[\"figure.figsize\"] = (16,5)\n",
+    "\n",
+    "import librosa\n",
+    "import librosa.display\n",
+    "\n",
+    "from TTS.models.tacotron import Tacotron \n",
+    "from TTS.layers import *\n",
+    "from TTS.utils.data import *\n",
+    "from TTS.utils.audio import AudioProcessor\n",
+    "from TTS.utils.generic_utils import load_config\n",
+    "from TTS.utils.text import text_to_sequence\n",
+    "\n",
+    "import IPython\n",
+    "from IPython.display import Audio"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def create_speech(m, s):\n",
+    "    text_cleaner = [CONFIG.text_cleaner]\n",
+    "    seq = np.array(text_to_sequence(s, text_cleaner))\n",
+    "            \n",
+    "#     mel = np.zeros([seq.shape[0], CONFIG.num_mels, 1], dtype=np.float32)\n",
+    "    \n",
+    "    if use_cuda:\n",
+    "        chars_var = torch.autograd.Variable(torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()\n",
+    "#         mel_var = torch.autograd.Variable(torch.from_numpy(mel).type(torch.cuda.FloatTensor), volatile=True).cuda() \n",
+    "    else:\n",
+    "        chars_var = torch.autograd.Variable(torch.from_numpy(seq), volatile=True).unsqueeze(0)\n",
+    "#         mel_var = torch.autograd.Variable(torch.from_numpy(mel).type(torch.FloatTensor), volatile=True)\n",
+    "        \n",
+    "    mel_out, linear_out, alignments = model.forward(chars_var)\n",
+    "    linear_out = linear_out[0].data.cpu().numpy()\n",
+    "    alignment = alignments[0].cpu().data.numpy()\n",
+    "    spec = ap._denormalize(linear_out)\n",
+    "    \n",
+    "    wav = ap.inv_spectrogram(linear_out.T)\n",
+    "    wav = wav[:ap.find_endpoint(wav)]\n",
+    "    out = io.BytesIO()\n",
+    "    ap.save_wav(wav, out)\n",
+    "    return wav, alignment, spec\n",
+    "\n",
+    "\n",
+    "def visualize(alignment, spectrogram):\n",
+    "    label_fontsize = 16\n",
+    "    figure(figsize=(16,16))\n",
+    "    \n",
+    "    subplot(2,1,1)\n",
+    "    imshow(alignment.T, aspect=\"auto\", origin=\"lower\", interpolation=None)\n",
+    "    xlabel(\"Decoder timestamp\", fontsize=label_fontsize)\n",
+    "    ylabel(\"Encoder timestamp\", fontsize=label_fontsize)\n",
+    "    colorbar()\n",
+    "\n",
+    "    subplot(2,1,2)\n",
+    "    librosa.display.specshow(spectrogram.T, sr=CONFIG.sample_rate, \n",
+    "                             hop_length=hop_length, x_axis=\"time\", y_axis=\"linear\")\n",
+    "    xlabel(\"Time\", fontsize=label_fontsize)\n",
+    "    ylabel(\"Hz\", fontsize=label_fontsize)\n",
+    "    tight_layout()\n",
+    "    colorbar()\n",
+    "    \n",
+    "    \n",
+    "def tts(model, text, figures=True):\n",
+    "    waveform, alignment, spectrogram = create_speech(model, text)\n",
+    "    if figures:\n",
+    "        visualize(alignment, spectrogram)\n",
+    "    IPython.display.display(Audio(waveform, rate=CONFIG.sample_rate))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "MODEL_PATH = '../result/January-26-2018_02:44AM/checkpoint_1800.pth.tar'\n",
+    "CONFIG_PATH = '../result/January-26-2018_02:44AM/config.json'\n",
+    "OUT_FOLDER = '../result/January-26-2018_02:44AM/test/'\n",
+    "CONFIG = load_config(CONFIG_PATH)\n",
+    "use_cuda = False\n",
+    "hop_length = 250"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load the model\n",
+    "model = Tacotron(CONFIG.embedding_size, CONFIG.hidden_size,\n",
+    "        CONFIG.num_mels, CONFIG.num_freq, CONFIG.r)\n",
+    "ap = AudioProcessor(CONFIG.sample_rate, CONFIG.num_mels, CONFIG.min_level_db,\n",
+    "                         CONFIG.frame_shift_ms, CONFIG.frame_length_ms, CONFIG.preemphasis,\n",
+    "                         CONFIG.ref_level_db, CONFIG.num_freq, CONFIG.power, griffin_lim_iters=80)         \n",
+    "\n",
+    "if use_cuda:\n",
+    "    model = torch.nn.DataParallel(model.cuda())\n",
+    "else:\n",
+    "    cp = torch.load(MODEL_PATH, map_location=lambda storage, loc: storage)\n",
+    "    \n",
+    "    # remove DataPatallel wrapper\n",
+    "    new_state_dict = OrderedDict()\n",
+    "    for k, v in cp['model'].items():\n",
+    "        name = k[7:] # remove `module.`\n",
+    "        new_state_dict[name] = v\n",
+    "    cp['model'] = new_state_dict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model.load_state_dict(cp['model'])\n",
+    "# model.decoder.eval();\n",
+    "model.encoder.eval();\n",
+    "model.postnet.eval();"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "sentences = [\n",
+    "    \"I try to speak my friend.\",\n",
+    "    \"I speak more than binary any more.\",\n",
+    "    \"I try ti implement a new TTS system.\"\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Warning! doesn't seems to be converged\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRqQ+AABXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YYA+AACr/2YAZv96/5AAWf/k/zIAdP/X/zoAiv8ZAMz/Xf/6ACL/fv8eAcD+dADw/zv/NQGk/hkAEAFb/tIA9P82/xsBv/5LAMEAp/6XAEMADv/LAJr/kv/LADP/NAArAJX/JQAXAL//xv+4ACH/GgC6ANj+xAC//6D/xAAj/0YAMgDU/83/CQBEAKL/PQCQ/3sA0/+t/3MAcv+rAHv/uv/dAGr/wv94ANX/+P/r/93/jwBm/wYAawCb/xgAAAAEAOP/LADL/wAAUwB7/z4ANgCS/y8AIAC//zEA3f/j/4kAXv/4/74AWv/2/1cA0/8SAND/FAA3AL7/8v8tAB0Akv9HACkAi/9vALz/BgAxALb/MgDn/xMA9P/r/zAA3//x/x4AAwDd/xUA/f8AAAwA1v8fAA0A4//9/xUA+/8CANj/NQAMAJH/iwC6/9f/dACQ/y0AEwDV/ywA3P8AADUAxP/4/1EAtP8FADAAyv8lAPr/2f83APj/y/88AOr/6/8zAM7/+/9GAMH/8P9SAKj/OwD4/8f/UgDZ//T/BgAcANr/FgABAM3/bwCa//7/aQCP/zMABQDd/yUA9//l/xgAHgC2/yYAKQC0/zEA+v/N/3kAhv8OAGgAdv9iAO3/s/+BAK//8f9IAKr/TwDX/+f/SACs/0QA7//I/1oA1P/b/1IAxv/7/zoAqv9bAOD/p/+aALH/3/9NALL/PwAJAKT/TwASAKX/VwDt/7H/hgCk/wEAQwCi/0UADACt/zYAMwCK/1kA/P+3/2wApv8nABAAw/9YALb/CwBDAJ7/MQAlAK//MAAKAM//SQDC/+r/cwCX/xkABgAEABsAo/+CAIn/KwBdAEn/igDd/+j/QACT/1gADQCn/0oA7P/v/yMA2P8jAOn/AQAVAOn/EgDR/1oAtP/k/6sAFP96AG8AC/+lAPj/tf+BAFH/SgCsAOb+egBBALP/HQC8/4cAff88ABoAqP9sAKb/MwDZ/1EAsP+3//oAD/9UAC0Aev/UAED/OwAuAKb/qQAS/48ATQAZ//UAXP8ZAHMAV/+jAEj/pQAQAAb/JgGJ/8H/UAAgAI//VgAJAGv/MwG7/lwA2gCe/hcBqf9m/z4BGf+u/20BiP5mABMB7v3nAYH/vv4SAob+KAD1ANT+NwAfAav+DwBpASL+MwEpAIn+ugEz/7f/qwBw/34Aof/M/w8B3P6t/0ICpv0hACACgf23AQz/tv9AAsr88wFNABX+mAK1/aYAsQEB/UoCl//g/p0Bx/5RAHMA2P8O/yoBLwBr/rQBFf/5/+UArf4VAZz/j/8JASr/9v/LABT/WADmAOD9KQJq/yT+zgNN/BsBIwJt/NMCzv6J/70BPv5XACQBxP7n/8YBnf00Ad4A1P0UAgP/m/8fATb/VADT/2n/fAGw/479HwOT/239HAM7/kAADAHB/YcC7f4t//4A9P+sAHn9WAJZAH/9kwJl/j4AdAFh/UUCn/8M/uECg/4f/0ECCv4/AOIBWf1vAUgAkP5SAun9hv81A7/9e/4dA+f+Yv/r/00AxgHu/EYBAQGP/o8BQf67//8CSv6j/RoDyP/w/hwAov9pAVL/m//Z//b/QQLL/dX98QSQ/sf8lAKlAHn/nP4mAK0CWP6d/UsDPQAU/dABKgA9/5UBDf3PAZoCX/o/A5UB6vxXAfT+oAIp/jD+uANO/s/+BwIy/icCwf7g/MQHJPmYADoFl/jBBoX8Hv67BbX6wgIG//T+6AM+/Pf+WQS1/ib9FgKb/z4Cpv3j/KYGYP3s/YUB2v7/BNn5mv+IB8z4JALnADH+MARw+eYDtwIU+uEDcP3zAjoBUPcACSwAPfdlCWL7Bv+VBiv0CAw//vXxHRQu9vX6ygsm9McJP/2K91INGPnU/p8Cw/0WBfT55QGBAHQC4fy++5sNqfDxBgcGMvFcDsX4BP9wB6v1ggbkAn/3XAMbBXj7vvwcBvz9Kv/bAT/5pgoN/YL0uQ/b+vL3ig0B9SgCsQrU7BsOuQGE8SgPbvd3A9b/F/sBCfH35AUB+vkCHwaB8soMufkQ/dcLDfIfB9wBePm0BrT6CAUv+/D/8AiK8j0JcP8o+A4ORPHpBToKfuqJDwMChfQrCLf5wQir/PL3JQqX/IMAMf6JAI0BJf39BzPzYAR1Dh/qjAgbCF70zAff+nH9wRDk7AT+Phpy5XcHSAYh9kEQSOsWBtEQm+86+UMRJ/5w8kELAvzpAhgAr/XnEDD4DfOUGWTxxPVYGQvpJAvTBrLkFiHm8iXwoxwf65gEiArs7AgQFP0L9/wJ5fxU/vz/iQQN+ZoFQf5P/T4HyvMQD1z1H/6fDjvuFwr3/9v8aP4wBl39N/lmDdLvzxSk6uoA5CTOx0UfUg911q0kSu4i+5EbH9sZFnQEJerMGdPoyw0IBQ7nOSGi5RcHIRWW0j4rVvc74dgr/d+dC3wTe8fnOUH/P8KIQNvqRfRVFBbmvR5Y6YEE8QuL6y0Sf/gkAL38hggI+JMGVvvK9kQfqtqJFHz+zffrEh7ewSXi6qf92Q7X6xsd0eG+CYUM5vTn/c3/+xL03g4YEgaZ4Nkcsfjq9mILEfwU+g8KtPhAARELe+PsImXsW/U8LRu9nzAeCUfEez+q3i0BcRrdzgYtSfxV2uYjs/xi8VIHxf84AuUAiPemAj8QXeri+/UnVtXx/lEyJ8nlD+sXiNgTJf7q4/ZdIqPfmgxW/6j9Dgy859ATKP6o+1H4GQ7m//TrRCE31NsuSeip5+pGtqAMVvvh/MTle4OHHkH6Bc+x/38XkhkWMj7AtJUjWPj+CPv4T/p2/0sUXfsz0c1BEuhc6SoeD+ecIdTeLf8dL6LAHSWlBC7bgDr5xPAMQTlTspQVJzFJvSodpBRL0fkphu3L9jsZD+4gALECYAir8iUJaQK24fA4SdyM3QFVP76FCMMhSsSdTS/PDde1bT6vf/1SOE+5IkFB5Grdx0cKxusTDAqB6e0Sifku/3z4wxu91HkgSwxGw0o/Zu6J6ksY6+9ZAjcmj7/wBO5dhpoiDfslhvQIC2nBBjsjDeTjGeMfE1pKc6F+/5Q/m+IaAHzubQK8LurdW93BMOIF1+Hw+BQet/+D5PsLDghEBADxTfeaHojwPPYLE0nm6hs0+I3vdRgB2xs1O+Rh4bA0dd+RIeDKfBIRQO6kTx5VFsr0XAaF1L0nCyXXrKsfbS7Xt9Q2RubP7ytAUrOVHXAikMqlG5D/u+6kJavXgf/JM/PNDfzVItL8QOB7C1QaQOZ0CWPuiRa9Eju2ST7cF6at4ijZGCHqGfzE9IMJeioC1HTR32j9yfHsli/NsxBr2sAI0rJtZaZ6LFD9ktUyPP7iUPz7+dEY2PwC3mca5wb7BwHLFyT5IIfD/y0xzZ0viRD5n3By98GO6ylKM74OIoXqiv9aKyHVOvFBKGEHJ84DG/cMm/KvA6jlfSkrA9/E1iznEfPPXCO16lgGQR/vuMlFjOsU2GZIZrqPKYoFr8ufQN3ZEQESHwXP3x3iDfzk0QDYC8gGv+pfEgLxKgtdA8LpNiAP5LgMIAUW7MwbY+pIBfIC5v3ZDIzbZC+f4lD12DM8vEIxH/Fl7XwvY8ouE5UYLOW0+08SE/gcBxzw+gOtEtDxZv2p7I5CML06BJM9K6X0WanGwfUoTYGcUkvq4hvpBEInt7gfaROM348OLAGE6wAwFNt43lxfEazjF1IdgsHXUBq74A64Ma63AC1T9ZH6FBP23oYYkgVf5S4WS/xb9vUI8v0GAnj3pgwS8v8R7ekTDSIPjdKlNmXUmhgtBarVpjrP1SEUePSn//8Z7tTpI97lniCR5NnoB1l2mucsxxwvulhF+9pZ9RYpneXz7TArEegu8eAjqOCQET34dQKSAEQAyP6p9dkiVdAQHHMKYdoIKirixgNWHpTR7RbEE0zVvyd25O0FTxtnzu4peu5m+8IWsN0JGYQHTdgxKwDwruzHK1nLXSlD+ArgtTSC0VsXXwVw4dknEOf7/BUTG/Rp+jkQjPK2/fAXntsAG98D39hRL6n06tcyOm/fePc8KtzDWDLw9kHZnznt2MoFtBMC4EIdm/Yy8wMRRfYDDZzpkw0nDp3cBiOH7MP8DBwu2b0ShxP51xAbJgOd36k018vNFsEUZ8oXQj/EeiACDGrLxUIWzq4M+BqS0sUc1ghp3pUXSwxJ1oMk0wBr2dstS+0D7OgpktwECKcY5NR4IIP94uxCFVvxKgU2BR77d/d5DHwGJOZqFNf/hvOGEyLiihqQBgnXECNKAQ/qEw6v/4Xz+h334Tv1SD0qwAYN/im0v0c2wetj48I/AsjhDSEZxdoXGf368foTAHwFbwCE9IURAvHdABwSe+yv+m4dX+nX95UeKePyDoL+6vEeHsrhhgmoC1LuDBGw7Z0MlQUl6ysQLwHb9wD+bwyW9qT+SQrR7pMVEPIw9eMj6N08BFUU5O7mBW/3jgl6BajzR/u3D7EAG+7/DSz5jQhB/vrs/B0/8Bn31RWM7gUEtQkM7r0ORALT5hMn7d/HA+gfxsnCMQTrifHqIrjewBPe/H3zjA4rBEnqcgv1EYbcUhzB+CfwnySn0VYdNgyg0/Un7vCa/ZUNSefRE+sG4eRCER0ICer9EsT2I/0WEdHnlA2KBh/wzwoe+wkCuQTm9LkGPQJC/UL/7Pz/CBP53gKe+20BWw7U4/wVb//u7DobguscAVIPjfN8/a8Ht/8X/ZT+mf5NDPryjP/pCev4TASN+EUH3f6Z/jf9uAQGAprzEhEx8kYImfvI/l4LQOyHEW750P0uBIP6OQmc9uYCggQb99cLZ/VZ/1sQoOqkC3MASfqYCfvuYhMT+HH2SRPB7EUNPgB57jsXx/TS+jcOJu9GDAgLine truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABBIAAAR4CAYAAAC7N/X6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3X+87XVdJ/rXmyOoV/Tyy+wEmFqU\nMVrYPaFd5ypSGlY3KR1HmhTNG9nooyxzUKcRM0m716JMs6ERgXL8EaUyDg75A2O6iXpMRPwJouUR\nFBFFUcPY+z1/7LX3WR7PXmsdOHt999o8nz6+j72+n893re/bxV/nvV7fz6e6OwAAAACzOGDoAgAA\nAIDFoZEAAAAAzEwjAQAAAJiZRgIAAAAwM40EAAAAYGYaCQAAAMDMNBIAAABgAVTVOVV1fVVduc78\n/avq3VV1S1X95h5zJ1XVx6vq6qp6ztj4favqPVV1VVW9vqoOmlaHRgIAAAAshnOTnDRh/sYkv5rk\npeODVbUtySuSPDrJsUlOqapjR9O/l+Ss7j4myZeSPHVaERoJAAAAsAC6+9KsNAvWm7++u9+X5F/2\nmDo+ydXdfU13fzPJ65I8pqoqyYlJLhhdd16Sk6fVcafbUvxWs+3gu/WdDjts6DIAAAAW2q033pil\nm79WQ9exv/3EI+7WX7xxacPv8/4rbvlwkn8eGzq7u8/eDx99ZJLPjJ3vSvLgJIcn+XJ33zo2fuS0\nD9NISHKnww7Ldz3rmUOXAQAAsNCu/f0/HLqEDfHFG5fy3ovvveH32bb9qn/u7h0b8NF7a+70hPGJ\nPNoAAAAAW9uuJEePnR+V5NokNyQ5pKrutMf4RBoJAAAAMEEnWZ7D/zbQ+5IcM9qh4aAkT0hyYXd3\nkkuSPG503alJ3jztwzzaAAAAAAugql6b5IQkR1TVriRnJDkwSbr7T6vqO5PsTHKPJMtV9cwkx3b3\nV6rqGUkuTrItyTnd/eHRx56e5HVV9aIkH0jyqml1aCQAAADARJ2l3tDEwGxVdJ8yZf5zWXk8YW9z\nFyW5aC/j12RlV4eZebQBAAAAmJlEAgAAAEywskbC1M0M7jAkEgAAAICZSSQAAADAFBu8q8JCkUgA\nAAAAZiaRAAAAABN0OkttjYRVEgkAAADAzCQSAAAAYAq7NuwmkQAAAADMTCIBAAAAJugkSxIJayQS\nAAAAgJlJJAAAAMAU1kjYbWETCVV1l6p6b1V9sKo+XFW/PRo/t6o+VVWXj47jhq4VAAAAtopFTiTc\nkuTE7r65qg5M8ndV9dbR3LO7+4IBawMAAGCL6CRLLZGwamEbCd3dSW4enR44OvyXBQAAgA20sI82\nJElVbauqy5Ncn+Rt3f2e0dSZVXVFVZ1VVXde572nVdXOqtq5dPPX5lYzAAAAi2d5DseiWOhGQncv\ndfdxSY5KcnxVPSDJc5PcP8mPJDksyenrvPfs7t7R3Tu2HXy3udUMAAAAi2yhGwmruvvLSd6V5KTu\nvq5X3JLk1UmOH7Q4AAAAFlqnszSHY1EsbCOhqu5ZVYeMXt81yY8n+VhVbR+NVZKTk1w5XJUAAACw\ntSzsYotJtic5r6q2ZaUh8obufktVvbOq7pmkklye5GlDFgkAAMCC62RpcQIDG25hGwndfUWSB+1l\n/MQBygEAAIA7hIVtJAAAAMA8dBZrV4WNtrBrJAAAAADzJ5EAAAAAE1WWUkMXsWlIJAAAAAAzk0gA\nAACACTrJsl0b1kgkAAAAADOTSAAAAIAprJGwm0QCAAAAMDOJBAAAAJigI5EwTiIBAAAAmJlEAgAA\nAEyx3BIJqyQSAAAAgJlJJAAAAMAE1kj4VhIJAAAAwMw0EgAAAICZebQBAAAAJuhUlvwOv8Y3AQAA\nAMxMIgEAAACmsP3jbhIJAAAAwMwkEgAAAGAC2z9+K4kEAAAAYGYSCQAAADBRZan9Dr/KNwEAAADM\nTCIBAAAAJugky36HX+ObAAAAAGYmkQAAAABT2LVhN4kEAAAAYGYSCQAAADBBt10bxvkmAAAAgJlJ\nJAAAAMAUy9ZIWCORAAAAAMxMIgEAAAAm6CRLfodf45sAAAAAZiaRAAAAABPZtWGcbwIAAACYmUYC\nAAAATNBJlnPAhh/TVNU5VXV9VV25znxV1cuq6uqquqKqfng0/oiqunzs+OeqOnk0d25VfWps7rhp\ndXi0AQAAABbDuUlenuT8deYfneSY0fHgJK9M8uDuviTJcUlSVYcluTrJ34y979ndfcGsRWgkAAAA\nwBRLXUOXkO6+tKruM+GSxyQ5v7s7yWVVdUhVbe/u68aueVySt3b3129rHR5tAAAAgM3hiKraOXac\nto/vPzLJZ8bOd43Gxj0hyWv3GDtz9CjEWVV152k3kUgAAACACTqVpfn8Dn9Dd++4He/fW2yi1yar\ntid5YJKLx+afm+RzSQ5KcnaS05O8cNJNJBIAAABga9iV5Oix86OSXDt2/vgkb+zuf1kd6O7resUt\nSV6d5PhpN9FIAAAAgCmW+4ANP/aDC5M8abR7w0OS3LTH+ginZI/HGkYphVRVJTk5yV53hBjn0QYA\nAABYAFX12iQnZGUthV1JzkhyYJJ0958muSjJT2ZlV4avJ3nK2Hvvk5W0wt/u8bGvqap7ZuWxiMuT\nPG1aHRoJAAAAMEEn81ojYXId3adMme8kT19n7tP59oUX090n7msdw38TAAAAwMKQSAAAAIAJOpWl\n3tuGCHdMEgkAAADAzCQSAAAAYIplv8Ov8U0AAAAAM5NIAAAAgAm6k6X2O/wq3wQAAAAwM4kEAAAA\nmKiyHLs2rJJIAAAAAGYmkQAAAAATdKyRMM43AQAAAMxMIgEAAACmWPI7/BrfBAAAADAziQQAAACY\noFNZbrs2rJJIAAAAAGYmkQAAAABTWCNhN98EAAAAMDOJBAAAAJigkyy33+FXLew3UVV3qar3VtUH\nq+rDVfXbo/H7VtV7quqqqnp9VR00dK0AAACwVSxsIyHJLUlO7O4fSnJckpOq6iFJfi/JWd19TJIv\nJXnqgDUCAACw8CpLczgWxcI2EnrFzaPTA0dHJzkxyQWj8fOSnDxAeQAAALAlLfQaCVW1Lcn7k3xv\nklck+WSSL3f3raNLdiU5cp33npbktCTZduihG18sAAAAC8kaCd9qob+J7l7q7uOSHJXk+CQ/sLfL\n1nnv2d29o7t3bDv4bhtZJgAAAGwZC51IWNXdX66qdyV5SJJDqupOo1TCUUmuHbQ4AAAAFt4irWGw\n0RY2kVBV96yqQ0av75rkx5N8NMklSR43uuzUJG8epkIAAADYehY5kbA9yXmjdRIOSPKG7n5LVX0k\nyeuq6kVJPpDkVUMWCQAAwGLrLmskjFnYRkJ3X5HkQXsZvyYr6yUAAAAA+9nCNhIAAABgXpYkEtb4\nJgAAAICZSSQAAADABJ1k2a4NayQSAAAAgJlJJAAAAMBEZY2EMb4JAAAAYGYSCQAAADBBJ1luaySs\nkkgAAAAAZiaRAAAAAFMs+R1+jW8CAAAAmJlGAgAAADAzjzYAAADABJ2y2OIYiQQAAABgZhIJAAAA\nMMWy3+HX+CYAAACAmUkkAAAAwATdyZI1EtZIJAAAAAAzk0gAAACAKezasJtEAgAAADAziQQAAACY\noFNZbr/Dr/JNAAAAADOTSAAAAIAplmKNhFUSCQAAAMDMJBIAAABggo5dG8ZJJAAAAAAzk0gAAACA\niezaMM43AQAAAMxMIgEAAACmWLZrwxqJBAAAAGBmGgkAAAAwQXey1LXhxzRVdU5VXV9VV64zX1X1\nsqq6uqquqKofHptbqqrLR8eFY+P3rar3VNVVVfX6qjpoWh0aCQAAALAYzk1y0oT5Ryc5ZnScluSV\nY3Pf6O7jRsfPjI3/XpKzuvuYJF9K8tRpRWgkAAAAwBTLfcCGH9N096VJbpxwyWOSnN8rLktySFVt\nX+/iqqokJya5YDR0XpKTp9WhkQAAAACbwxFVtXPsOG0f339kks+Mne8ajSXJXUafeVlVrTYLDk/y\n5e6+dS/Xr8uuDQAAADBBp7I8wxoG+8EN3b3jdrx/b0X26O+9u/vaqrpfkndW1YeSfGXC9euSSAAA\nAICtYVeSo8fOj0pybZJ09+rfa5K8K8mDktyQlccf7rTn9ZNoJAAAAMAUy6kNP/aDC5M8abR7w0OS\n3NTd11XVoVV15ySpqiOSPDTJR7q7k1yS5HGj95+a5M3TbuLRBgAAAFgAVfXaJCdkZS2FXUnOSHJg\nknT3nya5KMlPJrk6ydeTPGX01h9I8p+rajkrgYKXdPdHRnOnJ3ldVb0oyQeSvGpaHRoJAAAAMEEn\n81ojYXId3adMme8kT9/L+N8neeA677kmyfH7UodHGwAAAICZSSQAAADAFMvtd/hVvgkAAABgZhIJ\nAAAAMEnXplgjYbOQSAAAAABmJpEAAAAAE3SS5UgkrJJIAAAAAGYmkQAAAABTWCNhN4kEAAAAYGYS\nCQAAADBBRyJhnEQCAAAAMDOJBAAAAJhCImE3iQQAAABgZhIJAAAAMEGnJBLGSCQAAAAAM5NIAAAA\ngCmWI5GwSiIBAAAAmJlEAgAAAEzSdm0YJ5EAAAAAzEwiAQAAACboSCSMk0gAAAAAZiaRAAAAAFNI\nJOwmkQAAAADMbGEbCVV1dFVdUlUfraoPV9WvjcZfUFWfrarLR8dPDl0rAAAAi6tTWe6NPxbFIj/a\ncGuSZ3X3P1TV3ZO8v6reNpo7q7tfOmBtAAAAsCUtbCOhu69Lct3o9Ver6qNJjhy2KgAAALaiXqDE\nwEZb2EcbxlXVfZI8KMl7RkPPqKorquqcqjp0nfecVlU7q2rn0s1fm1OlAAAAsNgWvLine truncated
+      "text/plain": [
+       "<matplotlib.figure.Figure at 0x7f5aeb66bb38>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "tts(model, sentences[2])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# os.makedirs(OUT_FOLDER, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "# wav, alignments, spec = create_speech(model, sentences[1])\n",
+    "# out_path = os.path.join(OUT_FOLDER, 'speec_{}.wav'.format(1))\n",
+    "# with open(out_path, \"wb\") as f:\n",
+    "#     f.write(wav)\n",
+    "# print(\" > Speech saved : {}\".format(out_path))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import IPython\n",
+    "# import glob\n",
+    "\n",
+    "# # wav_files = glob.glob(OUT_FOLDER+'/**/*.wav', recursive=True)\n",
+    "# # assert len(wav_files) > 0\n",
+    "# # IPython.display.Audio(wav_files[1])\n",
+    "# IPython.display.display(IPython.display.Audio(wav, rate=CONFIG.sample_rate))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

config.json

@@ -1,8 +1,8 @@
 {
   "num_mels": 80,
-  "num_freq": 1024,
+  "num_freq": 1025,
   "sample_rate": 20000,
-  "frame_length_ms": 50.0,
+  "frame_length_ms": 50,
   "frame_shift_ms": 12.5,
   "preemphasis": 0.97,
   "min_level_db": -100,
@@ -12,11 +12,11 @@
   "text_cleaner": "english_cleaners",
 
   "epochs": 2000,
-  "lr": 0.001,
-  "lr_patience": 2,
+  "lr": 0.003,
+  "lr_patience": 5,
   "lr_decay": 0.5,
   "batch_size": 256,
-  "griffinf_lim_iters": 60,
+  "griffin_lim_iters": 60,
   "power": 1.5,
   "r": 5,
 

debug_config.py

@@ -20,6 +20,8 @@
   "power": 1.5,
   "r": 5,
 
+  "num_loader_workers": 16,
+
   "save_step": 1,
   "data_path": "/data/shared/KeithIto/LJSpeech-1.0",
   "output_path": "result",

layers/attention.py

@@ -73,7 +73,8 @@ class AttentionWrapper(nn.Module):
             alignment.data.masked_fill_(mask, self.score_mask_value)
 
         # Normalize attention weight
-        alignment = F.softmax(alignment, dim=0)
+        alignment = F.softmax(alignment, dim=-1) ## TODO: might be buggy
+        print(alignment.size())
 
         # Attention context vector
         # (batch, 1, dim)

models/tacotron.py

@@ -2,7 +2,7 @@
 import torch
 from torch.autograd import Variable
 from torch import nn
-from utils.text.symbols import symbols
+from TTS.utils.text.symbols import symbols
 from TTS.layers.tacotron import Prenet, Encoder, Decoder, CBHG
 
 class Tacotron(nn.Module):

notebooks/PlayGround.ipynb

@@ -0,0 +1,354 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Populating the interactive namespace from numpy and matplotlib\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "import os\n",
+    "import sys\n",
+    "import io\n",
+    "import torch \n",
+    "import numpy as np\n",
+    "from collections import OrderedDict\n",
+    "\n",
+    "%pylab inline\n",
+    "rcParams[\"figure.figsize\"] = (16,5)\n",
+    "sys.path.append('/home/erogol/projects/')\n",
+    "\n",
+    "import librosa\n",
+    "import librosa.display\n",
+    "\n",
+    "from TTS.models.tacotron import Tacotron \n",
+    "from TTS.layers import *\n",
+    "from TTS.utils.data import *\n",
+    "from TTS.utils.audio import AudioProcessor\n",
+    "from TTS.utils.generic_utils import load_config\n",
+    "from TTS.utils.text import text_to_sequence\n",
+    "\n",
+    "import IPython\n",
+    "from IPython.display import Audio\n",
+    "from utils import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def tts(model, text, CONFIG, use_cuda, ap, figures=True):                                                                \n",
+    "    waveform, alignment, spectrogram = create_speech(model, text, CONFIG, use_cuda, ap) \n",
+    "    if figures:                                                                                                         \n",
+    "        visualize(alignment, spectrogram, CONFIG)                                                                       \n",
+    "    IPython.display.display(Audio(waveform, rate=CONFIG.sample_rate))  \n",
+    "    return alignment"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ROOT_PATH = '../result/January-26-2018_09:10AM/'\n",
+    "MODEL_PATH = ROOT_PATH + '/checkpoint_27600.pth.tar'\n",
+    "CONFIG_PATH = ROOT_PATH + '/config.json'\n",
+    "OUT_FOLDER = ROOT_PATH + '/test/'\n",
+    "CONFIG = load_config(CONFIG_PATH)\n",
+    "use_cuda = False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# load the model\n",
+    "model = Tacotron(CONFIG.embedding_size, CONFIG.hidden_size,\n",
+    "        CONFIG.num_mels, CONFIG.num_freq, CONFIG.r)\n",
+    "ap = AudioProcessor(CONFIG.sample_rate, CONFIG.num_mels, CONFIG.min_level_db,\n",
+    "                    CONFIG.frame_shift_ms, CONFIG.frame_length_ms, CONFIG.preemphasis,\n",
+    "                    CONFIG.ref_level_db, CONFIG.num_freq, CONFIG.power, griffin_lim_iters=80)         \n",
+    "\n",
+    "if use_cuda:\n",
+    "    model = torch.nn.DataParallel(model.cuda())\n",
+    "else:\n",
+    "    cp = torch.load(MODEL_PATH, map_location=lambda storage, loc: storage)\n",
+    "    \n",
+    "    # remove DataPatallel wrapper\n",
+    "    new_state_dict = OrderedDict()\n",
+    "    for k, v in cp['model'].items():\n",
+    "        name = k[7:] # remove `module.`\n",
+    "        new_state_dict[name] = v\n",
+    "    cp['model'] = new_state_dict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model.load_state_dict(cp['model'])\n",
+    "# model.decoder.eval(); ##\n",
+    "model.encoder.eval();\n",
+    "model.postnet.eval();\n",
+    "# model.eval();"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "sentences = [\n",
+    "    \"I try to speak my friend.\",\n",
+    "    \"I speak more than binary any more.\",\n",
+    "    \"I try ti implement a new TTS system.\"\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# tts(model, sentences[2], CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(13099, 3)\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "df = pd.read_csv('/data/shared/KeithIto/LJSpeech-1.0/metadata.csv', delimiter='|')\n",
+    "print(df.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "torch.Size([32, 10])"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "a = torch.randn(32, 10)\n",
+    "b = torch.nn.functional.softmax(torch.autograd.Variable(a), dim=-1)\n",
+    "b.size()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Variable containing:\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       " 1.0000\n",
+       "[torch.FloatTensor of size 32]"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "b.sum(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "torch.Size([1, 156])\n",
+      "torch.Size([1, 156])\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRrgRAABXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YZQRAADQ+/n7J/zB9+T4svBL+cX2Uf0nAHz0HgksCOANvRBEAusG7fuk+yX+gvqA+w7xYv8K/PL8ZAkM+y8EjACDBN0Fwf/dBbEBdgZzBIgEiAd0CEMC+PtZ+Hz54v2c/fT1N/vi/1j99/l89xD4A/UA9BXxxPMF+XbzDvM685T2TfyR9ZPw6O2R62j2TfSU+rj9avwS/izuwPP6/az6BwDM/VoCfxFbEdASQwz6DYogwyByHfQXvA0ZEh4TvAcLF50UgRnvHL4UaxxxFIwZWw3BE4ApvhzhG1EfliNWIhEWDBQ4BdwADgPl+h8CvRZvH38nmi7aIIcZbxsGF08NExdGGVULSQk28Xz/GhEJ+RDvKOxH8LHo+d5s2LPGk8yA6HjsM+ht7CjaO9KK3UDIvsyzx/HBuNKi7rnvG+Dn9Enpbupi9dziXNRe2vH9WA/eEvX+/v00+Qn5KAdbx1HKjudA4E/XQLjDt9a0walMzYDiYuP+6YL7uwg4IloHwRnWPB0lQii8OWtlU18nUdxYK1wMWXBRaSnmHhkbQRjlFwD3u/am/frj3tto3U/Nd6/hrwGAHpsDxB7Cre3Qx6/Plb6B0uLhMPKw8BXjwPDb7/7y4+a551bfTP3ABDz/yP6uFRMzzydkIdgnNi1LNucvJTHTMC0Y8C00HzskTz2XHzoqtSAmCBQMxP6FB0oCn/lm9Gf75PgJ/N8J5fMP7Wn4HQR+/E31SuK74sHo7PAq75TuBu4z7g/uQOiE9M71UP31+SX7w/1r90/3Cv798pLtEPdz9uwAtPuH/RT7RgalBqQJ2gmV/jgGcgG3CO0KBAoVCg8LygIODUwNmA/RDUoJsAucB7sF0QliDj4H/gcTAxIInAf+Btr/gQKrBH0GdxG6BC8FTgCHA7ADPwCh/cz65/w8/1r9D/kG+7L2SfuW+pT9avhW+qj3C/fm+Z/0H/iJ9xX37fJR9n317ffS9U/zU++u7q/ypfR69VX1I/e990r8U/dl91f3TPkh+sT4fPrF+EX5mPiy+Fj0pvgC/PT9efv8+iT6Pf4Q/8P7Yf+c+Z/79f1l/Kj59/gn+Cz5hfvB/oP9tf1PAEMByAKlADUCXv7gADEBVwI1AoUCDQV8BFcFpQKvBBcFcAQEBukD4wOZBB0BmAFnAPX+AQFV/s3/Ev8u/zYBmwBgAJv/LAFZAkIC+QEiAY3/pgEUANcBL/4EANcBwgG5AgkBdwHgAr0CMAPYAlkBSAFdAYkBJAB8/0L/ZAB1AhMBTAGJAI4AQgHJAegAOQGPAGQBegKnAQsCGQFdAvMB/gC5ALwA8QBsAF3/xP8oAHkArP+Z/1z/IP4k/6z+GP6E/WT8UP3y/Mn9wv30/er9cP4B/w3/a/9TAC4BuAEzAb0B0QBmANcBUAH3AAMBDwGtAX4AdgDp/xAA0wA+Aa4AGf+g/3X/vv1U/vn8Gv2H/UL9hf6Q/Xb96/42/08APv9//6X/uf+IAFcAAAG2AGwAbAClAI8B7wDxAOEBkgCYAQMBewAEAVkA4wB7Ad0APAFkAL3/8f+uALgAPAEoArABIQINAhUCFwIqAqYCrgJGAy8DVQP3Au0ClQLWAZUCEwIKA9oClAKPAuoCogLwAY0BrQG8AWICXAJzAusCKgISAqYCAQPRArkCVgJkArgBvQHfAdoB+gG+AQgCkQGkAfEBPgGPABMAYQAKAYcAmABAAEYAYABJAKL//P4w/7//Ef/X/s/+QP4M/pL+CP6K/vP+xP5E/qr+P/4z/vz9N/3H/UT+f/5h/oP++/3p/Wj+H/6c/dH9Sv48/rv9aP4s/lb+M/6Q/mb+8/4R/k7+o/7z/q3+2v7x/jv/n//d/gL+lP4H/oL+TP5d/vn9AP7d/UL+ef5Y/nv+Zf5s/gH+pv0//hT+zP45/xX/Lf/M/o3/ff9y/97+G/4N/hH+m/4k/9v+MP+u/s3+P/6w/T3++v5v//H+CP/8/nv+R/5f/+MAvgDxAOUAywBIAF8A0f/+/oL/Sf/u/6L/Uv9M/o3+bP6E/rP+c/5B/pX9O/7c/lQAswAmAbIAR/8TAD4BGgJ7AcUAXgGWASEBagDbAD0A8gDEACsBhQHjAe4BdQGxAIMAyf/O/10AsQC4Ad0BuwH4AQICSQEPAYMAy/5i/pP9Vf3u/F79Hv3G/G795/25/XH9rf0y/sj+6P4i/9L+lf6z/lH+Wf6z/sH+pv3j/yr/owCCASsBJgH6AYUC+AGDApoCKwKqAesBtAH5AU0CKQIHAl4BXgEWAt8A1AF3AvUC4QJ9ArgCqASMBLgDvgPCAwcEvgIpA1MDZAN7AggDpQJtArgB9wH8Aa8BNgHKAG4BJgKfAUIBKQENANv/sP7z/un+Sv/r/tD+j/65/hr/uP/3/3b+of45/qD9Ef25/RD+Yf4b/+T+4f4dAGD/UP8YAHIAtgD4APUAXQFxAScC1gLOAeoBNQJWAlcC0wHrAZYBrAGsACEAf//O/xcAJwFcATMBgQECAUkAQgA0/3L/d/+v/3QAhABWAP3/VwCeAPoAWAEkAlIBiwHQAQwCEQLkAYgCSAILApsB9AEDAp0BvQELAssBswGtATMCMAJXAnEC1wHvAVECpAGFARsBCQKTAacBYwFmAYkA9QCXAMcAeADaAFQAEQBN/z3/cf9mAH0AtP9vAO8AfgFkAQAB9AATAT4BkAG9AX4BiwBAAMD/FQBwALv/1f/2/5wApwD9ALgAhQBrAewAiQHVAM4AQQDH/y4AaABHAVUBawFrAb0A8gBqASEBCQH9ANsAUgFeAcIB8QEqAnIBugGcApkCGgJSAukB0wHAAaUCRQLsAX0CEALiAbIBawHfAU8CxwIoAzUDJwNnA6AD1AN+BLEEbAQ7BKMErQSHA9IDlQOZAwwEFQRKBFIEAAQAAx4DJgO1ApkC1AHxAVkClgIxAn8CoAK6AusChgP7AzYEiQSwBBQFZwQRBHgExQTwBFMEBgMEAw8DqgOtA7ICggJwArAB1wGWAZMBuwGGAokCVgJ1AiACdgJBAgYCLQLeAbYBiwGpAN0A9QBkAUUBNQGUAagB5wBFAQwBlQDSAFUBCAE2AfYAlQCsACMAIgDM//j+8v5B/8H+dv6A/ij+qf7i/eP9E/4P/hT/C/+Z/v3+gv8PAH8AWgDC/37/Sf93/93/bP/l/p3+av5u/iv+ff5N/nf+Hf70/ZD92/0y/vD99/2I/QP+y/33/aP+Pv5m/cL90v0j/gX+2v5y/lD/ov8gALT/sv/h/ur+1f4K/0T/R/+K/5r/GQCmANL/hP/c/j7/vv77/o//Rv+Z/7z/KAAqAP//cQCtAMgA//95/zL/IP9x/5//lP/Y/8n/O//h/iL/6//w/yUAFwAJAAAA6wCdAEQARwHhAA0B9AC2AKkA4QAsATABOAGEAZ4AowCNAKkA7gDlAGYALgBYAIsA8P9H/2f/vP65/lv/jv/5/9v/7v+UAJcAIwBOALr/YQCFACcAEgBhAEYA8f/r/nj+2P4f/1n/Jv/UALEAegApAMj/MwDJ/3H/hP/j/mP+R/7F/kf+pP7M/uf+K//t/jb/HwBX/3X+SP4j/hL+4P2g/k7+Ov4y/i7+Qv4G/y3/nP+z/9T/8P8yAIUANgCNANgArAHUARIChwKYAo8CZwJvAlsCWgJfAm4CYwKVAkEC9QEMAtwBPQKAAoICUAJDAaIBGwH/AO3/1QD+AKQATgBbADoAkf/W/gj/Nf90/vH+p/7M/jr+NP7//TT9wv1n/Xj9W/36/cv9H/4k/v/9Ff6W/Zv9JP4w/9L/lP/6/5T/eP/e/s3/5P/W/6//jAA6AAoAswBsAf4AGwHNAHYAkQDC/xsAwQDiAGYBJQGmAIMBGAG7AIYAXgDFAGsBWAGaALAAdgA/AagAxwBQAKQA8QD3/3gA+wDRAOUAQwG5AIoADgAUAOH/DgCDAHgAkgDHAF8AIwG2ARMCRQK3AaoBngDZACsAzv9h/7P+hf+N/6j/AgA6AFQApgA/AI7/vf9v/0n/SP94/yMAhv8r/5j/jf94/3X/qP9J/5P/DgA9AM8AiwAyAOL/BgAdAAkA2v9AAJf/rv/i/tj+S/8KAPb/yv/E/zcA0gD3ADQAOwB+AOoAIQHzAJkAFQBJAGwA2wAgAX0Av/9gAE8AigBkAO7/gP8UAFIAKQBEAJ0AYQByAJ4AlgCpAPUA0wA+Ab0A4wBVAbMBvAJXAq4BqwFFAagAWwCMAJMAHQD//0z/Hf/E/tf+X/6m/ub+7/6v/kT/Ov8b/7L+1P5N/73/o/8SAFsAJQAjAFYATwCZ/1r/Uf8a//H+7P5q/uf9Dv7O/bf9Pf4k/pr9KP4B/rT9fv3q/JD8evzF+4b73/qQ+ir6Afss+z77wfqw+sD62/pO+9v7FPwy/MX8kv2b/df93f31/UH+NP6P/qH9Of3U/Hr8Ufsr+8n6dfrJ+g779voj+636LftG+4n7lvtb/Kv8MvxF/OT7y/uv/H386PsC/Dv9zPxF/H38Uvy//If97f1X/mL+yP3r/f79sP4K/2L+zP3k/cb9Z/7b/qP/tv/i/4T/v//d/4//e/+0/6b/tP9WAIoA2QA6AMb/HP9Y/u3+E//r/v/+FP87/9//3f8KAFEAnwCUADUAxP/9/1AAgP/U/9b/TQBTAIwAFwHZALsAngE4AR8BNADP/6T/QgBoANr/rP+I/97/3//Q/8H/4f+QAHMAewCS/xAAlP+O/4H/Dv+X/tX+mf6f/lb++f7h/sv+o/4l/mb+zv1U/ab9if2//av+Wv4q/oz+kP41/jn+PP42/p3+nv4S/67/dP9+/14A4f/h/13/0//A//X/QAAMAagAvgCwAKUATwHnAMgA9QDSAE0AaAC7AMsA2wDdAFILine truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "For although the Chinese took impressions from wood blocks engraved in relief for centuries before the woodcutters of the Netherlands, by a similar process\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABBIAAAR4CAYAAAC7N/X6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3X+05XdZH/r3M2d+JpAEEoGYpCYK\nYgXbtZAC9roQQRCVGrsKJYqSq9zmqtBa7a2AXoRSKNL2VkEUbypI4FYBY9WxhvJDoFYrkCAKBFFC\n+DUJBJJACPk1c8557h9nz2QzObPPmcz5nu8+Z16vtb7r7P39fvb388yeWWvPefbzeT7V3QEAAABY\njx1jBwAAAABsHRIJAAAAwLpJJAAAAADrJpEAAAAArJtEAgAAALBuEgkAAADAukkkAAAAAOsmkQAA\nAACsm0QCAAAAsG47xw5gHuyuPb03p44dBgAAwJZ2Z27Lwb6rxo5jo333d57aN928NPg87//gXW/t\n7icPPtEJkkhIsjen5tE7vmvsMABgtlJICMB8e+/S28YOYRA33byU97317ww+z8LZHztr8Ek2gETC\nYd1jRwAAs/Xw34QAAPfUSZazPHYYc8NXGwAAAMC6qUgAAACAmTpLrSLhMBUJAAAAwLqpSACAraK2\nXRNsALabbdp6bqVHwjb9w90LKhIAAACAdZNIAAAAANbN0obDlIsCAABwDLZ/vJuKBAAAAGDdVCQA\nAADADJ3OUmu2eJhEQpJUpXbuGjsKAJipFhQSAjDf6k5Lxk8GEgkAAACwBts/3s1XGwAAAMC6SSQA\nAADADJ1kKT34sZaqenJV/U1VXVNVz1vl+p6qetPk+nur6vzJ+UdV1V9Ojr+qqn889ZpPVtWHJteu\nWs/7YWlDkpy6N0uPeNjYUQDATMu75P8BmG/L73vX2CFsW1W1kORXkzwxyYEkV1bV/u7+yNSwZyX5\nYnc/uKouSvLyJE9P8uEkj+zuxao6O8lfVdUfdvfi5HXf2d03rjcW/yMBAACANSynBz/W8Kgk13T3\ntd19MMkbk1x41JgLk1w2eXx5kidUVXX37VNJg73JiTV8kEgAAACA+XBWVV01dVwyde2cJJ+Zen5g\nci6rjZkkDm5JcmaSVNWjq+rqJB9K8uNTiYVO8raqev9R8x2TpQ1JFs45mPu97NNjhwEAM1159deP\nHQIAzHTwmrEjGEYnWepN2bXhxu5+5DGurba35tFBHXNMd783ycOq6u8muayq3tLddyb537r7+qp6\nQJK3V9VHu/tPZgWpIgEAAADm34Ek5009PzfJ9ccaU1U7k5ye5ObpAd3910luS/LwyfPrJz8/n+T3\nsrKEYiaJBAAAAFjD8iYca7gyyUOq6oKq2p3koiT7jxqzP8nFk8dPTfLO7u7Ja3YmSVV9XZKHJvlk\nVZ1aVfednD81yZOy0phxJksbknQqB5e8FQDMt9q3NHYIADCbr6oHM9lx4TlJ3ppkIclru/vqqnpx\nkqu6e3+S1yR5Q1Vdk5VKhIsmL//2JM+rqkNZyVn8ZHffWFVfn+T3qipZyQ/8Vnf/97Vi8dszAAAA\nzNDpLJ3YRgcbE0f3FUmuOOrcL0w9vjPJ01Z53RuSvGGV89cm+fvHG4d8EQAAALBuKhIAAABglk6W\nxi9ImBsSCUmWD+zKHf/6AWOHAQAznX/aajs6AcD8uOnGsSNgM0gkAAAAwAydde2qcNLQIwEAAABY\nNxUJSequQ1n41A1jhwEAM+3ct3fsEABgph13LY4dwkAqS7HE8DAVCQAAAMC6qUgAAACAGTrJsl0b\njpBISJIdldq7Z+woAGCmPsXSBgDm3A7l/ycDiQQAAABYgx4Jd9MjAQAAAFg3FQkAAAAwQ0dFwjSJ\nhCRJJeUfBQBzzmcVADAHJBIAAABgDcstoX+YHgkAAADAuqlISJLFxSzf9MWxowCAmerWr4wdAgDM\ntrg4dgSD0CPhq6lIAAAAANZNRQIAAADM0Kks+R7+CImEJN2dPnhw7DAAYCafVQDMu15aHjsENoFE\nAgAAAKzBrg13U5sBAAAArJuKBAAAAJjBrg1fTSLhsOUeOwIAmG2H/8AAAOOTSAAAAICZKkutM8Bh\nc/9OVNVrq+rzVfXhVa79X1XVVXXW5HlV1Sur6pqq+mBVPWLzIwYAAIDtaytUJLwuyauSvH76ZFWd\nl+SJST49dfp7kjxkcjw6yasnP2eqqtSurfBWAHAyq50+qwCYb3Voey7D6yTL8/89/KaZ+3eiu/8k\nyc2rXPqlJD+blb/Twy5M8vpe8Z4kZ1TV2ZsQJgAAAJwUtuRXG1X1/Umu6+6/qvqqjNc5ST4z9fzA\n5NxnNzE8AAAAthm7NtxtyyUSquqUJD+f5EmrXV7l3KrbMVTVJUkuSZK9O09Lnfe1GxYjAAxh+b57\nxw4BAGbqq3ePHQKbYMslEpJ8Q5ILkhyuRjg3yV9U1aOyUoFw3tTYc5Ncv9pNuvvSJJcmyen7zrb3\nIwAAAKvqtmvDtC33TnT3h7r7Ad19fnefn5XkwSO6+3NJ9id55mT3hsckuaW7LWsAAACADTL3FQlV\n9dtJHpfkrKo6kOSF3f2aYwy/Isn3Jrkmye1JfnRTggQAAGBbW9Yj4Yi5TyR09w+ucf38qced5NnH\nO8eh03bm+u9+wPEHBwCbaOcdVuIBMN8WP7EwdghsgrlPJAAAAMCYOsnS1usMMBjvBAAAALBuKhKS\nLC8kB+87dhQAsIa2NhOAObdtv6q2a8M07wQAAACwbioSAAAAYIZOsux7+CMkEpLsWEz23qwTNgDz\nbeGusSMAgNlqeewI2AwSCQAAALCGJb2KjlCbAQAAAKybigQAAACYoVNZ8j38ERIJSRbu6px+7aGx\nwwCAmZZ3+Q8MAPNtxyG9504GEgkAAACwhuWW0D/MOwEAAACsm4qEJDtuP5h9f/GpscMAgJnq1H1j\nhwAAM+24Y3HsEAbRiR4JU7wTAAAAwLqpSAAAAIAZOpWlrrHDmBsSCUmyvJy+7baxowCAmfr228cO\nAQBmW9yeSxv4apY2AAAAAOumIgEAAADWsOx7+CO8EwAAAMC6qUhI0t3pQ9byAAAAnIjuHjuEQXQn\nS+17+MO8EwAAAMC6qUgAAACAmSrLsf3jYRIJSaoqtctbAcCcW1gYOwIAmKkW/bJ9MvDbMwAAAMzQ\n0SNhmncCAAAAWDcVCQAAALCGJd/DHyGRkKRP3ZtDj/ymscMAgJmW9uqRAMB8W/6zt40dAptAIgEA\nAABm6FSWWyPJw9RmAAAAAOumIiHJ8oMWc8fzvjR2GAAw0/XXnjV2CAAw08Grx45gOHok3M07AQAA\nAFtAVT25qv6mqq6pquetcn1PVb1pcv29VXX+5PyjquovJ8dfVdU/Xu89V6MiAQAAAGboJMs97vfw\nVbWQ5FeTPDHJgSRXVtX+7v7I1LBnJflidz+4qi5K8vIkT0/y4SSP7O7Fqjo7yV9V1R9m5Y+21j3v\nQSIhyaE7duXzH3zg2GEAwEx7b9fkCYD5Vos+qwb0qCTXdPe1SVJVb0xyYZLpX/ovTPKiyePLk7yq\nqqq7b58aszcrCYT13vMeLG0AAACAmSpLm3AkOauqrpo6LpkK4pwkn5l6fmByLquN6e7FJLckOTNJ\nqurRVXV1kg8l+fHJ9fXc8x5UJAAAAMB8uLG7H3mMa6uVe/R6x3T3e5M8rKr+bpLLquot67znPUgk\nAAAAwAzz0CMhK9UC5009PzfJ9ccYc6CqdiY5PcnN0wO6+6+r6rYkD1/nPe9BIiHJ3hvuyjf+0ifG\nDgMAZupT940dAgDM9LmbDo0dwnZ2ZZKHVNUFSa5LclGSHzpqzP4kFyf58yRPTfLO7u7Jaz4zabb4\ndUkemuSTSb60jnveg0QCAAAArGFp1VUAm2eSBHhOkrcmWUjy2u6+uqpenOSq7t6f5DVJ3lBV12Sl\nEuGiycu/PcnzqupQkuUkP9ndNybJavdcKxaJBAAAANgCuvuKJFccde4Xph7fmeRpq7zuDUnesN57\nrkUiIUmWlrN8y5fHjgIAZvvKbWNHAACzHVocO4JBdNc89EiYG94JAAAAYN1UJAAAAMAallQkHCGR\ncNjCwtgRAMBMtXvX2CEAwGw7xm1IyOaQSAAAAIAZOsnyyLs2zBO1GQAAAMC6qUgAAACAmUqPhCkS\nCUmya2fqQV8zdhQAMNPyffaNHQIAzNS36+dzMpBIAAAAgBk6yXLrkXCY2gwAAABg3VQkJFnatzNf\n+eazxg4DAGZa2u2bEADm2/InFsYOYTBLvoc/wjsBAAAArJuKBAAAAJihU3okTJFISLJw51Lu87Ev\njR0GAMy0eLpdGwCYbzsOLo8dAptAIgEAAADWsKwzwBHeCQAAAGDdVCQAAADADN3Jkh4JR0gkJMnB\nQ+lPXTd2FAAw08IOhYQAzLe6466xQ2ATSCQAAADAGuzacLe5/2qjql5bVZ+vqg9PnfsPVfXRqvpg\nVf1eVZ0xde35VXVNVf1NVX33OFEDAADA9rQVKhJel+RVSV4/de7tSZ7f3YtV9fIkz0/y3Kr65iQX\nJXlYkq9N8o6q+sbuXpo1QXenFxcHCR4AAOBk0d1jhzCITmW55/57+E0z9+9Ed/9JkpuPOve27j78\nm/97kpw7eXxhkjd2913d/Ykk1yR51KYFCwAAANvcVqhIWMuPJXnT5PE5WUksHHZgcu4equqSJJck\nyd6cMmR8AAAAbHFL0SPhsC2dSKiqn0+ymOS/HD61yrBVa2u6+9IklybJ6Tu/pnecIpkAwHyrPbvH\nDgEAZqobF8YOgU2wZRMJVXVxkqckeULfvRDnQJLzpoadm+T6zY4NAACA7aNj14ZpcLine truncated
+      "text/plain": [
+       "<matplotlib.figure.Figure at 0x7f5878081438>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "aling = tts(model, df.iloc[1, 1], CONFIG, use_cuda, ap)\n",
+    "print(df.iloc[1, 1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([ 1.], dtype=float32)"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.unique(aling)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# import IPython\n",
+    "# import glob\n",
+    "\n",
+    "# # wav_files = glob.glob(OUT_FOLDER+'/**/*.wav', recursive=True)\n",
+    "# # assert len(wav_files) > 0\n",
+    "# # IPython.display.Audio(wav_files[1])\n",
+    "# IPython.display.display(IPython.display.Audio(wav, rate=CONFIG.sample_rate))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/TestModelOnLJSpeech.ipynb

@@ -0,0 +1,188 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Populating the interactive namespace from numpy and matplotlib\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "import io\n",
+    "import torch \n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from collections import OrderedDict\n",
+    "\n",
+    "%pylab inline\n",
+    "rcParams[\"figure.figsize\"] = (16,5)\n",
+    "sys.path.append('/home/erogol/projects/')\n",
+    "\n",
+    "import librosa\n",
+    "import librosa.display\n",
+    "\n",
+    "from TTS.models.tacotron import Tacotron \n",
+    "from TTS.layers import *\n",
+    "from TTS.utils.data import *\n",
+    "from TTS.utils.audio import AudioProcessor\n",
+    "from TTS.utils.generic_utils import load_config\n",
+    "from TTS.utils.text import text_to_sequence\n",
+    "\n",
+    "import IPython\n",
+    "from IPython.display import Audio\n",
+    "from utils import visualize"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Constants\n",
+    "ROOT_PATH = '../result/January-26-2018_09:10AM/'\n",
+    "MODEL_PATH = ROOT_PATH + '/checkpoint_27600.pth.tar'\n",
+    "CONFIG_PATH = ROOT_PATH + '/config.json'\n",
+    "OUT_FOLDER = ROOT_PATH + '/test/'\n",
+    "CONFIG = load_config(CONFIG_PATH)\n",
+    "use_cuda = False\n",
+    "hop_length = 250"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(13099, 3)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# read dataset meta file\n",
+    "WAV_PATH = '/data/shared/KeithIto/LJSpeech-1.0/wavs/'\n",
+    "df = pd.read_csv('/data/shared/KeithIto/LJSpeech-1.0/metadata.csv', delimiter='|')\n",
+    "print(df.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ap = AudioProcessor(CONFIG.sample_rate, CONFIG.num_mels, CONFIG.min_level_db,\n",
+    "                    CONFIG.frame_shift_ms, CONFIG.frame_length_ms, CONFIG.preemphasis,\n",
+    "                    CONFIG.ref_level_db, CONFIG.num_freq, CONFIG.power, griffin_lim_iters=80)\n",
+    "\n",
+    "def compute_spectrogram(file_name):\n",
+    "    w = librosa.load(file_name, CONFIG.sample_rate)[0]\n",
+    "    spec = ap.spectrogram(w)\n",
+    "    return spec, w\n",
+    "    \n",
+    "def plot_spectrogram(spec):\n",
+    "    figure(figsize=(16, 8))\n",
+    "    librosa.display.specshow(spec.T, sr=CONFIG.sample_rate,\n",
+    "                             hop_length=hop_length, x_axis='time',\n",
+    "                             y_axis='linear')\n",
+    "    xlabel('Time', fontsize=16)\n",
+    "    ylabel(\"HZ\", fontsize=16)\n",
+    "    tight_layout()\n",
+    "    colorbar()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "file_name = df.iloc[1,0] \n",
+    "file_name = WAV_PATH + file_name + '.wav'\n",
+    "spec, w = compute_spectrogram(file_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRo7mBQBXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YWrmBQD0/x//zADD/rj/uQFFARwBYv+OAG0AiP/A/mUA3//z/93/4v9F//3+uf+3/zT/3P7c/27+rgDa/9wAjgGz/+kA5v56/4v/9v+XAC0BjQATAJQB/P+NAEH/Iv8gAfP+/AApACUAqQDr/sb/D/9m/+f/QQBuACoALADy/8/+vf97/y4BOAAgAGwAXwDx//3/8P89/7IAIf/3/2z/0AFEASYBfwB9//f/Yf4r/yf/jwDT/uEALgEs//L+j/7DAAf/8wCDAOP/4QB+/4IAmf6QAfT/sQAlAE3/QAJU/SsB/v/P//v+WAATANb/zQEw/6YCc/77AKP/0P8bABX/SP9z/1MAn/6GAJ795wEAAE0BB//9/20Btf6CAQr/xgFR/7gC4f9vAfcAf/5r/1/8DAFE/rICv/+VAEgAiwDt/zD9ZQE3/HUBff7BAg8BjgDA/8D9RQJK/SMCiv2BAhT+Sv+h/y4ABQNz/uQBA/1yAez8zf+C/lUANgAF/1QA7v61Ap7/fwHk/iMCOv4RAOb+0v///wkAQwPs+4kB0v5AAVT/eP+6AV0ALAHk/cMCFP/cALEA3wBFAFD+iAOi/n7/vf9v/kIA//3s/v8BHgFq/1ECv/1rABEDT/3TABMAjwIXAhT9Ov6DAC7/m/4IAG79wQARAOkANgHN/qIAyv4o/+L+Jv8B/0cAdgBrAPcAZgAfAZr/HQI4Anj/mf+l/1UCrf8K/84BNADV/jv+FwDG/iEBpf5V//r+rf6dAfH7PgPG/18C+v/k/iMCzv6o/2n+0AWT+zEExQChAJUAwv3pAWz7NQNI/5YEzf3BAK79G/+T/N/7aQGM/b8ELfxEBBv+1wOHACL8GABZ/1UEQfyBAcX/LgSn/coB2v51/CsDx/zJAYD8xgIcAVYDvADo/1QAMfxgAOj9igMo/eoAk/6a/Yn+7gC+AjL8sQOXAccCCv46AgUB0f/NAE7+TgKn/OkAr/wk/w78iv7UAIX7qgEfALgD+QCyAukBcgCeAAwAzgI8/sL+bgASAGr+zf/m/nL9Yf+8/9L/DwHdAMUC7AKX/4oCrQCd/w3+YP+BAJT+MgG9/qv+Jf7L/a4Ad/9H/S8BTQHyAY0DmQO8AUn+NAAQ/W7+6f2V/5EBiP40AAEA0wDO/aT/7P5eALMBZf9C/zQA/AKOAP4AfvwVAfIAVADQ/kr/kgIK/7IDlf34/2IAgQG3ART9aAAA/2YAt/5//tr/uf9KAPP/8P8w/+//k/9j/wX/RwH9AuoA5gC5AcoA/f78/mf/9/7i/lkAdf9GAFr/L/7l/23+OwEMAcr/vgB9AGAA+f3S/jX/KQH6Afb/fQCtAAcARf9u/uz+2wCVAQ4CjP8o/5n/5v7q/x3/+v/9/9r/UAAIAGEAEwA3AI8Bwf7m/n8DsP5nAHYBRwAKAR3/ywBR/sb/xP4cAJv/EP6LAewAkgEx/8gBlwC6/gABsP4QAM/97wAKAOX+dACT/jYBev8HAagA2wCuAOMBXwC//+X/av6dAC7+3v8B/6z+vP6c/47/M/5q/zP/hQA4/6kBLwEvAOsA8gBlANf/+ABp/6MBYv+xAFYAMv9dALIAhwL0/2AC0/8EA9kBMwCJAA39kgG+/Yn+6v0v/kr/8f1n/gX/6f5S/t3/eP1c/8H/PQIQAfQAgQNpAW0DgAKlAC8BUwHF/5z/mf7A/eb/LAA6/un9zv05/kn9yf4//pH94wDQ//H/GgD7Ab0CbgPzAcACPgMxAcoBFADSABcAPwB6/UH9uP2e/o0A0vzf/pb+a/8aAAMAGACe/mUB3wAEAgYCWQIfAgEA6v9YAFP/mf7O/3T+7P8P/8f9j/2o/mQAYAF1AC4BMAOIAXsCKv/KAJsAqP82Aor/eP/8/gv/bP5C/5H/o/3b/rL93f5g/vr8ggH2ANMBwQD9AbIB1f/bAZYASwGiAFUBVgAmAT3/GQAZ/338Fv9W/yAAl/7y/xn/rv9ZAPP+xgAiA7UCuQLtAf4A/QCe/xH/OP5j/yP+pP6d/m3+H/+u/iH+y/5rALv/rQHXAGQBNQEJAK0Aq/8aAE0AKgHuALr/HQBiAAABlgCu/57/YwDMAUoAngClAIT/Jf86/uP9Uv79/tb+Jf8L/3//ov+G/+YAigAiAWYB2QBPAZP/MgBr//f/M/++/Rn/Vf21/nz+Zv82AOkAlQDKAOIB/QDCA30DkQMuA6QC3AAdALL+zP7R//T9hv/v/TD/FP5l/ub/1P3N/or/pf9N/4MAMgCwAPQAOgDx/7T/Zv9XAJYALQCFAdIBHQK+/yYAJADa/xIAxv7G/wkASQAL/4T/L/6k/hT/kv5W/5n/RAGJANwAJQCE/7r/GgAT/yMAtAAwANsAUf/V/0r/4/+Z/18AlAAVAXMC+AGsAvAAnAG7AKUAkwDZ/zQBMgC6/6v+DP+u/kX+Bf0z/YD+Wv5x/3H/1wCWAGoAwf/0//z/+v9XAMz/5P8x/7z/fP7p/fn9s/1Z/tD+Df9S/3D/4P9eAOP/HQBgAIgAegDZ/wgAlgCyACgAUACj/+j/4wAUARcBZgGNAmUCRQNWAwcEvwTVBDkF0ATzBEQFlQVRBRwFkgQ4BOkCDwI+AVcA4P7X/Q39Rvvc+rz4Ofg29xz2w/V29Zz1sfUQ95r2c/gw+Lj4L/oG+7z8X/yK/XD9Rf7B/VT+9v3y/cr+l/1+/Sr8zPwm/PD9Uv+uAesEYAYkC5YOkRPEF+wbPyBxJOMmvyf+J+MlkSQ9HyYZYxEcCQ8BCfiu78fmpN9e2g3XEdSM03bUEdcb2xTfiOOU6NrtZPKq9lL57Pua/WT+Ff85/jr97fyc/G/88/s+/Jz8Kf3k/aT9s/3z/yQDLwbuCa0KBw5jE8sYxx5FIesk2ylALYkvvS/zLv8u4ypwJJYcShNvCwACQvg57qDkOt0B2TTWG9TU0k7TrNe621HgP+VI6wDyBvaN+Bv6i/xJ/kn+Yvz1+jz5ivd39jD1hfTG85vz/vQY9y/45vqC/A7/jP9FALz/CgC3AW4CfwWbBMkHFwu1EO0XrhzUIn0ozC5gNEE3pTgnOv83LzThLAsjexn9DawCIPa66JvdrdTWzuTKGchZyBvLVNAn1p7c4+Px65PzU/kq/RYAMQOIBJsDHwEJ/i/7/Pjt9XPzevHR8JPxUPK/84b28vg7/Bn+U/8YAEf/jAEnATcEOwWvBuwKVA31FO4aPSLNKcEvjzYBPJU+3kAfQBs96TekLVYjUBbFCNr6z+vM3evR2cdAwWe+zrzevrrCY8k/0m7bxuQV73P4OABIBkMJWQ14Do8NqgrwBcMClv52+nH2jvJv8Drv4O4J8JHx5vOM9gT4uvmZ+vb60vwH/DkA6gGhBIAJvwrcE6ka3yJGK/AwkDlMQC5Dg0UPRHVBkzx3MXwmwxdgCWr6DOqp20nOwsJ/u2K3KLbItzC7iMNBzp7Z+eNU7zX6KgQDC88OuBJ1FPsTkBAyC7wGyAKC/Y/5XvRe8evv+u7X78XwY/JC9aH26ven+Lf3/PmH+X3+WACeAw0IvgodFYccKyY9LXs0fD1GRBZHg0jdR4JEEj4oMdcklxV7Bob1A+SE1MfGx7tytAOyfLEbtMG4f8JPzpDaaeZX8of+iAi7D+0T4BdpGZoYRhTnDf0IcAOE/uf44vM48Ivu2O377UvvkvDR87P0QvZ29Zr3s/f++oj/zQD5BlkHNRB4GZMiLyzjMdg6lkMtSLdJ1kn0RpVDuzcRKicbTwvT+0/peNilyLW7LrIqrnetA69osxS7RceL1EThc+64+8MHBxHsFW8aQx0zHYoaoxPfDVkHZwFb/GH2fPHT7YHsReyR7SzuV/HK8j705PNe9Jz13Pc//ln/egXKBvsM8xfmIJws6DJlOopD0knOTN9N+kqRRzY9uC4hH7MNXP1I61vZhsjWuRqvcqq/qeurALCxt8fD6NLm4HDuWvydCMwSjhgqHMse2h7JG7EVpA2nBt7/EfpO9XLvDOzV6X/p+OtW7SbxL/PE9I7zHPSu9OX4+/8RAWcHVQcbD5cauiOlMMM2iT/yRz1MsE+WT5FMPUiXO/srnhoVCLn3CuUZ03zBR7KwqOSlQKfVqqGw/rngx3bXhuZm9SYENxBmGcEdXiBZIq4gZh30FPYLUgPR+uH1HvEt7NPoW+YY5yXquewE8ZzzFfWq8zDz6/Jz+bn/LQPyBsgGJxGxHKcoVDTSOyxGKk48UbRT0lLXTw1JTTmMJ30TXv9e7THba8mBt6inIaC2oBul+6pZskq/Tc+63+nuHP+pDgAaACFlItojIyNjIDQb1BHJBzn9u/Qp8LrsDOkL5+jklecY6n7uVvIb9en1gPMR8qDxkPnt/oEFUQYRCWkUhyCnLyw6IEXsT99VvFc9V2RVb1GeRXQzxR2DBrzw19zby/q6sqlenM2XIZw8pCqtr7njySDcMOyx+48M1Rq1JNwnJSfSJQciox1LFnkMswEj9kPvbeuL6UroqObF5ifpRezT8EP0tvZ19YHyz+7o72T2CP1PBGgFiwwwGHgmnDZJQmJQvlrCXgJfqFs3WD1Qzj+6KjsSH/hX4DXL1rrpq2icLpNdkt6ap6ZOsw7EvtbM6dn4MAhOF64jCytAKhUoYyLDHBQWmA3mBB35cu/Y6B/nTue/6BPpQ+vM7czwB/ST9az2avM08DbsXu9G9Bb8swLjB/gUjyEQMwNCE1B+X5lmJ2l7ZZdfX1iGSao02xv8/7bkuMtyt66ooZockdeNhJNxnzKtXr9n0/To2vmjB7sVZSGkKvcr9CiYI70azxLyCV0CYfqa8PjpoOY55j7o+On+7MLwBfO89fv1hvYE9MPwk+yg7APxSvYk/5kEthHgIH0xnkOYUG1ggGtEbwBLine truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "IPython.display.display(Audio(w, rate=CONFIG.sample_rate))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABAUAAAI4CAYAAAASzVA9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsvVmMbel13/f7hj2csU4Nt+7Y3fc2\ne+DUFEW2KFnRaMahpQBJhAxG7ACGA0RCghh5C4y8+CEIECBPUZxBAhJLShAjfggSxJATMBrMSKYo\nkRTJJptkd7OHO9et6Yz77OEb8rD2PlVNk9RgiZLF8wcKVbXPOXt/4/rW+q/hqBgjW2yxxRZbbLHF\nFltsscUWW2yxxfce9J91A7bYYosttthiiy222GKLLbbYYos/G2xJgS222GKLLbbYYosttthiiy22\n+B7FlhTYYosttthiiy222GKLLbbYYovvUWxJgS222GKLLbbYYosttthiiy22+B7FlhTYYosttthi\niy222GKLLbbYYovvUWxJgS222GKLLbbYYosttthiiy22+B7Fd5UUUEr9T0qpJ0qpL1+6tqeU+qRS\n6vX29257XSmlfl4p9YZS6ktKqY9c+szfbN//ulLqb166/lGl1CvtZ35eKaW+m/3bYosttthiiy22\n2GKLLbbYYos/LXwrm/qbXv+2dvS3w3c7UuCXgL/6Tdf+DvBrMcbngV9r/wf4KeD59udngf8ehEQA\n/i7wg8DHgL/bEQnte3720ue++VlbbLHFFltsscUWW2yxxRZbbPEvKn6J72znfks7+jvhu0oKxBg/\nBZx90+V/Hfjl9u9fBv6NS9d/JQp+B5gopa4DnwA+GWM8izGeA58E/mr72jjG+OkYYwR+5dK9tthi\niy222GKLLbbYYostttjiX2h8G5v6Mr6dHf1tYf8kG/jHxNUY4yOAGOMjpdRhe/0mcO/S++63177T\n9fvf4vq3hFLqZxHmBOCjAJmZoLFAxNHQ+DmZmWCw6JY/8XhAYTAoFBCJgG7/M0qRaIVVoBWECL79\nCUQ0CqtpPynXXYgYpdAKlLp4LbavuRhRKHKjMO1nQ5Sf1ERpV1DUAXITse19vhlNUKxcpG8VRkm7\nmyBvjFE+o5HfMUobOkQu/o8RZr7kSpKjiASU9C92/Y1kphsDaY+Mk5L3AJWXv03XTnXBUHXjUvuI\nJ6BQROTvhpLGL1EqweocS0pGgtUXz49EApGhNZuxmvmaiU3J9UWvAuCjzJtWEaOQ/kSFi7TtlvfJ\ne9r/IzRRsXaRoYWAorg0rqYdP3/RNUJUJDq2PZHx6voaLrUBwEWFD/KaVWB0xFxqcxMUpY/t6+26\naR8UI7gYiRFSLa9191HtWgVFuDSxHhnrzMj7u2s+QIgRoxUhRmy7rjso5J4oaVMT4qY9Eah8JGnb\n8O7P0LZZ2uOiwgVZN926D8R2H8nnNTIO3RrxUcZ8nFyMW4gKz8W8xXatxbaf3frr9oaP7X27zyPz\nbtq52IzrpXly7bzJHpNxvuiPrIvawzrI7Kdak2v1rv0YW3nQrX3XtqmnI1rFzf4PUdZTtx/DpXVz\nIT8u9p65JDtC+75uLnyAzECiu7Hq5JIiftOcdL3u5rZ7XogiY0KERMv7L+/77nmX0Y23bWXh5X3u\nIxTek2pNjFDHQN8YkvYN3R5ItPS/ChEXwOqLdR949zpVXOwBYCOLAxft1ApSLeMckD7VPjJMVNt/\n6XEnq9SlvvgIx81Ddu11Uq1kzNXFnNatvE4uTXgTZV+kWpHqbjXJvJXtHrk8VvrSvMVLMjVp11H3\nLK1kbDbnRXuPGMFoMO38hXaODRfrfLMPLp1P3Rqw7fi7SzIILs4m2UlsZEnXzoj0Rys5B2Pb7rxd\nmDFC0wqenombNd7drNvHPipSHbHtGVUHhY+KrF27nvac4mKfgsigRQOjhAs53I1Je4rI/lSbNamV\nyBQXIWv3X0SRGY/WAaUgBEUICh91+7yIUpdOx9iulaja00pG5fK+je0cuKCoo8K2+1zG6mI8XVAb\neaCAREXspXOj250xQhlkojrZLmtNUQdZl0krO1wU/cBFaEIk1YqBDe0YxY1saoIWeRAvZIHeyB9F\nFWQvWRXk3OzkV2jPNiVnZ/f5bu7q0OlGYFX8Z87Tiz13MafdvXXbBhQ4rym8wShZG6pti6wZzcpr\nch02+1W389uNncgo1b7WznXsZKiiadeZaWUD7TrrzqjNemtfc/FC/uq2b6aVNa49o3PT6Vrq3eeN\nks9WXnS/bh71N723mz8Dm3a5oN4tl4C1C+RGb/Y07frp9KjMyJpw8ULf7PTQZiNTZW9HZA5pbxWA\npu1Lt586OdXhQn969+diuze6Mep04qUL7CSKiJz9LkaqEOkbLWvnm55h9bvv3+lghet0zXb9B0Ud\nRFdJ9YVMie08+djJoYgnkmn9Ln38sv4bEHnenTU+wCJU5CrdXOtgN/PervNvksvdypOzCdY+bMZp\n0PW5/byLkRBFh02UFj09BiKBFLuRDTI+ikBsZdwl24BOvshcNnhSjPQNaKJHo4lEHJ5IQGMIeJq4\nRqFRSrdj54lEtLLEGPCxaZ9v6FYMMbTz0mm9m1VwEmO8wl8wfOITH4unp7M/1Wd87nOvfQUoL136\nxRjjL/4Rb/Pt7OVH3+4Dfx5IgW8H9S2uxT/G9W+JdnB/EUApFcHy1M7HGcQJgcCUh9w7/zWe2vk4\nO+GAfuwDMNVTNJpJ2CFVCT4GPJ6ByrBKM7CGW0PDfgq5gcLDvIFFA2sX6VnFbqrITKQJinkTOVkH\nhqlmlLBRNH2U959WgfPKYbXixZ2USSrCsfSwcopnBrIZz2rF/RW8fxLZTQKJjhtFvcPj0vKZY89H\nDww7iRzsD9d6I7SNgqGNWA11uFDgdHuAdf+vXeT/nr/Oz914nkRFyqCY1kI4rB0smsCdkWE/iwxM\nYDf1KCIuatZesfaaN5eKlRMFTiECttcSHC4qpnXk7rJh7mqM0jTRM1dLHvM6D2e/TZpe4Ur/fexx\ni2fVdQ57llUTKbzHx8jKN/zQwZDMRAqn+NXzd/iZg2d4fug2SkLhFdNGk2oYmMDQBlIdWHvDWa3Z\nT0XAlUEooYH18jmnOao0Xz4P/KUripVXfPE08OF9zV4q96m8YuVFqBoVWTnNjZ7DqkAdNEX7WhMU\nC6dI2jYAnNaa81rmZD+FSRoZ2UCIUAbFUal5derl9cwwSGTujJID8bSM1D7y1NDQt7LO9tJIquOG\nCClb2e2jrKM3544XJ5aeiZtr51Vk5QK7mWbVRA57mv0sbgzTVIviqRU8LjX3V4EruaZvI5VXvL3w\nXB9oei3Z4AJkJpIo2Rtdex6vNed1ZF4H6hCpfaAMjuu9nP1cMbBiJOylcUPqzJzm86eBT1wP9K2M\nW+E0C2fYSfzmeWXQ1EEUkzpIv42W8Zo3ilxHxoncc+VlHY9sZGAjedtWpSJ1UKycZtqImJkkkZ6J\nNK3y1jeiYB/XhrcW8JX5EoDb/QHPjWV+OxKg9jBrFDvtc5+U0LOKD4wb+q2y3gTF2hvGicOqiIuK\n0mumjRBdiZY2LZxm3lwmSGTfll5W+dBKv8+ryHtGcKPnZB15zcIpZo3CteRDz4hhFFqlMNUXxEii\nI/NGc3clsudGX0iGWS3PLr0oKX1Lq7hKX6tWSdrPYO1ljXYG4byBz50tud0f0ITIvXLFh3ZG3BRR\ny7QWpflGX+TRm4vIWeW4klt2M8XASj9Pq8jaBXZSTWagdDCtAy7IHpiktPMXKZy08ZmBKPeFV9xd\nwb1lw49eS0jb9eWCovC0BlZsx0zk9X977z/nX9n9D7jZN+y0cr4J0p+3F45zV3OQZBvl8qSp2Esy\nnh4abvVFHrigmDXwxjxw2NMt6SFGfm6gZ8SQrTwsm8iiiVzvaxItiuq0hr6ReegU0ibA2stnRGbI\n/BVeZPfIXqzzwslzfJR2z2uZp0ECh7lcP6sULkb2U5kPq1syiAvjx7dEUdP+vDEP9K1iJ1WsHcyb\nwPt2xFipAzwo5Dkf2LmQM6o977r2zRrFrV5gL3W4CA/WCdNGcbvvMSoyd4aeCSQqkptAv5Wbj8uU\n3ziCn7wKk9SxkziaoFg6g4saqwLHVcLbK41RcJBFxkkg14HT2vBMv2GcNKy94cW9c/r9WtpdGxbr\njGWVkhrPIG3IUteSggrvNVVjWTUJpTMYHcm0R6mIC5rSW5qgqbzmuEp4WBr20kCqI30TGFlPouU8\nPirTzZlkVORq3nCQVeTGU3qDjxqjAk3QvLbooxXsJJ5cBwJwXCW8uTTspJEbuSfVgdPaclYrpjU8\nKgK3BoYf2KvYSxuGSYPVMn5P1j2mTbKR06kOjKzD6kjpDW+tMvZSz5WsYtYkrREcOSpTbvUrEh0o\nnKEJmr71lF5zf51yrxBDfieB/dTTRIUGciOy7jJRmenIwDqStk196+glMtZHqz6/Px2yYwNXskbG\np7cW/afK+Ox5nxdHtRjhUTFKHAPbkOhA6Q2Vl3VgVKRnHX3raIJmUSec1QmPS1ln+2ncGLlJS2Kc\n1gajYGRlvQCcNYbzWnFewThVHGSBkQ2UXnNcKb4+C7ywo5kkoiOt3MV5YxRMG8Ub88gkE8JhL4Vh\nInK9O5tSA6eVnEmi+0VOK03hRS5pJXLpy+cVz41TcqM2xKeccZF3Fg3P7SSMEzit4LQMjFPNbqrI\nTeTRGvYzxX4m9+/OyU4/KL3i/srzvh2Na/ds1p7fndE+TCJDEzd6p29JpDKIHJg2MLQie1cO/umT\nNT99M6cMirNKcVwG3inWfHi3z8CKDBNZKM/Yb8e2Wyul1zwsDV86CzwzNFzNhcB7ULine truncated
+      "text/plain": [
+       "<matplotlib.figure.Figure at 0x7fcd1d97b8d0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "plot_spectrogram(spec)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/utils.py

@@ -0,0 +1,51 @@
+import io
+import librosa
+import torch
+import numpy as np
+from TTS.utils.text import text_to_sequence
+from matplotlib import pylab as plt
+
+hop_length = 250
+
+def create_speech(m, s, CONFIG, use_cuda, ap):
+    text_cleaner = [CONFIG.text_cleaner]
+    seq = np.array(text_to_sequence(s, text_cleaner))
+            
+#     mel = np.zeros([seq.shape[0], CONFIG.num_mels, 1], dtype=np.float32)
+    
+    if use_cuda:
+        chars_var = torch.autograd.Variable(torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
+#         mel_var = torch.autograd.Variable(torch.from_numpy(mel).type(torch.cuda.FloatTensor), volatile=True).cuda() 
+    else:
+        chars_var = torch.autograd.Variable(torch.from_numpy(seq), volatile=True).unsqueeze(0)
+#         mel_var = torch.autograd.Variable(torch.from_numpy(mel).type(torch.FloatTensor), volatile=True)
+        
+    mel_out, linear_out, alignments =m.forward(chars_var)
+    linear_out = linear_out[0].data.cpu().numpy()
+    alignment = alignments[0].cpu().data.numpy()
+    spec = ap._denormalize(linear_out)
+    wav = ap.inv_spectrogram(linear_out.T)
+    wav = wav[:ap.find_endpoint(wav)]
+    out = io.BytesIO()
+    ap.save_wav(wav, out)
+    return wav, alignment, spec
+
+
+def visualize(alignment, spectrogram, CONFIG):
+    label_fontsize = 16
+    plt.figure(figsize=(16,16))
+
+    plt.subplot(2,1,1)
+    plt.imshow(alignment.T, aspect="auto", origin="lower", interpolation=None)
+    plt.xlabel("Decoder timestamp", fontsize=label_fontsize)
+    plt.ylabel("Encoder timestamp", fontsize=label_fontsize)
+    plt.colorbar()
+
+    plt.subplot(2,1,2)
+    librosa.display.specshow(spectrogram.T, sr=CONFIG.sample_rate, 
+                             hop_length=hop_length, x_axis="time", y_axis="linear")
+    plt.xlabel("Time", fontsize=label_fontsize)
+    plt.ylabel("Hz", fontsize=label_fontsize)
+    plt.tight_layout()
+    plt.colorbar()
+

synthesis.py

@@ -38,17 +38,11 @@ def main(args):
 
     # Sentences for generation
     sentences = [
-        "And it is worth mention in passing that, as an example of fine typography,",
-        # From July 8, 2017 New York Times:
-        'Scientists at the CERN laboratory say they have discovered a new particle.',
-        'There’s a way to measure the acute emotional intelligence that has never gone out of style.',
-        'President Trump met with other leaders at the Group of 20 conference.',
-        'The Senate\'s bill to repeal and replace the Affordable Care Act is now imperiled.',
-        # From Google's Tacotron example page:
-        'Generative adversarial network or variational auto-encoder.',
-        'The buses aren\'t the problem, they actually provide a solution.',
-        'Does the quick brown fox jump over the lazy dog?',
-        'Talib Kweli confirmed to AllHipHop that he will be releasing an album in the next year.',
+        "I try my best to translate text to speech. But I know I need more work",
+        "The new Firefox, Fast for good.",
+        "Technology is continually providing us with new ways to create and publish stories.",
+        "For these stories to achieve their full impact, it requires tool.",
+        "I am allien and I am here to destron your world."
     ]
 
     # Synthesis and save to wav files

train.py

@@ -111,6 +111,8 @@ def main(args):
         progbar = Progbar(len(dataset) / c.batch_size)
 
         for i, data in enumerate(dataloader):
+            start_time = time.time()
+
             text_input = data[0]
             magnitude_input = data[1]
             mel_input = data[2]
@@ -128,42 +130,40 @@ def main(args):
 
             if use_cuda:
                 text_input_var = Variable(torch.from_numpy(text_input).type(
-                    torch.cuda.LongTensor), requires_grad=False).cuda()
+                    torch.cuda.LongTensor)).cuda()
                 mel_input_var = Variable(torch.from_numpy(mel_input).type(
-                    torch.cuda.FloatTensor), requires_grad=False).cuda()
+                    torch.cuda.FloatTensor)).cuda()
                 mel_spec_var = Variable(torch.from_numpy(mel_input).type(
-                    torch.cuda.FloatTensor), requires_grad=False).cuda()
+                    torch.cuda.FloatTensor)).cuda()
                 linear_spec_var = Variable(torch.from_numpy(magnitude_input)
-                    .type(torch.cuda.FloatTensor), requires_grad=False).cuda()
+                    .type(torch.cuda.FloatTensor)).cuda()
 
             else:
                 text_input_var = Variable(torch.from_numpy(text_input).type(
-                    torch.LongTensor), requires_grad=False)
+                    torch.LongTensor),)
                 mel_input_var = Variable(torch.from_numpy(mel_input).type(
-                    torch.FloatTensor), requires_grad=False)
+                    torch.FloatTensor))
                 mel_spec_var = Variable(torch.from_numpy(
-                    mel_input).type(torch.FloatTensor), requires_grad=False)
+                    mel_input).type(torch.FloatTensor))
                 linear_spec_var = Variable(torch.from_numpy(
-                    magnitude_input).type(torch.FloatTensor),
-                                          requires_grad=False)
+                    magnitude_input).type(torch.FloatTensor))
 
             mel_output, linear_output, alignments =\
                 model.forward(text_input_var, mel_input_var)
 
             mel_loss = criterion(mel_output, mel_spec_var)
-            linear_loss = torch.abs(linear_output - linear_spec_var)
-            linear_loss = 0.5 * \
-                torch.mean(linear_loss) + 0.5 * \
-                torch.mean(linear_loss[:, :n_priority_freq, :])
+            #linear_loss = torch.abs(linear_output - linear_spec_var)
+            #linear_loss = 0.5 * \
+                #torch.mean(linear_loss) + 0.5 * \
+                #torch.mean(linear_loss[:, :n_priority_freq, :])
+            linear_loss = 0.5 * criterion(linear_output, linear_spec_var) \
+                    + 0.5 * criterion(linear_output[:, :, :n_priority_freq],
+                                      linear_spec_var[: ,: ,:n_priority_freq])
             loss = mel_loss + linear_loss
-            loss = loss.cuda()
-
-            start_time = time.time()
+            # loss = loss.cuda()
 
             loss.backward()
-
-            nn.utils.clip_grad_norm(model.parameters(), 1.)
-
+            grad_norm = nn.utils.clip_grad_norm(model.parameters(), 1.)
             optimizer.step()
 
             step_time = time.time() - start_time
@@ -171,7 +171,8 @@ def main(args):
 
             progbar.update(i+1, values=[('total_loss', loss.data[0]),
                                       ('linear_loss', linear_loss.data[0]),
-                                      ('mel_loss', mel_loss.data[0])])
+                                      ('mel_loss', mel_loss.data[0]),
+                                      ('grad_norm', grad_norm)])
 
             tb.add_scalar('Train/TotalLoss', loss.data[0], current_step)
             tb.add_scalar('Train/LinearLoss', linear_loss.data[0],

utils/audio.py

@@ -81,10 +81,10 @@ class AudioProcessor(object):
 
     def inv_spectrogram(self, spectrogram):
         '''Converts spectrogram to waveform using librosa'''
-        S = _denormalize(spectrogram)
-        S = _db_to_amp(S + self.ref_level_db)  # Convert back to linear
+        S = self._denormalize(spectrogram)
+        S = self._db_to_amp(S + self.ref_level_db)  # Convert back to linear
         # Reconstruct phase
-        return inv_preemphasis(_griffin_lim(S ** self.power))
+        return self.apply_inv_preemphasis(self._griffin_lim(S ** self.power))
 
 
     def _griffin_lim(self, S):
@@ -93,18 +93,13 @@ class AudioProcessor(object):
         '''
         angles = np.exp(2j * np.pi * np.random.rand(*S.shape))
         S_complex = np.abs(S).astype(np.complex)
-        y = _istft(S_complex * angles)
+        y = self._istft(S_complex * angles)
         for i in range(self.griffin_lim_iters):
-            angles = np.exp(1j * np.angle(_stft(y)))
-            y = _istft(S_complex * angles)
+            angles = np.exp(1j * np.angle(self._stft(y)))
+            y = self._istft(S_complex * angles)
         return y
 
 
-    def _istft(self, y):
-        _, hop_length, win_length = _stft_parameters()
-        return librosa.istft(y, hop_length=hop_length, win_length=win_length)
-
-
     def melspectrogram(self, y):
         D = self._stft(self.apply_preemphasis(y))
         S = self._amp_to_db(self._linear_to_mel(np.abs(D)))
@@ -115,11 +110,15 @@ class AudioProcessor(object):
         n_fft, hop_length, win_length = self._stft_parameters()
         return librosa.stft(y=y, n_fft=n_fft, hop_length=hop_length, win_length=win_length)
 
+    def _istft(self, y):
+        _, hop_length, win_length = self._stft_parameters()
+        return librosa.istft(y, hop_length=hop_length, win_length=win_length)
+
 
     def find_endpoint(self, wav, threshold_db=-40, min_silence_sec=0.8):
         window_length = int(self.sample_rate * min_silence_sec)
         hop_length = int(window_length / 4)
-        threshold = _db_to_amp(threshold_db)
+        threshold = self._db_to_amp(threshold_db)
         for x in range(hop_length, len(wav) - window_length, hop_length):
             if np.max(wav[x:x + window_length]) < threshold:
                 return x + hop_length

utils/data.py

@@ -3,7 +3,9 @@ import numpy as np
 
 def pad_data(x, length):
     _pad = 0
-    return np.pad(x, (0, length - x.shape[0]), mode='constant', constant_values=_pad)
+    return np.pad(x, (0, length - x.shape[0]),
+                  mode='constant',
+                  constant_values=_pad)
 
 
 def prepare_data(inputs):