master update from loc-sens-attn-smaller-post-cbhg

.compute

@@ -1,3 +1,10 @@
 #!/bin/bash
 source ../tmp/venv/bin/activate
-python train.py --config_path config.json 
+# ls /snakepit/jobs/1023/keep/
+# source /snakepit/jobs/1023/keep/venv/bin/activate
+# source  /snakepit/jobs/1047/keep/venv/bin/activate
+# python extract_feats.py --data_path /snakepit/shared/data/keithito/LJSpeech-1.1/wavs --out_path /snakepit/shared/data/keithito/LJSpeech-1.1/loader_data/  --config config.json --num_proc 32
+# python train.py --config_path config_kusal.json --debug true
+python train.py --config_path config.json --debug true
+# python -m cProfile -o profile.cprof train.py --config_path config.json --debug true
+# nvidia-smi

README.md

@@ -23,6 +23,7 @@ Checkout [here](https://mycroft.ai/blog/available-voices/#the-human-voice-is-the
 | [iter-62410](https://drive.google.com/open?id=1pjJNzENL3ZNps9n7k_ktGbpEl6YPIkcZ)| [99d56f7](https://github.com/mozilla/TTS/tree/99d56f7e93ccd7567beb0af8fcbd4d24c48e59e9)           | [link](https://soundcloud.com/user-565970875/99d56f7-iter62410 )|First model with plain Tacotron implementation.|
 | [iter-170K](https://drive.google.com/open?id=16L6JbPXj6MSlNUxEStNn28GiSzi4fu1j) | [e00bc66](https://github.com/mozilla/TTS/tree/e00bc66) |[link](https://soundcloud.com/user-565970875/april-13-2018-07-06pm-e00bc66-iter170k)|More stable and longer trained model.|
 | Best: [iter-270K](https://drive.google.com/drive/folders/1Q6BKeEkZyxSGsocK2p_mqgzLwlNvbHFJ?usp=sharing)|[256ed63](https://github.com/mozilla/TTS/tree/256ed63)|[link](https://soundcloud.com/user-565970875/sets/samples-1650226)|Stop-Token prediction is added, to detect end of speech.|
+| Best: [iter-K] | [bla]() | [link]() | Location Sensitive attention |
 
 ## Data
 Currently TTS provides data loaders for

ThisBranch.txt

@@ -0,0 +1,10 @@
+- location sensitive attention
+- attention masking
+x updated audio pre-processing (removed preemphasis and updated normalization)
+- testing with example senteces
+- optional eval-iteration
+- plot alignments and specs for test sentences
+- use lws as an alternative audio processor
+- make configurable audio module
+- make precomputed of on-the-fly dataloader configurable
+- use of preemphasis is configurable. It is skipped if config.preamphasis == 0

config.json

@@ -1,11 +1,14 @@
 {
-    "model_name": "best-model",
+    "model_name": "TTS-smaller-anneal_lr",
+    "audio_processor": "audio",
     "num_mels": 80,
     "num_freq": 1025,
-    "sample_rate": 20000,
+    "sample_rate": 22000,
     "frame_length_ms": 50,
     "frame_shift_ms": 12.5,
     "preemphasis": 0.97,
+    "min_mel_freq": 125,
+    "max_mel_freq": 7600,
     "min_level_db": -100,
     "ref_level_db": 20,
     "embedding_size": 256,
@@ -13,9 +16,14 @@
 
     "epochs": 1000,
     "lr": 0.002,
+    "lr_scheduler": "AnnealLR",
     "warmup_steps": 4000,
+
+    "lr_decay": 0.5,
+    "decay_step": 100000,
+
     "batch_size": 32,
-    "eval_batch_size":32,
+    "eval_batch_size":-1,
     "r": 5,
 
     "griffin_lim_iters": 60,
@@ -24,9 +32,14 @@
     "num_loader_workers": 4,
 
     "checkpoint": true,
-    "save_step": 750,
-    "print_step": 50,
+
+    "save_step": 25000,
+    "print_step": 10,
+    "run_eval": false,
     "data_path": "/snakepit/shared/data/keithito/LJSpeech-1.1/",
+    "meta_file_train": "metadata.csv",
+    "meta_file_val": null,
+    "dataset": "LJSpeech",
     "min_seq_len": 0,
-    "output_path": "experiments/"
-  }
+    "output_path": "../keep/"
+}

config_kusal.json

@@ -0,0 +1,41 @@
+{
+    "model_name": "TTS-larger-kusal",
+    "audio_processor": "audio",
+     "num_mels": 80,
+    "num_freq": 1025,
+    "sample_rate": 22000,
+    "frame_length_ms": 50,
+    "frame_shift_ms": 12.5,
+    "preemphasis": 0.97,
+    "min_mel_freq": 125,
+    "max_mel_freq": 7600,
+    "min_level_db": -100,
+    "ref_level_db": 20,
+    "embedding_size": 256,
+    "text_cleaner": "english_cleaners",
+
+    "epochs": 1000,
+    "lr": 0.002,
+    "lr_decay": 0.5,
+    "decay_step": 100000,
+    "warmup_steps": 4000,
+    "batch_size": 32,
+    "eval_batch_size":-1,
+    "r": 5,
+
+    "griffin_lim_iters": 60,
+    "power": 1.5,
+
+    "num_loader_workers": 8,
+
+    "checkpoint": true,
+    "save_step": 25000,
+    "print_step": 10,
+    "run_eval": false,
+    "data_path": "/snakepit/shared/data/mycroft/kusal/",
+    "meta_file_train": "prompts.txt",
+    "meta_file_val": null,
+    "dataset": "Kusal",
+    "min_seq_len": 0,
+    "output_path": "../keep/"
+}

datasets/Kusal.py

@@ -0,0 +1,162 @@
+import os
+import glob
+import random
+import numpy as np
+import collections
+import librosa
+import torch
+from torch.utils.data import Dataset
+
+from utils.text import text_to_sequence
+from utils.data import (prepare_data, pad_per_step, prepare_tensor,
+                        prepare_stop_target)
+
+
+class MyDataset(Dataset):
+    def __init__(self,
+                 root_dir,
+                 csv_file,
+                 outputs_per_step,
+                 text_cleaner,
+                 ap,
+                 min_seq_len=0):
+        self.root_dir = root_dir
+        self.wav_dir = os.path.join(root_dir, 'wav')
+        self.wav_files = glob.glob(os.path.join(self.wav_dir, '*.wav'))
+        self._create_file_dict()
+        self.csv_dir = os.path.join(root_dir, csv_file)
+        with open(self.csv_dir, "r", encoding="utf8") as f:
+            self.frames = [
+                line.split('\t') for line in f
+                if line.split('\t')[0] in self.wav_files_dict.keys()
+            ]
+        self.outputs_per_step = outputs_per_step
+        self.sample_rate = ap.sample_rate
+        self.cleaners = text_cleaner
+        self.min_seq_len = min_seq_len
+        self.ap = ap
+        print(" > Reading Kusal from - {}".format(root_dir))
+        print(" | > Number of instances : {}".format(len(self.frames)))
+        self._sort_frames()
+
+    def load_wav(self, filename):
+        """ Load audio and trim silence """
+        try:
+            audio = librosa.core.load(filename, sr=self.sample_rate)[0]
+            margin = int(self.sample_rate * 0.1)
+            audio = audio[margin:-margin]
+            return self._trim_silence(audio)
+        except RuntimeError as e:
+            print(" !! Cannot read file : {}".format(filename))
+
+    def _trim_silence(self, wav):
+        return librosa.effects.trim(
+            wav, top_db=40, frame_length=1024, hop_length=256)[0]
+
+    def _create_file_dict(self):
+        self.wav_files_dict = {}
+        for fn in self.wav_files:
+            parts = fn.split('-')
+            key = parts[1]
+            value = fn
+            try:
+                self.wav_files_dict[key].append(value)
+            except:
+                self.wav_files_dict[key] = [value]
+
+    def _sort_frames(self):
+        r"""Sort sequences in ascending order"""
+        lengths = np.array([len(ins[2]) for ins in self.frames])
+
+        print(" | > Max length sequence {}".format(np.max(lengths)))
+        print(" | > Min length sequence {}".format(np.min(lengths)))
+        print(" | > Avg length sequence {}".format(np.mean(lengths)))
+
+        idxs = np.argsort(lengths)
+        new_frames = []
+        ignored = []
+        for i, idx in enumerate(idxs):
+            length = lengths[idx]
+            if length < self.min_seq_len:
+                ignored.append(idx)
+            else:
+                new_frames.append(self.frames[idx])
+        print(" | > {} instances are ignored by min_seq_len ({})".format(
+            len(ignored), self.min_seq_len))
+        self.frames = new_frames
+
+    def __len__(self):
+        return len(self.frames)
+
+    def __getitem__(self, idx):
+        sidx = self.frames[idx][0]
+        sidx_files = self.wav_files_dict[sidx]
+        file_name = random.choice(sidx_files)
+        wav_name = os.path.join(self.wav_dir, file_name)
+        text = self.frames[idx][2]
+        text = np.asarray(
+            text_to_sequence(text, [self.cleaners]), dtype=np.int32)
+        wav = np.asarray(self.load_wav(wav_name), dtype=np.float32)
+        sample = {'text': text, 'wav': wav, 'item_idx': self.frames[idx][0]}
+        return sample
+
+    def collate_fn(self, batch):
+        r"""
+            Perform preprocessing and create a final data batch:
+            1. PAD sequences with the longest sequence in the batch
+            2. Convert Audio signal to Spectrograms.
+            3. PAD sequences that can be divided by r.
+            4. Convert Numpy to Torch tensors.
+        """
+
+        # Puts each data field into a tensor with outer dimension batch size
+        if isinstance(batch[0], collections.Mapping):
+            keys = list()
+
+            wav = [d['wav'] for d in batch]
+            item_idxs = [d['item_idx'] for d in batch]
+            text = [d['text'] for d in batch]
+
+            text_lenghts = np.array([len(x) for x in text])
+            max_text_len = np.max(text_lenghts)
+
+            linear = [self.ap.spectrogram(w).astype('float32') for w in wav]
+            mel = [self.ap.melspectrogram(w).astype('float32') for w in wav]
+            mel_lengths = [m.shape[1] + 1 for m in mel]  # +1 for zero-frame
+
+            # compute 'stop token' targets
+            stop_targets = [
+                np.array([0.] * (mel_len - 1)) for mel_len in mel_lengths
+            ]
+
+            # PAD stop targets
+            stop_targets = prepare_stop_target(stop_targets,
+                                               self.outputs_per_step)
+
+            # PAD sequences with largest length of the batch
+            text = prepare_data(text).astype(np.int32)
+            wav = prepare_data(wav)
+
+            # PAD features with largest length + a zero frame
+            linear = prepare_tensor(linear, self.outputs_per_step)
+            mel = prepare_tensor(mel, self.outputs_per_step)
+            assert mel.shape[2] == linear.shape[2]
+            timesteps = mel.shape[2]
+
+            # B x T x D
+            linear = linear.transpose(0, 2, 1)
+            mel = mel.transpose(0, 2, 1)
+
+            # convert things to pytorch
+            text_lenghts = torch.LongTensor(text_lenghts)
+            text = torch.LongTensor(text)
+            linear = torch.FloatTensor(linear)
+            mel = torch.FloatTensor(mel)
+            mel_lengths = torch.LongTensor(mel_lengths)
+            stop_targets = torch.FloatTensor(stop_targets)
+
+            return text, text_lenghts, linear, mel, mel_lengths, stop_targets, item_idxs[
+                0]
+
+        raise TypeError(("batch must contain tensors, numbers, dicts or lists;\
+                         found {}".format(type(batch[0]))))

datasets/LJSpeech.py

@@ -6,34 +6,35 @@ import torch
 from torch.utils.data import Dataset
 
 from utils.text import text_to_sequence
-from utils.audio import AudioProcessor
-from utils.data import (prepare_data, pad_per_step,
-                            prepare_tensor, prepare_stop_target)
+from utils.data import (prepare_data, pad_per_step, prepare_tensor,
+                        prepare_stop_target)
 
 
-class LJSpeechDataset(Dataset):
-
-    def __init__(self, csv_file, root_dir, outputs_per_step, sample_rate,
-                 text_cleaner, num_mels, min_level_db, frame_shift_ms,
-                 frame_length_ms, preemphasis, ref_level_db, num_freq, power,
+class MyDataset(Dataset):
+    def __init__(self,
+                 root_dir,
+                 csv_file,
+                 outputs_per_step,
+                 text_cleaner,
+                 ap,
                  min_seq_len=0):
-
-        with open(csv_file, "r", encoding="utf8") as f:
-            self.frames = [line.split('|') for line in f]
         self.root_dir = root_dir
+        self.wav_dir = os.path.join(root_dir, 'wavs')
+        self.csv_dir = os.path.join(root_dir, csv_file)
+        with open(self.csv_dir, "r", encoding="utf8") as f:
+            self.frames = [line.split('|') for line in f]
         self.outputs_per_step = outputs_per_step
-        self.sample_rate = sample_rate
+        self.sample_rate = ap.sample_rate
         self.cleaners = text_cleaner
         self.min_seq_len = min_seq_len
-        self.ap = AudioProcessor(sample_rate, num_mels, min_level_db, frame_shift_ms,
-                                 frame_length_ms, preemphasis, ref_level_db, num_freq, power)
+        self.ap = ap
         print(" > Reading LJSpeech from - {}".format(root_dir))
         print(" | > Number of instances : {}".format(len(self.frames)))
         self._sort_frames()
 
     def load_wav(self, filename):
         try:
-            audio = librosa.core.load(filename, sr=self.sample_rate)
+            audio = librosa.core.load(filename, sr=self.sample_rate)[0]
             return audio
         except RuntimeError as e:
             print(" !! Cannot read file : {}".format(filename))
@@ -63,12 +64,11 @@ class LJSpeechDataset(Dataset):
         return len(self.frames)
 
     def __getitem__(self, idx):
-        wav_name = os.path.join(self.root_dir,
-                                self.frames[idx][0]) + '.wav'
+        wav_name = os.path.join(self.wav_dir, self.frames[idx][0]) + '.wav'
         text = self.frames[idx][1]
-        text = np.asarray(text_to_sequence(
-            text, [self.cleaners]), dtype=np.int32)
-        wav = np.asarray(self.load_wav(wav_name)[0], dtype=np.float32)
+        text = np.asarray(
+            text_to_sequence(text, [self.cleaners]), dtype=np.int32)
+        wav = np.asarray(self.load_wav(wav_name), dtype=np.float32)
         sample = {'text': text, 'wav': wav, 'item_idx': self.frames[idx][0]}
         return sample
 
@@ -97,12 +97,13 @@ class LJSpeechDataset(Dataset):
             mel_lengths = [m.shape[1] + 1 for m in mel]  # +1 for zero-frame
 
             # compute 'stop token' targets
-            stop_targets = [np.array([0.]*(mel_len-1))
-                            for mel_len in mel_lengths]
+            stop_targets = [
+                np.array([0.] * (mel_len - 1)) for mel_len in mel_lengths
+            ]
 
             # PAD stop targets
-            stop_targets = prepare_stop_target(
-                stop_targets, self.outputs_per_step)
+            stop_targets = prepare_stop_target(stop_targets,
+                                               self.outputs_per_step)
 
             # PAD sequences with largest length of the batch
             text = prepare_data(text).astype(np.int32)
@@ -126,8 +127,8 @@ class LJSpeechDataset(Dataset):
             mel_lengths = torch.LongTensor(mel_lengths)
             stop_targets = torch.FloatTensor(stop_targets)
 
-            return text, text_lenghts, linear, mel, mel_lengths, stop_targets, item_idxs[0]
+            return text, text_lenghts, linear, mel, mel_lengths, stop_targets, item_idxs[
+                0]
 
         raise TypeError(("batch must contain tensors, numbers, dicts or lists;\
-                         found {}"
-                         .format(type(batch[0]))))
+                         found {}".format(type(batch[0]))))

datasets/LJSpeechCached.py

@@ -0,0 +1,154 @@
+import os
+import numpy as np
+import collections
+import librosa
+import torch
+from torch.utils.data import Dataset
+
+from utils.text import text_to_sequence
+from utils.data import (prepare_data, pad_per_step, prepare_tensor,
+                        prepare_stop_target)
+
+
+class MyDataset(Dataset):
+    def __init__(self,
+                 root_dir,
+                 csv_file,
+                 outputs_per_step,
+                 text_cleaner,
+                 ap,
+                 min_seq_len=0):
+        self.root_dir = root_dir
+        self.wav_dir = os.path.join(root_dir, 'wavs')
+        self.feat_dir = os.path.join(root_dir, 'loader_data')
+        self.csv_dir = os.path.join(root_dir, csv_file)
+        with open(self.csv_dir, "r", encoding="utf8") as f:
+            self.frames = [line.split('|') for line in f]
+        self.outputs_per_step = outputs_per_step
+        self.sample_rate = ap.sample_rate
+        self.cleaners = text_cleaner
+        self.min_seq_len = min_seq_len
+        self.items = [None] * len(self.frames)
+        print(" > Reading LJSpeech from - {}".format(root_dir))
+        print(" | > Number of instances : {}".format(len(self.frames)))
+        self._sort_frames()
+
+    def load_wav(self, filename):
+        try:
+            audio = librosa.core.load(filename, sr=self.sample_rate)
+            return audio
+        except RuntimeError as e:
+            print(" !! Cannot read file : {}".format(filename))
+
+    def load_np(self, filename):
+        data = np.load(filename).astype('float32')
+        return data
+
+    def _sort_frames(self):
+        r"""Sort sequences in ascending order"""
+        lengths = np.array([len(ins[1]) for ins in self.frames])
+
+        print(" | > Max length sequence {}".format(np.max(lengths)))
+        print(" | > Min length sequence {}".format(np.min(lengths)))
+        print(" | > Avg length sequence {}".format(np.mean(lengths)))
+
+        idxs = np.argsort(lengths)
+        new_frames = []
+        ignored = []
+        for i, idx in enumerate(idxs):
+            length = lengths[idx]
+            if length < self.min_seq_len:
+                ignored.append(idx)
+            else:
+                new_frames.append(self.frames[idx])
+        print(" | > {} instances are ignored by min_seq_len ({})".format(
+            len(ignored), self.min_seq_len))
+        self.frames = new_frames
+
+    def __len__(self):
+        return len(self.frames)
+
+    def __getitem__(self, idx):
+        if self.items[idx] is None:
+            wav_name = os.path.join(self.wav_dir, self.frames[idx][0]) + '.wav'
+            mel_name = os.path.join(self.feat_dir,
+                                    self.frames[idx][0]) + '.mel.npy'
+            linear_name = os.path.join(self.feat_dir,
+                                       self.frames[idx][0]) + '.linear.npy'
+            text = self.frames[idx][1]
+            text = np.asarray(
+                text_to_sequence(text, [self.cleaners]), dtype=np.int32)
+            wav = np.asarray(self.load_wav(wav_name)[0], dtype=np.float32)
+            mel = self.load_np(mel_name)
+            linear = self.load_np(linear_name)
+            sample = {
+                'text': text,
+                'wav': wav,
+                'item_idx': self.frames[idx][0],
+                'mel': mel,
+                'linear': linear
+            }
+            self.items[idx] = sample
+        else:
+            sample = self.items[idx]
+        return sample
+
+    def collate_fn(self, batch):
+        r"""
+            Perform preprocessing and create a final data batch:
+            1. PAD sequences with the longest sequence in the batch
+            2. Convert Audio signal to Spectrograms.
+            3. PAD sequences that can be divided by r.
+            4. Convert Numpy to Torch tensors.
+        """
+
+        # Puts each data field into a tensor with outer dimension batch size
+        if isinstance(batch[0], collections.Mapping):
+            keys = list()
+
+            wav = [d['wav'] for d in batch]
+            item_idxs = [d['item_idx'] for d in batch]
+            text = [d['text'] for d in batch]
+            mel = [d['mel'] for d in batch]
+            linear = [d['linear'] for d in batch]
+
+            text_lenghts = np.array([len(x) for x in text])
+            max_text_len = np.max(text_lenghts)
+            mel_lengths = [m.shape[1] + 1 for m in mel]  # +1 for zero-frame
+
+            # compute 'stop token' targets
+            stop_targets = [
+                np.array([0.] * (mel_len - 1)) for mel_len in mel_lengths
+            ]
+
+            # PAD stop targets
+            stop_targets = prepare_stop_target(stop_targets,
+                                               self.outputs_per_step)
+
+            # PAD sequences with largest length of the batch
+            text = prepare_data(text).astype(np.int32)
+            wav = prepare_data(wav)
+
+            # PAD features with largest length + a zero frame
+            linear = prepare_tensor(linear, self.outputs_per_step)
+            mel = prepare_tensor(mel, self.outputs_per_step)
+            assert mel.shape[2] == linear.shape[2]
+            timesteps = mel.shape[2]
+
+            # B x T x D
+            linear = linear.transpose(0, 2, 1)
+            mel = mel.transpose(0, 2, 1)
+
+            # convert things to pytorch
+            text_lenghts = torch.LongTensor(text_lenghts)
+            text = torch.LongTensor(text)
+            linear = torch.FloatTensor(linear)
+            mel = torch.FloatTensor(mel)
+            mel_lengths = torch.LongTensor(mel_lengths)
+            stop_targets = torch.FloatTensor(stop_targets)
+
+            return text, text_lenghts, linear, mel, mel_lengths, stop_targets, item_idxs[
+                0]
+
+        raise TypeError(("batch must contain tensors, numbers, dicts or lists;\
+                         found {}".format(type(batch[0]))))

datasets/TWEB.py

@@ -7,15 +7,25 @@ from torch.utils.data import Dataset
 
 from TTS.utils.text import text_to_sequence
 from TTS.utils.audio import AudioProcessor
-from TTS.utils.data import (prepare_data, pad_per_step,
-                            prepare_tensor, prepare_stop_target)
+from TTS.utils.data import (prepare_data, pad_per_step, prepare_tensor,
+                            prepare_stop_target)
 
 
 class TWEBDataset(Dataset):
-
-    def __init__(self, csv_file, root_dir, outputs_per_step, sample_rate,
-                 text_cleaner, num_mels, min_level_db, frame_shift_ms,
-                 frame_length_ms, preemphasis, ref_level_db, num_freq, power,
+    def __init__(self,
+                 csv_file,
+                 root_dir,
+                 outputs_per_step,
+                 sample_rate,
+                 text_cleaner,
+                 num_mels,
+                 min_level_db,
+                 frame_shift_ms,
+                 frame_length_ms,
+                 preemphasis,
+                 ref_level_db,
+                 num_freq,
+                 power,
                  min_seq_len=0):
 
         with open(csv_file, "r") as f:
@@ -25,8 +35,9 @@ class TWEBDataset(Dataset):
         self.sample_rate = sample_rate
         self.cleaners = text_cleaner
         self.min_seq_len = min_seq_len
-        self.ap = AudioProcessor(sample_rate, num_mels, min_level_db, frame_shift_ms,
-                                 frame_length_ms, preemphasis, ref_level_db, num_freq, power)
+        self.ap = AudioProcessor(sample_rate, num_mels, min_level_db,
+                                 frame_shift_ms, frame_length_ms, preemphasis,
+                                 ref_level_db, num_freq, power)
         print(" > Reading TWEB from - {}".format(root_dir))
         print(" | > Number of instances : {}".format(len(self.frames)))
         self._sort_frames()
@@ -63,11 +74,10 @@ class TWEBDataset(Dataset):
         return len(self.frames)
 
     def __getitem__(self, idx):
-        wav_name = os.path.join(self.root_dir,
-                                self.frames[idx][0]) + '.wav'
+        wav_name = os.path.join(self.root_dir, self.frames[idx][0]) + '.wav'
         text = self.frames[idx][1]
-        text = np.asarray(text_to_sequence(
-            text, [self.cleaners]), dtype=np.int32)
+        text = np.asarray(
+            text_to_sequence(text, [self.cleaners]), dtype=np.int32)
         wav = np.asarray(self.load_wav(wav_name)[0], dtype=np.float32)
         sample = {'text': text, 'wav': wav, 'item_idx': self.frames[idx][0]}
         return sample
@@ -97,12 +107,13 @@ class TWEBDataset(Dataset):
             mel_lengths = [m.shape[1] + 1 for m in mel]  # +1 for zero-frame
 
             # compute 'stop token' targets
-            stop_targets = [np.array([0.]*(mel_len-1))
-                            for mel_len in mel_lengths]
+            stop_targets = [
+                np.array([0.] * (mel_len - 1)) for mel_len in mel_lengths
+            ]
 
             # PAD stop targets
-            stop_targets = prepare_stop_target(
-                stop_targets, self.outputs_per_step)
+            stop_targets = prepare_stop_target(stop_targets,
+                                               self.outputs_per_step)
 
             # PAD sequences with largest length of the batch
             text = prepare_data(text).astype(np.int32)
@@ -126,8 +137,8 @@ class TWEBDataset(Dataset):
             mel_lengths = torch.LongTensor(mel_lengths)
             stop_targets = torch.FloatTensor(stop_targets)
 
-            return text, text_lenghts, linear, mel, mel_lengths, stop_targets, item_idxs[0]
+            return text, text_lenghts, linear, mel, mel_lengths, stop_targets, item_idxs[
+                0]
 
         raise TypeError(("batch must contain tensors, numbers, dicts or lists;\
-                         found {}"
-                         .format(type(batch[0]))))
+                         found {}".format(type(batch[0]))))

debug_config.py

@@ -10,7 +10,6 @@
     "hidden_size": 128,
     "embedding_size": 256,
     "text_cleaner": "english_cleaners",
-
     "epochs": 200,
     "lr": 0.01,
     "lr_patience": 2,
@@ -19,9 +18,7 @@
     "griffinf_lim_iters": 60,
     "power": 1.5,
     "r": 5,
-
     "num_loader_workers": 16,
-
     "save_step": 1,
     "data_path": "/data/shared/KeithIto/LJSpeech-1.0",
     "output_path": "result",

extract_feats.py

@@ -0,0 +1,108 @@
+'''
+Extract spectrograms and save them to file for training
+'''
+import os
+import sys
+import time
+import glob
+import argparse
+import librosa
+import numpy as np
+import tqdm
+from utils.generic_utils import load_config
+
+from multiprocessing import Pool
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--data_path', type=str, help='Data folder.')
+    parser.add_argument('--out_path', type=str, help='Output folder.')
+    parser.add_argument(
+        '--config', type=str, help='conf.json file for run settings.')
+    parser.add_argument(
+        "--num_proc", type=int, default=8, help="number of processes.")
+    parser.add_argument(
+        "--trim_silence",
+        type=bool,
+        default=False,
+        help="trim silence in the voice clip.")
+    args = parser.parse_args()
+    DATA_PATH = args.data_path
+    OUT_PATH = args.out_path
+    CONFIG = load_config(args.config)
+
+    print(" > Input path: ", DATA_PATH)
+    print(" > Output path: ", OUT_PATH)
+
+    audio = importlib.import_module('utils.' + c.audio_processor)
+    AudioProcessor = getattr(audio, 'AudioProcessor')
+    ap = AudioProcessor(
+        sample_rate=CONFIG.sample_rate,
+        num_mels=CONFIG.num_mels,
+        min_level_db=CONFIG.min_level_db,
+        frame_shift_ms=CONFIG.frame_shift_ms,
+        frame_length_ms=CONFIG.frame_length_ms,
+        ref_level_db=CONFIG.ref_level_db,
+        num_freq=CONFIG.num_freq,
+        power=CONFIG.power,
+        preemphasis=CONFIG.preemphasis,
+        min_mel_freq=CONFIG.min_mel_freq,
+        max_mel_freq=CONFIG.max_mel_freq)
+
+    def trim_silence(self, wav):
+        margin = int(CONFIG.sample_rate * 0.1)
+        wav = wav[margin:-margin]
+        return librosa.effects.trim(
+            wav, top_db=40, frame_length=1024, hop_length=256)[0]
+
+    def extract_mel(file_path):
+        # x, fs = sf.read(file_path)
+        x, fs = librosa.load(file_path, CONFIG.sample_rate)
+        if args.trim_silence:
+            x = trim_silence(x)
+        mel = ap.melspectrogram(x.astype('float32')).astype('float32')
+        linear = ap.spectrogram(x.astype('float32')).astype('float32')
+        file_name = os.path.basename(file_path).replace(".wav", "")
+        mel_file = file_name + ".mel"
+        linear_file = file_name + ".linear"
+        np.save(os.path.join(OUT_PATH, mel_file), mel, allow_pickle=False)
+        np.save(
+            os.path.join(OUT_PATH, linear_file), linear, allow_pickle=False)
+        mel_len = mel.shape[1]
+        linear_len = linear.shape[1]
+        wav_len = x.shape[0]
+        print(" > " + file_path, flush=True)
+        return file_path, mel_file, linear_file, str(wav_len), str(
+            mel_len), str(linear_len)
+
+    glob_path = os.path.join(DATA_PATH, "*.wav")
+    print(" > Reading wav: {}".format(glob_path))
+    file_names = glob.glob(glob_path, recursive=True)
+
+    if __name__ == "__main__":
+        print(" > Number of files: %i" % (len(file_names)))
+        if not os.path.exists(OUT_PATH):
+            os.makedirs(OUT_PATH)
+            print(" > A new folder created at {}".format(OUT_PATH))
+
+        r = []
+        if args.num_proc > 1:
+            print(" > Using {} processes.".format(args.num_proc))
+            with Pool(args.num_proc) as p:
+                r = list(
+                    tqdm.tqdm(
+                        p.imap(extract_mel, file_names),
+                        total=len(file_names)))
+                # r = list(p.imap(extract_mel, file_names))
+        else:
+            print(" > Using single process run.")
+            for file_name in file_names:
+                print(" > ", file_name)
+                r.append(extract_mel(file_name))
+
+        file_path = os.path.join(OUT_PATH, "meta_fftnet.csv")
+        file = open(file_path, "w")
+        for line in r:
+            line = ", ".join(line)
+            file.write(line + '\n')
+        file.close()

layers/attention.py

@@ -1,20 +1,21 @@
 import torch
 from torch import nn
 from torch.nn import functional as F
+from utils.generic_utils import sequence_mask
 
 
 class BahdanauAttention(nn.Module):
-    def __init__(self, annot_dim, query_dim, hidden_dim):
+    def __init__(self, annot_dim, query_dim, attn_dim):
         super(BahdanauAttention, self).__init__()
-        self.query_layer = nn.Linear(query_dim, hidden_dim, bias=True)
-        self.annot_layer = nn.Linear(annot_dim, hidden_dim, bias=True)
-        self.v = nn.Linear(hidden_dim, 1, bias=False)
+        self.query_layer = nn.Linear(query_dim, attn_dim, bias=True)
+        self.annot_layer = nn.Linear(annot_dim, attn_dim, bias=True)
+        self.v = nn.Linear(attn_dim, 1, bias=False)
 
     def forward(self, annots, query):
         """
         Shapes:
-            - query: (batch, 1, dim) or (batch, dim)
             - annots: (batch, max_time, dim)
+            - query: (batch, 1, dim) or (batch, dim)
         """
         if query.dim() == 2:
             # insert time-axis for broadcasting
@@ -23,37 +24,97 @@ class BahdanauAttention(nn.Module):
         processed_query = self.query_layer(query)
         processed_annots = self.annot_layer(annots)
         # (batch, max_time, 1)
-        alignment = self.v(nn.functional.tanh(
-            processed_query + processed_annots))
+        alignment = self.v(
+            nn.functional.tanh(processed_query + processed_annots))
         # (batch, max_time)
         return alignment.squeeze(-1)
 
 
-def get_mask_from_lengths(inputs, inputs_lengths):
-    """Get mask tensor from list of length
+class LocationSensitiveAttention(nn.Module):
+    """Location sensitive attention following
+    https://arxiv.org/pdf/1506.07503.pdf"""
 
-    Args:
-        inputs: Tensor in size (batch, max_time, dim)
-        inputs_lengths: array like
-    """
-    mask = inputs.data.new(inputs.size(0), inputs.size(1)).byte().zero_()
-    for idx, l in enumerate(inputs_lengths):
-        mask[idx][:l] = 1
-    return ~mask
+    def __init__(self,
+                 annot_dim,
+                 query_dim,
+                 attn_dim,
+                 kernel_size=7,
+                 filters=20):
+        super(LocationSensitiveAttention, self).__init__()
+        self.kernel_size = kernel_size
+        self.filters = filters
+        padding = int((kernel_size - 1) / 2)
+        self.loc_conv = nn.Conv1d(
+            2,
+            filters,
+            kernel_size=kernel_size,
+            stride=1,
+            padding=padding,
+            bias=False)
+        self.loc_linear = nn.Linear(filters, attn_dim)
+        self.query_layer = nn.Linear(query_dim, attn_dim, bias=True)
+        self.annot_layer = nn.Linear(annot_dim, attn_dim, bias=True)
+        self.v = nn.Linear(attn_dim, 1, bias=False)
+
+    def forward(self, annot, query, loc):
+        """
+        Shapes:
+            - annot: (batch, max_time, dim)
+            - query: (batch, 1, dim) or (batch, dim)
+            - loc: (batch, 2, max_time)
+        """
+        if query.dim() == 2:
+            # insert time-axis for broadcasting
+            query = query.unsqueeze(1)
+        loc_conv = self.loc_conv(loc)
+        loc_conv = loc_conv.transpose(1, 2)
+        processed_loc = self.loc_linear(loc_conv)
+        processed_query = self.query_layer(query)
+        processed_annots = self.annot_layer(annot)
+        alignment = self.v(
+            nn.functional.tanh(processed_query + processed_annots +
+                               processed_loc))
+        # (batch, max_time)
+        return alignment.squeeze(-1)
 
 
-class AttentionRNN(nn.Module):
-    def __init__(self, out_dim, annot_dim, memory_dim,
-                 score_mask_value=-float("inf")):
-        super(AttentionRNN, self).__init__()
-        self.rnn_cell = nn.GRUCell(out_dim + memory_dim, out_dim)
-        self.alignment_model = BahdanauAttention(annot_dim, out_dim, out_dim)
-        self.score_mask_value = score_mask_value
+class AttentionRNNCell(nn.Module):
+    def __init__(self, out_dim, rnn_dim, annot_dim, memory_dim, align_model):
+        r"""
+        General Attention RNN wrapper
 
-    def forward(self, memory, context, rnn_state, annotations,
-                mask=None, annotations_lengths=None):
-        if annotations_lengths is not None and mask is None:
-            mask = get_mask_from_lengths(annotations, annotations_lengths)
+        Args:
+            out_dim (int): context vector feature dimension.
+            rnn_dim (int): rnn hidden state dimension.
+            annot_dim (int): annotation vector feature dimension.
+            memory_dim (int): memory vector (decoder output) feature dimension.
+            align_model (str): 'b' for Bahdanau, 'ls' Location Sensitive alignment.
+        """
+        super(AttentionRNNCell, self).__init__()
+        self.align_model = align_model
+        self.rnn_cell = nn.GRUCell(annot_dim + memory_dim, rnn_dim)
+        # pick bahdanau or location sensitive attention
+        if align_model == 'b':
+            self.alignment_model = BahdanauAttention(annot_dim, rnn_dim,
+                                                     out_dim)
+        if align_model == 'ls':
+            self.alignment_model = LocationSensitiveAttention(
+                annot_dim, rnn_dim, out_dim)
+        else:
+            raise RuntimeError(" Wrong alignment model name: {}. Use\
+                'b' (Bahdanau) or 'ls' (Location Sensitive)."
+                               .format(align_model))
+
+    def forward(self, memory, context, rnn_state, annots, atten, mask):
+        """
+        Shapes:
+            - memory: (batch, 1, dim) or (batch, dim)
+            - context: (batch, dim)
+            - rnn_state: (batch, out_dim)
+            - annots: (batch, max_time, annot_dim)
+            - atten: (batch, 2, max_time)
+            - mask: (batch,)
+        """
         # Concat input query and previous context context
         rnn_input = torch.cat((memory, context), -1)
         # Feed it to RNN
@@ -62,16 +123,18 @@ class AttentionRNN(nn.Module):
         # Alignment
         # (batch, max_time)
         # e_{ij} = a(s_{i-1}, h_j)
-        alignment = self.alignment_model(annotations, rnn_output)
-        # TODO: needs recheck.
+        if self.align_model is 'b':
+            alignment = self.alignment_model(annots, rnn_output)
+        else:
+            alignment = self.alignment_model(annots, rnn_output, atten)
         if mask is not None:
-            mask = mask.view(query.size(0), -1)
-            alignment.data.masked_fill_(mask, self.score_mask_value)
+            mask = mask.view(memory.size(0), -1)
+            alignment.masked_fill_(1 - mask, -float("inf"))
         # Normalize context weight
         alignment = F.softmax(alignment, dim=-1)
         # Attention context vector
         # (batch, 1, dim)
         # c_i = \sum_{j=1}^{T_x} \alpha_{ij} h_j
-        context = torch.bmm(alignment.unsqueeze(1), annotations)
+        context = torch.bmm(alignment.unsqueeze(1), annots)
         context = context.squeeze(1)
         return rnn_output, context, alignment

layers/custom_layers.py

@@ -2,7 +2,6 @@
 import torch
 from torch import nn
 
-
 # class StopProjection(nn.Module):
 #     r""" Simple projection layer to predict the "stop token"
 

layers/losses.py

@@ -1,24 +1,10 @@
 import torch
 from torch.nn import functional
 from torch import nn
-
-
-# from https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1
-def _sequence_mask(sequence_length, max_len=None):
-    if max_len is None:
-        max_len = sequence_length.data.max()
-    batch_size = sequence_length.size(0)
-    seq_range = torch.arange(0, max_len).long()
-    seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
-    if sequence_length.is_cuda:
-        seq_range_expand = seq_range_expand.cuda()
-    seq_length_expand = (sequence_length.unsqueeze(1)
-                         .expand_as(seq_range_expand))
-    return seq_range_expand < seq_length_expand
+from utils.generic_utils import sequence_mask
 
 
 class L1LossMasked(nn.Module):
-
     def __init__(self):
         super(L1LossMasked, self).__init__()
 
@@ -44,14 +30,53 @@ class L1LossMasked(nn.Module):
         # target_flat: (batch * max_len, dim)
         target_flat = target.view(-1, target.shape[-1])
         # losses_flat: (batch * max_len, dim)
-        losses_flat = functional.l1_loss(input, target_flat, size_average=False,
-                                         reduce=False)
+        losses_flat = functional.l1_loss(
+            input, target_flat, size_average=False, reduce=False)
         # losses: (batch, max_len, dim)
         losses = losses_flat.view(*target.size())
-        
+
         # mask: (batch, max_len, 1)
-        mask = _sequence_mask(sequence_length=length,
-                              max_len=target.size(1)).unsqueeze(2)
+        mask = sequence_mask(
+            sequence_length=length, max_len=target.size(1)).unsqueeze(2)
+        losses = losses * mask.float()
+        loss = losses.sum() / (length.float().sum() * float(target.shape[2]))
+        return loss
+
+
+class MSELossMasked(nn.Module):
+    def __init__(self):
+        super(MSELossMasked, self).__init__()
+
+    def forward(self, input, target, length):
+        """
+        Args:
+            input: A Variable containing a FloatTensor of size
+                (batch, max_len, dim) which contains the
+                unnormalized probability for each class.
+            target: A Variable containing a LongTensor of size
+                (batch, max_len, dim) which contains the index of the true
+                class for each corresponding step.
+            length: A Variable containing a LongTensor of size (batch,)
+                which contains the length of each data in a batch.
+        Returns:
+            loss: An average loss value masked by the length.
+        """
+        input = input.contiguous()
+        target = target.contiguous()
+
+        # logits_flat: (batch * max_len, dim)
+        input = input.view(-1, input.shape[-1])
+        # target_flat: (batch * max_len, dim)
+        target_flat = target.view(-1, target.shape[-1])
+        # losses_flat: (batch * max_len, dim)
+        losses_flat = functional.mse_loss(
+            input, target_flat, size_average=False, reduce=False)
+        # losses: (batch, max_len, dim)
+        losses = losses_flat.view(*target.size())
+
+        # mask: (batch, max_len, 1)
+        mask = sequence_mask(
+            sequence_length=length, max_len=target.size(1)).unsqueeze(2)
         losses = losses * mask.float()
         loss = losses.sum() / (length.float().sum() * float(target.shape[2]))
         return loss

layers/tacotron.py

@@ -1,8 +1,7 @@
 # coding: utf-8
 import torch
 from torch import nn
-from .attention import AttentionRNN
-from .attention import get_mask_from_lengths
+from .attention import AttentionRNNCell
 
 
 class Prenet(nn.Module):
@@ -12,15 +11,16 @@ class Prenet(nn.Module):
     Args:
         in_features (int): size of the input vector
         out_features (int or list): size of each output sample.
-            If it is a list, for each value, there is created a new layer.  
+            If it is a list, for each value, there is created a new layer.
     """
 
     def __init__(self, in_features, out_features=[256, 128]):
         super(Prenet, self).__init__()
         in_features = [in_features] + out_features[:-1]
-        self.layers = nn.ModuleList(
-            [nn.Linear(in_size, out_size)
-             for (in_size, out_size) in zip(in_features, out_features)])
+        self.layers = nn.ModuleList([
+            nn.Linear(in_size, out_size)
+            for (in_size, out_size) in zip(in_features, out_features)
+        ])
         self.relu = nn.ReLU()
         self.dropout = nn.Dropout(0.5)
 
@@ -48,12 +48,21 @@ class BatchNormConv1d(nn.Module):
         - output: batch x dims
     """
 
-    def __init__(self, in_channels, out_channels, kernel_size, stride, padding,
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size,
+                 stride,
+                 padding,
                  activation=None):
         super(BatchNormConv1d, self).__init__()
-        self.conv1d = nn.Conv1d(in_channels, out_channels,
-                                kernel_size=kernel_size,
-                                stride=stride, padding=padding, bias=False)
+        self.conv1d = nn.Conv1d(
+            in_channels,
+            out_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            bias=False)
         # Following tensorflow's default parameters
         self.bn = nn.BatchNorm1d(out_channels, momentum=0.99, eps=1e-3)
         self.activation = activation
@@ -98,36 +107,66 @@ class CBHG(nn.Module):
             - output: batch x time x dim*2
     """
 
-    def __init__(self, in_features, K=16, projections=[128, 128], num_highways=4):
+    def __init__(self,
+                 in_features,
+                 K=16,
+                 conv_bank_features=128,
+                 conv_projections=[128, 128],
+                 highway_features=128,
+                 gru_features=128,
+                 num_highways=4):
         super(CBHG, self).__init__()
         self.in_features = in_features
+        self.conv_bank_features = conv_bank_features
+        self.highway_features = highway_features
+        self.gru_features = gru_features
+        self.conv_projections = conv_projections
         self.relu = nn.ReLU()
         # list of conv1d bank with filter size k=1...K
         # TODO: try dilational layers instead
-        self.conv1d_banks = nn.ModuleList(
-            [BatchNormConv1d(in_features, in_features, kernel_size=k, stride=1,
-                             padding=k // 2, activation=self.relu)
-                for k in range(1, K + 1)])
+        self.conv1d_banks = nn.ModuleList([
+            BatchNormConv1d(
+                in_features,
+                conv_bank_features,
+                kernel_size=k,
+                stride=1,
+                padding=k // 2,
+                activation=self.relu) for k in range(1, K + 1)
+        ])
         # max pooling of conv bank
         # TODO: try average pooling OR larger kernel size
         self.max_pool1d = nn.MaxPool1d(kernel_size=2, stride=1, padding=1)
-        out_features = [K * in_features] + projections[:-1]
-        activations = [self.relu] * (len(projections) - 1)
+        out_features = [K * conv_bank_features] + conv_projections[:-1]
+        activations = [self.relu] * (len(conv_projections) - 1)
         activations += [None]
         # setup conv1d projection layers
         layer_set = []
-        for (in_size, out_size, ac) in zip(out_features, projections, activations):
-            layer = BatchNormConv1d(in_size, out_size, kernel_size=3, stride=1,
-                                    padding=1, activation=ac)
+        for (in_size, out_size, ac) in zip(out_features, conv_projections,
+                                           activations):
+            layer = BatchNormConv1d(
+                in_size,
+                out_size,
+                kernel_size=3,
+                stride=1,
+                padding=1,
+                activation=ac)
             layer_set.append(layer)
         self.conv1d_projections = nn.ModuleList(layer_set)
         # setup Highway layers
-        self.pre_highway = nn.Linear(projections[-1], in_features, bias=False)
-        self.highways = nn.ModuleList(
-            [Highway(in_features, in_features) for _ in range(num_highways)])
+        if self.highway_features != conv_projections[-1]:
+            self.pre_highway = nn.Linear(
+                conv_projections[-1], highway_features, bias=False)
+        self.highways = nn.ModuleList([
+            Highway(highway_features, highway_features)
+            for _ in range(num_highways)
+        ])
         # bi-directional GPU layer
         self.gru = nn.GRU(
-            in_features, in_features, 1, batch_first=True, bidirectional=True)
+            gru_features,
+            gru_features,
+            1,
+            batch_first=True,
+            bidirectional=True)
 
     def forward(self, inputs):
         # (B, T_in, in_features)
@@ -137,7 +176,7 @@ class CBHG(nn.Module):
         if x.size(-1) == self.in_features:
             x = x.transpose(1, 2)
         T = x.size(-1)
-        # (B, in_features*K, T_in)
+        # (B, hid_features*K, T_in)
         # Concat conv1d bank outputs
         outs = []
         for conv1d in self.conv1d_banks:
@@ -145,35 +184,51 @@ class CBHG(nn.Module):
             out = out[:, :, :T]
             outs.append(out)
         x = torch.cat(outs, dim=1)
-        assert x.size(1) == self.in_features * len(self.conv1d_banks)
+        assert x.size(1) == self.conv_bank_features * len(self.conv1d_banks)
         x = self.max_pool1d(x)[:, :, :T]
         for conv1d in self.conv1d_projections:
             x = conv1d(x)
-        # (B, T_in, in_features)
-        # Back to the original shape
+        # (B, T_in, hid_feature)
         x = x.transpose(1, 2)
-        if x.size(-1) != self.in_features:
+        # Back to the original shape
+        x += inputs
+        if self.highway_features != self.conv_projections[-1]:
             x = self.pre_highway(x)
         # Residual connection
         # TODO: try residual scaling as in Deep Voice 3
         # TODO: try plain residual layers
-        x += inputs
         for highway in self.highways:
             x = highway(x)
-        # (B, T_in, in_features*2)
+        # (B, T_in, hid_features*2)
         # TODO: replace GRU with convolution as in Deep Voice 3
         self.gru.flatten_parameters()
         outputs, _ = self.gru(x)
         return outputs
 
 
+class EncoderCBHG(nn.Module):
+    def __init__(self):
+        super(EncoderCBHG, self).__init__()
+        self.cbhg = CBHG(
+            128,
+            K=16,
+            conv_bank_features=128,
+            conv_projections=[128, 128],
+            highway_features=128,
+            gru_features=128,
+            num_highways=4)
+
+    def forward(self, x):
+        return self.cbhg(x)
+
+
 class Encoder(nn.Module):
     r"""Encapsulate Prenet and CBHG modules for encoder"""
 
     def __init__(self, in_features):
         super(Encoder, self).__init__()
         self.prenet = Prenet(in_features, out_features=[256, 128])
-        self.cbhg = CBHG(128, K=16, projections=[128, 128])
+        self.cbhg = EncoderCBHG()
 
     def forward(self, inputs):
         r"""
@@ -188,6 +243,21 @@ class Encoder(nn.Module):
         return self.cbhg(inputs)
 
 
+class PostCBHG(nn.Module):
+    def __init__(self, mel_dim):
+        super(PostCBHG, self).__init__()
+        self.cbhg = CBHG(
+            mel_dim,
+            K=8,
+            conv_bank_features=80,
+            conv_projections=[160, mel_dim],
+            highway_features=80,
+            gru_features=80,
+            num_highways=4)
+    def forward(self, x):
+        return self.cbhg(x)
+
+
 class Decoder(nn.Module):
     r"""Decoder module.
 
@@ -195,20 +265,25 @@ class Decoder(nn.Module):
         in_features (int): input vector (encoder output) sample size.
         memory_dim (int): memory vector (prev. time-step output) sample size.
         r (int): number of outputs per time step.
-        eps (float): threshold for detecting the end of a sentence.
     """
 
     def __init__(self, in_features, memory_dim, r):
         super(Decoder, self).__init__()
         self.r = r
+        self.in_features = in_features
         self.max_decoder_steps = 200
         self.memory_dim = memory_dim
         # memory -> |Prenet| -> processed_memory
         self.prenet = Prenet(memory_dim * r, out_features=[256, 128])
-        # processed_inputs, processed_memory -> |Attention| -> Attention, Alignment, RNN_State
-        self.attention_rnn = AttentionRNN(256, in_features, 128)
+        # processed_inputs, processed_memory -> |Attention| -> Attention, attention, RNN_State
+        self.attention_rnn = AttentionRNNCell(
+            out_dim=128,
+            rnn_dim=256,
+            annot_dim=in_features,
+            memory_dim=128,
+            align_model='ls')
         # (processed_memory | attention context) -> |Linear| -> decoder_RNN_input
-        self.project_to_decoder_in = nn.Linear(256+in_features, 256)
+        self.project_to_decoder_in = nn.Linear(256 + in_features, 256)
         # decoder_RNN_input -> |RNN| -> RNN_state
         self.decoder_rnns = nn.ModuleList(
             [nn.GRUCell(256, 256) for _ in range(2)])
@@ -216,7 +291,7 @@ class Decoder(nn.Module):
         self.proj_to_mel = nn.Linear(256, memory_dim * r)
         self.stopnet = StopNet(r, memory_dim)
 
-    def forward(self, inputs, memory=None):
+    def forward(self, inputs, memory=None, mask=None):
         """
         Decoder forward step.
 
@@ -228,34 +303,42 @@ class Decoder(nn.Module):
             memory (None): Decoder memory (autoregression. If None (at eval-time),
               decoder outputs are used as decoder inputs. If None, it uses the last
               output as the input.
+            mask (None): Attention mask for sequence padding.
 
         Shapes:
             - inputs: batch x time x encoder_out_dim
             - memory: batch x #mel_specs x mel_spec_dim
         """
         B = inputs.size(0)
+        T = inputs.size(1)
         # Run greedy decoding if memory is None
         greedy = not self.training
         if memory is not None:
             # Grouping multiple frames if necessary
             if memory.size(-1) == self.memory_dim:
                 memory = memory.view(B, memory.size(1) // self.r, -1)
-                " !! Dimension mismatch {} vs {} * {}".format(memory.size(-1),
-                                                              self.memory_dim, self.r)
+                " !! Dimension mismatch {} vs {} * {}".format(
+                    memory.size(-1), self.memory_dim, self.r)
             T_decoder = memory.size(1)
-        # go frame - 0 frames tarting the sequence
+        # go frame as zeros matrix
         initial_memory = inputs.data.new(B, self.memory_dim * self.r).zero_()
-        # Init decoder states
+        # decoder states
         attention_rnn_hidden = inputs.data.new(B, 256).zero_()
-        decoder_rnn_hiddens = [inputs.data.new(B, 256).zero_()
-            for _ in range(len(self.decoder_rnns))]
-        current_context_vec = inputs.data.new(B, 256).zero_()
-        stopnet_rnn_hidden = inputs.data.new(B, self.r * self.memory_dim).zero_()
+        decoder_rnn_hiddens = [
+            inputs.data.new(B, 256).zero_()
+            for _ in range(len(self.decoder_rnns))
+        ]
+        current_context_vec = inputs.data.new(B, self.in_features).zero_()
+        stopnet_rnn_hidden = inputs.data.new(B,
+                                             self.r * self.memory_dim).zero_()
+        # attention states
+        attention = inputs.data.new(B, T).zero_()
+        attention_cum = inputs.data.new(B, T).zero_()
         # Time first (T_decoder, B, memory_dim)
         if memory is not None:
             memory = memory.transpose(0, 1)
         outputs = []
-        alignments = []
+        attentions = []
         stop_tokens = []
         t = 0
         memory_input = initial_memory
@@ -264,12 +347,16 @@ class Decoder(nn.Module):
                 if greedy:
                     memory_input = outputs[-1]
                 else:
-                    memory_input = memory[t-1]
+                    memory_input = memory[t - 1]
             # Prenet
             processed_memory = self.prenet(memory_input)
             # Attention RNN
-            attention_rnn_hidden, current_context_vec, alignment = self.attention_rnn(
-                processed_memory, current_context_vec, attention_rnn_hidden, inputs)
+            attention_cat = torch.cat(
+                (attention.unsqueeze(1), attention_cum.unsqueeze(1)), dim=1)
+            attention_rnn_hidden, current_context_vec, attention = self.attention_rnn(
+                processed_memory, current_context_vec, attention_rnn_hidden,
+                inputs, attention_cat, mask)
+            attention_cum += attention
             # Concat RNN output and attention context vector
             decoder_input = self.project_to_decoder_in(
                 torch.cat((attention_rnn_hidden, current_context_vec), -1))
@@ -284,45 +371,53 @@ class Decoder(nn.Module):
             output = self.proj_to_mel(decoder_output)
             stop_input = output
             # predict stop token
-            stop_token, stopnet_rnn_hidden = self.stopnet(stop_input, stopnet_rnn_hidden)
+            stop_token, stopnet_rnn_hidden = self.stopnet(
+                stop_input, stopnet_rnn_hidden)
             outputs += [output]
-            alignments += [alignment]
+            attentions += [attention]
             stop_tokens += [stop_token]
             t += 1
-            if (not greedy and self.training) or (greedy and memory is not None):
+            if (not greedy and self.training) or (greedy
+                                                  and memory is not None):
                 if t >= T_decoder:
                     break
             else:
-                if t > inputs.shape[1]/2 and stop_token > 0.8:
+                if t > inputs.shape[1] / 4 and stop_token > 0.6:
                     break
                 elif t > self.max_decoder_steps:
-                    print(" !! Decoder stopped with 'max_decoder_steps'. \
-                          Something is probably wrong.")
+                    print("   | > Decoder stopped with 'max_decoder_steps")
                     break
         assert greedy or len(outputs) == T_decoder
         # Back to batch first
-        alignments = torch.stack(alignments).transpose(0, 1)
+        attentions = torch.stack(attentions).transpose(0, 1)
         outputs = torch.stack(outputs).transpose(0, 1).contiguous()
         stop_tokens = torch.stack(stop_tokens).transpose(0, 1)
-        return outputs, alignments, stop_tokens
+        return outputs, attentions, stop_tokens
+
 
-    
 class StopNet(nn.Module):
     r"""
     Predicting stop-token in decoder.
-    
+
     Args:
         r (int): number of output frames of the network.
         memory_dim (int): feature dimension for each output frame.
     """
-    
+
     def __init__(self, r, memory_dim):
+        r"""
+        Predicts the stop token to stop the decoder at testing time
+
+        Args:
+            r (int): number of network output frames.
+            memory_dim (int): single feature dim of a single network output frame.
+        """
         super(StopNet, self).__init__()
         self.rnn = nn.GRUCell(memory_dim * r, memory_dim * r)
         self.relu = nn.ReLU()
         self.linear = nn.Linear(r * memory_dim, 1)
         self.sigmoid = nn.Sigmoid()
-        
+
     def forward(self, inputs, rnn_hidden):
         """
         Args:
@@ -333,4 +428,4 @@ class StopNet(nn.Module):
         outputs = self.relu(rnn_hidden)
         outputs = self.linear(outputs)
         outputs = self.sigmoid(outputs)
-        return outputs, rnn_hidden
+        return outputs, rnn_hidden

models/tacotron.py

@@ -2,33 +2,37 @@
 import torch
 from torch import nn
 from utils.text.symbols import symbols
-from layers.tacotron import Prenet, Encoder, Decoder, CBHG
+from layers.tacotron import Prenet, Encoder, Decoder, PostCBHG
 
 
 class Tacotron(nn.Module):
-    def __init__(self, embedding_dim=256, linear_dim=1025, mel_dim=80,
-                 r=5, padding_idx=None):
+    def __init__(self,
+                 embedding_dim=256,
+                 linear_dim=1025,
+                 mel_dim=80,
+                 r=5,
+                 padding_idx=None):
         super(Tacotron, self).__init__()
         self.r = r
         self.mel_dim = mel_dim
         self.linear_dim = linear_dim
-        self.embedding = nn.Embedding(len(symbols), embedding_dim,
-                                      padding_idx=padding_idx)
+        self.embedding = nn.Embedding(
+            len(symbols), embedding_dim, padding_idx=padding_idx)
         print(" | > Number of characters : {}".format(len(symbols)))
         self.embedding.weight.data.normal_(0, 0.3)
         self.encoder = Encoder(embedding_dim)
         self.decoder = Decoder(256, mel_dim, r)
-        self.postnet = CBHG(mel_dim, K=8, projections=[256, mel_dim])
-        self.last_linear = nn.Linear(mel_dim * 2, linear_dim)
+        self.postnet = PostCBHG(mel_dim)
+        self.last_linear = nn.Linear(self.postnet.cbhg.gru_features * 2, linear_dim)
 
-    def forward(self, characters, mel_specs=None):
+    def forward(self, characters, mel_specs=None, mask=None):
         B = characters.size(0)
         inputs = self.embedding(characters)
         # batch x time x dim
         encoder_outputs = self.encoder(inputs)
         # batch x time x dim*r
         mel_outputs, alignments, stop_tokens = self.decoder(
-            encoder_outputs, mel_specs)
+            encoder_outputs, mel_specs, mask)
         # Reshape
         # batch x time x dim
         mel_outputs = mel_outputs.view(B, -1, self.mel_dim)

notebooks/TacotronPlayGround.ipynb

@@ -0,0 +1,783 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Populating the interactive namespace from numpy and matplotlib\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/erogol/miniconda3/envs/pytorch4/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['plt']\n",
+      "`%matplotlib` prevents importing * from pylab and numpy\n",
+      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "import os\n",
+    "import sys\n",
+    "import io\n",
+    "import torch \n",
+    "import time\n",
+    "import numpy as np\n",
+    "from collections import OrderedDict\n",
+    "from matplotlib import pylab as plt\n",
+    "\n",
+    "%pylab inline\n",
+    "rcParams[\"figure.figsize\"] = (16,5)\n",
+    "sys.path.append('/home/erogol/projects/')\n",
+    "\n",
+    "import librosa\n",
+    "import librosa.display\n",
+    "\n",
+    "from TTS.models.tacotron import Tacotron \n",
+    "from TTS.layers import *\n",
+    "from TTS.utils.data import *\n",
+    "from TTS.utils.audio import AudioProcessor\n",
+    "from TTS.utils.generic_utils import load_config\n",
+    "from TTS.utils.text import text_to_sequence\n",
+    "\n",
+    "import IPython\n",
+    "from IPython.display import Audio\n",
+    "from utils import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[0m\u001b[01;31mcheckpoint_33464.pth.tar\u001b[0m  \u001b[01;31mcheckpoint_36096.pth.tar\u001b[0m\r\n",
+      "\u001b[01;31mcheckpoint_33840.pth.tar\u001b[0m  \u001b[01;31mcheckpoint_36472.pth.tar\u001b[0m\r\n",
+      "\u001b[01;31mcheckpoint_34216.pth.tar\u001b[0m  \u001b[01;31mcheckpoint_36848.pth.tar\u001b[0m\r\n",
+      "\u001b[01;31mcheckpoint_34592.pth.tar\u001b[0m  \u001b[01;31mcheckpoint_37224.pth.tar\u001b[0m\r\n",
+      "\u001b[01;31mcheckpoint_34968.pth.tar\u001b[0m  \u001b[01;34mcheckpoints\u001b[0m/\r\n",
+      "\u001b[01;31mcheckpoint_35344.pth.tar\u001b[0m  config.json\r\n",
+      "\u001b[01;31mcheckpoint_35720.pth.tar\u001b[0m  events.out.tfevents.1526635608.mlc1\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "ls /data/shared/erogol_models/May-18-2018_02:26AM-los_sen_attn-debug "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def tts(model, text, CONFIG, use_cuda, ap, figures=True):\n",
+    "    t_1 = time.time()\n",
+    "    waveform, alignment, spectrogram, stop_tokens = create_speech(model, text, CONFIG, use_cuda, ap) \n",
+    "    print(\" >  Run-time: {}\".format(time.time() - t_1))\n",
+    "    if figures:                                                                                                         \n",
+    "        visualize(alignment, spectrogram, stop_tokens, CONFIG)                                                                       \n",
+    "    IPython.display.display(Audio(waveform, rate=CONFIG.sample_rate))  \n",
+    "    return alignment, spectrogram, stop_tokens"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Set constants\n",
+    "ROOT_PATH = '/data/shared/erogol_models/May-18-2018_02:26AM-los_sen_attn-debug/'\n",
+    "MODEL_PATH = ROOT_PATH + '/checkpoint_37224.pth.tar'\n",
+    "CONFIG_PATH = ROOT_PATH + '/config.json'\n",
+    "OUT_FOLDER = ROOT_PATH + '/test/'\n",
+    "CONFIG = load_config(CONFIG_PATH)\n",
+    "use_cuda = True"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " | > Number of characters : 149\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Tacotron(\n",
+       "  (embedding): Embedding(149, 256)\n",
+       "  (encoder): Encoder(\n",
+       "    (prenet): Prenet(\n",
+       "      (layers): ModuleList(\n",
+       "        (0): Linear(in_features=256, out_features=256, bias=True)\n",
+       "        (1): Linear(in_features=256, out_features=128, bias=True)\n",
+       "      )\n",
+       "      (relu): ReLU()\n",
+       "      (dropout): Dropout(p=0.5)\n",
+       "    )\n",
+       "    (cbhg): CBHG(\n",
+       "      (relu): ReLU()\n",
+       "      (conv1d_banks): ModuleList(\n",
+       "        (0): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(1,), stride=(1,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (1): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(2,), stride=(1,), padding=(1,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (2): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (3): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(2,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (4): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (5): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(6,), stride=(1,), padding=(3,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (6): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (7): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (8): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(9,), stride=(1,), padding=(4,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (9): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(10,), stride=(1,), padding=(5,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (10): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(5,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (11): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(12,), stride=(1,), padding=(6,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (12): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(13,), stride=(1,), padding=(6,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (13): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(14,), stride=(1,), padding=(7,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (14): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(15,), stride=(1,), padding=(7,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (15): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(16,), stride=(1,), padding=(8,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "      )\n",
+       "      (max_pool1d): MaxPool1d(kernel_size=2, stride=1, padding=1, dilation=1, ceil_mode=False)\n",
+       "      (conv1d_projections): ModuleList(\n",
+       "        (0): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(2048, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "          (activation): ReLU()\n",
+       "        )\n",
+       "        (1): BatchNormConv1d(\n",
+       "          (conv1d): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)\n",
+       "          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        )\n",
+       "      )\n",
+       "      (pre_highway): Linear(in_features=128, out_features=128, bias=False)\n",
+       "      (highways): ModuleList(\n",
+       "        (0): Highway(\n",
+       "          (H): Linear(in_features=128, out_features=128, bias=True)\n",
+       "          (T): Linear(in_features=128, out_features=128, bias=True)\n",
+       "          (relu): ReLU()\n",
+       "          (sigmoid): Sigmoid()\n",
+       "        )\n",
+       "        (1): Highway(\n",
+       "          (H): Linear(in_features=128, out_features=128, bias=True)\n",
+       "          (T): Linear(in_features=128, out_features=128, bias=True)\n",
+       "          (relu): ReLU()\n",
+       "          (sigmoid): Sigmoid()\n",
+       "        )\n",
+       "        (2): Highway(\n",
+       "          (H): Linear(in_features=128, out_features=128, bias=True)\n",
+       "          (T): Linear(in_features=128, out_features=128, bias=True)\n",
+       "          (relu): ReLU()\n",
+       "          (sigmoid): Sigmoid()\n",
+       "        )\n",
+       "        (3): Highway(\n",
+       "          (H): Linear(in_features=128, out_features=128, bias=True)\n",
+       "          (T): Linear(in_features=128, out_features=128, bias=True)\n",
+       "          (relu): ReLU()\n",
+       "          (sigmoid): Sigmoid()\n",
+       "        )\n",
+       "      )\n",
+       "      (gru): GRU(128, 128, batch_first=True, bidirectional=True)\n",
+       "    )\n",
+       "  )\n",
+       "  (decoder): Decoder(\n",
+       "    (prenet): Prenet(\n",
+       "      (layers): ModuleList(\n",
+       "        (0): Linear(in_features=400, out_features=256, bias=True)\n",
+       "        (1): Linear(in_features=256, out_features=128, bias=True)\n",
+       "      )\n",
+       "      (relu): ReLU()\n",
+       "      (dropout): Dropout(p=0.5)\n",
+       "    )\n",
+       "    (attention_rnn): AttentionRNN(\n",
+       "      (rnn_cell): GRUCell(384, 256)\n",
+       "      (alignment_model): LocationSensitiveAttention(\n",
+       "        (loc_conv): Conv1d(2, 32, kernel_size=(31,), stride=(1,), padding=(15,), bias=False)\n",
+       "        (loc_linear): Linear(in_features=32, out_features=256, bias=True)\n",
+       "        (query_layer): Linear(in_features=256, out_features=256, bias=True)\n",
+       "        (annot_layer): Linear(in_features=256, out_features=256, bias=True)\n",
+       "        (v): Linear(in_features=256, out_features=1, bias=False)\n",
+       "      )\n",
+       "    )\n",
+       "    (project_to_decoder_in): Linear(in_features=512, out_features=256, bias=True)\n",
+       "    (decoder_rnns): ModuleList(\n",
+       "      (0): GRUCell(256, 256)\n",
+       "      (1): GRUCell(256, 256)\n",
+       "    )\n",
+       "    (proj_to_mel): Linear(in_features=256, out_features=400, bias=True)\n",
+       "    (stopnet): StopNet(\n",
+       "      (rnn): GRUCell(400, 400)\n",
+       "      (relu): ReLU()\n",
+       "      (linear): Linear(in_features=400, out_features=1, bias=True)\n",
+       "      (sigmoid): Sigmoid()\n",
+       "    )\n",
+       "  )\n",
+       "  (postnet): CBHG(\n",
+       "    (relu): ReLU()\n",
+       "    (conv1d_banks): ModuleList(\n",
+       "      (0): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(80, 80, kernel_size=(1,), stride=(1,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "      (1): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(80, 80, kernel_size=(2,), stride=(1,), padding=(1,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "      (2): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(80, 80, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "      (3): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(80, 80, kernel_size=(4,), stride=(1,), padding=(2,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "      (4): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(80, 80, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "      (5): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(80, 80, kernel_size=(6,), stride=(1,), padding=(3,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "      (6): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(80, 80, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "      (7): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(80, 80, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "    )\n",
+       "    (max_pool1d): MaxPool1d(kernel_size=2, stride=1, padding=1, dilation=1, ceil_mode=False)\n",
+       "    (conv1d_projections): ModuleList(\n",
+       "      (0): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(640, 256, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)\n",
+       "        (bn): BatchNorm1d(256, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "        (activation): ReLU()\n",
+       "      )\n",
+       "      (1): BatchNormConv1d(\n",
+       "        (conv1d): Conv1d(256, 80, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)\n",
+       "        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n",
+       "      )\n",
+       "    )\n",
+       "    (pre_highway): Linear(in_features=80, out_features=80, bias=False)\n",
+       "    (highways): ModuleList(\n",
+       "      (0): Highway(\n",
+       "        (H): Linear(in_features=80, out_features=80, bias=True)\n",
+       "        (T): Linear(in_features=80, out_features=80, bias=True)\n",
+       "        (relu): ReLU()\n",
+       "        (sigmoid): Sigmoid()\n",
+       "      )\n",
+       "      (1): Highway(\n",
+       "        (H): Linear(in_features=80, out_features=80, bias=True)\n",
+       "        (T): Linear(in_features=80, out_features=80, bias=True)\n",
+       "        (relu): ReLU()\n",
+       "        (sigmoid): Sigmoid()\n",
+       "      )\n",
+       "      (2): Highway(\n",
+       "        (H): Linear(in_features=80, out_features=80, bias=True)\n",
+       "        (T): Linear(in_features=80, out_features=80, bias=True)\n",
+       "        (relu): ReLU()\n",
+       "        (sigmoid): Sigmoid()\n",
+       "      )\n",
+       "      (3): Highway(\n",
+       "        (H): Linear(in_features=80, out_features=80, bias=True)\n",
+       "        (T): Linear(in_features=80, out_features=80, bias=True)\n",
+       "        (relu): ReLU()\n",
+       "        (sigmoid): Sigmoid()\n",
+       "      )\n",
+       "    )\n",
+       "    (gru): GRU(80, 80, batch_first=True, bidirectional=True)\n",
+       "  )\n",
+       "  (last_linear): Linear(in_features=160, out_features=1025, bias=True)\n",
+       ")"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# load the model\n",
+    "model = Tacotron(CONFIG.embedding_size, CONFIG.num_freq, CONFIG.num_mels, CONFIG.r)\n",
+    "\n",
+    "# load the audio processor\n",
+    "\n",
+    "ap = AudioProcessor(CONFIG.sample_rate, CONFIG.num_mels, CONFIG.min_level_db,\n",
+    "                    CONFIG.frame_shift_ms, CONFIG.frame_length_ms, CONFIG.preemphasis,\n",
+    "                    CONFIG.ref_level_db, CONFIG.num_freq, CONFIG.power, griffin_lim_iters=30)         \n",
+    "\n",
+    "\n",
+    "# load model state\n",
+    "if use_cuda:\n",
+    "    cp = torch.load(MODEL_PATH)\n",
+    "else:\n",
+    "    cp = torch.load(MODEL_PATH, map_location=lambda storage, loc: storage)\n",
+    "\n",
+    "# load the model\n",
+    "model.load_state_dict(cp['model'])\n",
+    "if use_cuda:\n",
+    "    model.cuda()\n",
+    "model.eval()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### EXAMPLES FROM TRAINING SET"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df = pd.read_csv('/data/shared/KeithIto/LJSpeech-1.0/metadata_val.csv', delimiter='|')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Hosty took the necessary steps to have the Dallas office of the FBI, rather than the New Orleans office, reestablished as the office with principal responsibility.\n",
+      " >  Run-time: 9.547307014465332\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRmT7BQBXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YUD7BQD7//f/+/8AAP7/7//h/+T/8f/2/+z/5v/y/wYAEwATAAwACAAOABYADgD4/+f/6f/1//b/6//p//j/BwAHAP///f8FABAADgADAPv/+P/8/wIABwABAPP/7v/7/wwACgD2/+f/8v8JAA4A+//r//r/FgAgAA8A/v///wkACgD7/+r/6v/7/wYABQD//wAAEQAfABYA+f/n//D/AQABAO7/3v/h//L/BAAMAAkABwAQABwAHwAZABUAFwANAPn/5//l//P//v/1/93/z//i/wQAFAAGAPr/BAAYACEAHgAVAAwACAAJAAcA+v/n/+v//v8FAPL/2f/e//n/EQAZABIACQAQACcANQAlAAAA6f/0/wUAAADl/9D/2//5/wwAAQDw//P/CAAbABIA/P/w//j/CQAQAAYA8P/n//z/HQAhAAIA7v/8/xQAFAD4/9z/4/8EABIA8//C/7f/5P8VAA4A0v+v/9//OgByAFQADgAAAD4AcwBRAPT/vP/a/wwA/P+t/3r/n//1/xwA/v/a//X/QwB1AFcAFwAAAB8AQwAxAPH/zP/f//z/9P/O/7f/x//1/xsAFQD1//X/KABXADsA8P/Q//7/KwAVANP/q//H/wUAIwD5/7z/z/8jAFIALAD6/wIAJwAdAPD/1//i//P/7//U/77/zP8AACsAGwDt/+//JwBRAEUAJAAdACYALQAuABUA5f/D/8D/wf+m/4L/h/+4//H/GwA7AFoAdwCQAKUAoAByADgADQDf/5n/XP9U/3f/gv9b/0v/j/8MAHAAiAB9AI8AywD0ANMAdgAQANL/oP9f/zb/Tf+H/6D/l/+g/+D/LQBGABwA7v8OAHQAugCpAHEAXQBxAGQA/f9s/yb/XP+3/8b/kP+I//b/gACeAEQA1P+i/7r/9/8nAC4ALwBhAK0AsABKAMr/d/9a/1r/av91/4n/2v9wAAUBJQGrAAcAuv/J/9z/tP9t/2X/zP9WAKoArgCCAEYA9v+d/0X/+f7V/gT/kf8UADAASADjAMYBHAKPAb4ARQAoAPX/Yf+r/kf+YP68/h//h//c/wsAJABYAKwA7gD7AMIAYgA2AIcA3wCcANH/Hv/9/jD/Q/8F/7T+wf5m/1sA+QAaAT0BvgEwAvMBDwE4ANf/h//u/ir+w/3//Xr+zv4e/7z/iQADAQoBDQEwATsB/gByAMn/Yv+E/+n/8/+T/3v//v+RAKEATwAXACgAPADz/23/Jf9Z/6//x/+8/9P/HwB+AMYAwgCFAEkACwCy/2P/UP9T/y//EP9w/z4A5QD7AKkAkgAIAYsBXgFmAGn/Hf9a/3j/HP+B/kT+uv55/+v/AQBMAOsAOAH3ANAAKwF9ASUBQQB6/z3/fP/G/4z/0f5G/oL+Vf8QAE8AOgAqAFQAuwAYAQoBpwBtAIMAewAWAKb/gP+I/3P/Yf9+/5T/ev96/9L/TQB7AEIAFQAuAG4AjABVAAMA0f+7/8H/0P/k/9b/u//1/3EAwQCoAHQAcACBAHwAMACL/+X+wf4V/zD/vP5n/vv+HQDLAKsAcwDwAOEBSAKzAdYAiQC1AIUAkf+X/nT+8/4K/27+8f04/ij/BwBTAEMASQDHAI8BBwLMAQoBiQCOAJIAKgB8//n+wP7V/hT/Nv86/07/qf8eAGkAiQCkAOsAGwHfAFEA7//f/9L/h/8A/8f+HP+R/9b//v9WAOkATAE8ARYBHQEHAYQAw/8r/9j+uv7G/tj+zP7h/lT/FQDKABwBJwEaAQgB7wDJAIkAGQCQ/zj/If8a/xf/I/9N/5P/zf/4/zwAogD5AAMB0gCpAJUAZgAEAI7/QP8l/wr/0v61/uv+TP+e/+X/SQCyAOsAAAEMAT4BiAGMARMBUgDX/9D/1/9f/3D+tf3C/XL+DP9a/6X/PgApAdIB7AHGAcEBygFVATIAKP/2/kT/PP+f/iD+gf6U/6AABAHRAJ0A8gCHAYgBzgD7/43/V/8b/83+rP6y/sv+Kv+a/+T/DwBmAPIAUwFFAeUAjgBaAEMALQDb/0z/2/7o/jv/V/8r/xj/aP+y/6//lv+9/yYAYABQADkAOQBoAKUArgBnAPH/xP/v/xcA5v+a/8H/GwBnAI0AkgCkANoAIAEnAcgAQwASABsA0f9O/wb/Jf+M/9H/7f/8/xUAiQAbAU0B4QBRAG0ADwFLAdEATAAzAEwANwDX/17/5v6V/ov+f/5o/n/+wf4s/5L/1v/m/9j/BwBAABcAtf+b/7v/mv9Q/zj/Xf9i/zT/K/9U/4P/nf+Y/3D/bf/r/4MAlABaAJAASAG0AXEBEAH4AMgANwCW/1f/Tf8g/wD/Pv8JAPoAlwH9AWwC6QILA7wCPwLSAUABWACj/0L/CP/z/g//cf/K//7/WwAJAbgB5QGBAR8BLwFPAeAA+v8x/8r+Y/7N/RT9jPyk/AH9O/0y/VH9Hv7z/k//Zv+d//P/5P99/zj/Hf/q/ov+LP75/e79Ef5z/u/+Qf9f/4f/5/+TAC8BTgEgARQBfAENAjUC5gGRAaMB/QEmArwB+QCSALkABAHbAGMAYAAKAcMBAgI7AtwC0AONBLcErASgBKEEoAQ9BD4D4wHNADYA0/8d/wj+P/3+/CX9X/1o/W/9a/1n/W79cP0v/XT8wvtn+0P7DPuH+k/6vvqa+3j83/wh/af9cf40/2f/Cv++/uL+IP/1/mv+Mv56/tL++P4l/7X/lgB9ATsC2QJMA6gD8APdA18DrAL/AXMB/ADKAPsAWwHfAXcCUQNuBJcFugaIBxUIaAjSCGoJnwlrCcMICQhtB44GbAX4A08CdwCV/uD8LPtZ+Yb3HvYg9Ur00fMM9BT1MPYV9zz49fkd/Mn9sf75/tD+pf5g/t/9uvxQ+3n6W/pg+tn5iPkp+mv7X/yc/PP8AP6D/28AWAAFAFkAQwGlAUMB4ABFAS8CnQKXAroChAOrBFgFWAUiBZEFxAbgBzEIIwjZCJcKeQyuDXsOrA8tESkSGRI3ETMQCA8LDdcJtgWTARr+7fpP91Pzv+947ZfsQuwG7CHsRO3Q79byU/Vr99H5tPwE/9n/qP9k/yn/Mv5g/FD6yfi294T2qfVO9aX1hfZ098X4RfoK/Ar+gv89AIAA3wAhAWYA4v6g/Wf9lv1v/aX9Bf94AfID0gWzB+cJ+gsbDU4NlQ2uDmgQuBGXEgQU6RbEGr0dTB/9H4EgeyCKHnQapBTiDYkGaP7V9WXtF+YR4Dzb79eG1qDXv9oW3yzk1+mA8M73b/6kA54HyAraDMUMuwrtB9gE+ADr+3f20PGJ7lXs6eoO6r3puuqR7a7xp/W8+E/7//1HAEkBUAGSAG//xv3N+9H69fpP/D3+YABLA+0GaQuuD9wSJhVMF9oZGRy0HdUehCDtIoElFigCKjkreiuKKkco/yN5HfsUBQvx/9vzlOc73EHS2clwwwnAPMCBw4LJItLS3IHogPR0AMoLbxV8HK8gziEZICscfxb7DrAF0/ub8rrqEuSV3sXa69hY2ZTb7N5e4yLp7e/r9er5kPz8/kcB5wFzADL+VfzQ+zj8df3s/1QDIQiBDSgT5hjhHYMi6iWOKKkq9it7LZwu/y82MXIxdTHmMDQwAC55KBEg3RVKCyP/LfCT3yDQMMRauuCxA6xHq/qwRLpOxSjS9+GH9I0GyxX1Ifgr2jPNN6A2DDGuKNkeoxPFBjL55etP4EPXntBLzInJ3sgfyw7QhtYQ3ZnjXeoL8Yr2kvqH/VL/NQBXAKIA1wHUA5wGIgrDDnYU1xoaIYYmmisJMHQzkzVrNnY3TjhsOOI32TYjNsw0STIOLkIn6R3QEQgEGfWZ5CDTksF1stCnZ6HEnqiftaXFsQDCG9QC54f7yhAKJC8zGz7xRU9KxEmXQ8Y4TSvVHPANEP5r7XHdLdDCxjjAobs7ufy5573Iw4vKHtIt20jkBuzo8X/2n/sRABEDygRCBrsJrA4jFBQZIx7oJCAt5jR4Od47RT6VQRtEEEPdP+M8UTvWOSM2tjDaKo0kfRwBEbsCnPNV5JLUs8MJszuleJwQmbqZ8p0spuKzCscu3RT0NQpdH1IzYER6USFZu1oQV/1O5EPRNBciIg4b+wXq2tgsyOm6zLJMrzGtLazrrYCzjLwzxiXPedcP4H7p8fFg+Gz9hQKeCAoOjBK5F9oezCeNLwE2MDywQsRIVEuOS75Kp0mvR8RCFz0gOOczyi5LJwwfgha0DAMAFvGx4TfSsMLgskOlP5uslW2VPZmmoRiunr9g1rjuqAbeHLgyW0dAV0NhR2VLZYRg/VXsRn00IyGjDKb3BeOYz5e/cLLRqCKibZ7bnrKhMKetrga4KcPezVLYXuI87Bb2+f5VB90OARYOHjInxzDyOLo/8UbATuxU3lZtVY1T5lFBTklHzT5+N2MxRSq6IakYlQ/DBfv5WOx33QTOLL/hsZKlt5p5k02SuZfWoLesNr0i0+vslgaCHkg1UkonXDtoE253blhqKmLEVE1CmyzGFigCk+0l2LPD3LJtpnqdxJZek1uTuZbanMikZq80uwrIh9QC4HHsEPkdBvUQ6hmzI4Yu7TmHQl5JPVC6V9Zdil//XWZaaFcwU8BLHUJBNyQveSf4HdUS7AY6/bHyeOVT1njHSLu+r3Ck35pSlZuVXJo+o16wmsEe1xrvfgitIQ45sU2JXjFreHIpdB9wR2dbWiRJBzVnHoQHQfGW25XHf7R/pDKYEJAAjOiJ/Yo/j0+XoaKyrua7vMlu2NfnOPY5BKoRmB7SKpg16z9VSdtR7lhSXotiGmRLYwpgvFp5VExM6UJYOPwsMCJPFx8MKQAA9FXoIdw5z//BYLbjrF2k+J2EmsmbIaKqq6a5OcvY3zL3rg4WJzcLine truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAa4CAYAAAD7hW7eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XmYbVdZJ/7vqrpTbshIIIQkSsCEuVEMJA6NQZTBRmM7ghPggNrg3AIOP0KDCN0qihN2WsKgtgg8imkaRRBoVCAQwQEiQwiSwQxkIHPuvVVn/f6oilRqr3Nv1a2zq/Y59fk8Tz236j2r9lpVtWu467zvekutNQAAAABMr7mtXgAAAAAAG2ODBwAAAGDK2eABAAAAmHI2eAAAAACmnA0eAAAAgClngwcAAABgytngAQAAAJhyNngAAAAAppwNHgAAAIApt2OrFzAEu8ruuidHbvUyAHpXSmnGa62bvJJ7KnNjnm/YMd8JjXbvbA6dWxh1YrX94aYsNj7e2n3/JMniYnfo4pixcDBHHtEJldGY7711fE/W+e73T1lo3Lf79q/5msD2VnZ2f9fu/6Lu7+QkGd11z/jCTTdm8fbbx/wGnl5PevyR9YYbuz9bJ+3v/2nf22utT+59ImaSDZ4ke3JkzipP2OplAEzWXPcPsbld7c2R0b593eAmbvrMHbG3Hb/38Z3YXaef2By76/rbO7G6q/1rbu6WOzuxMuY/v/WmmzuxxVtuaY6FJMm4jdQv/Q+d2PwdB9qXOND4T8SY78mFY7sbRzuu696ji5d+pvn+TKHGz/eM+v+PJzOodS8l2XFi93ftZ3/1uObYfZ86+h5vX/nKX9/4ugbohhsX88G3f1Hv88yf9KkTep+EmWWDB2BWNf7YH417An+LM3hGd9yx5viOq65ujx2XgdOwuMUfLzNuzP1V3vePndhozH+umhll467biPmv/oyzmcNhmNuzpxP75C9/aXPsq77p1Z3Yb5z7Lc2x5YbL7vH29dc3njSaATXJKDJ4GTZn8AAAAABMORk8AAAAcFA1i+vIFoatYIMHYDuZhbT+WfgY4G7uZ2DCys5dzfgbLn13J/blf/xlzbGveMijOrG6+Mn2hKvKR2tdOMQKgb4o0QIAAACYcjJ4AAAA4CCWDlnWpIFhG3wGTynlglLKdaWUj66K/1gp5eOllI+VUv7HivjPlVIuLaV8opTypM1fMQAAAMDmmoYMntcm+e0kr787UEp5fJJzkzyq1rqvlHLf5fjDkjwtycOT3D/JO0spZ9RaFbgDAABTaW7v3nb8mKM7sZ/9279qjv3OB/zHTuxBix9ojq1VpkqLNukM3eAzeGqt701y46rwjyZ5ea113/KY65bj5yZ5Q611X631M0kuTfLYTVssAAAAwBYY/AbPGGck+Y+llItKKf+vlPKY5fjJSa5YMe7K5VhHKeXZpZSLSykXH8i+npcLAADAtKqpWaz9v8BGTEOJVsuOJMcnOTvJY5K8sZTywPVcoNZ6fpLzk+TocrzvJAAAYJDq/v3N+Lnv+qdO7Fe+9j+1r7FwRTO+YaWsmqifaYBDm9YNniuT/GldKg79YClllOSEJFclOXXFuFOWYwAAAHDYdNFi6Ka1ROstSR6fJKWUM5LsSnJ9kguTPK2UsruUclqS05N8cMtWCQAAALAJBp/BU0r54yTnJDmhlHJlkvOSXJDkguXW6fuTPGM5m+djpZQ3JrkkyUKS5+igBQAAbETZ0f1vUyuWJKN9jfM9W2erzM2355orndiLPnlRc+xLvvY/d2ILn728OZaNqUkWZfAwcIPf4Km1Pn3MQ98zZvxLk7y0vxUBAAAADMvgN3gAAABgqzmDh6GzwQMAAGy91d2Y7jaE1tHz3XKq0V13beiSc0fubcb/18f+ohP7oYc/pTl2dOsAyrGG8PUBktjgAQAAgIOqSRZtZjFw09pFCwAAAIBlMngAAICtV8Y897yZTXHHlInV/fs3dtmdu7rX/POjm2N/8IHndMcu3Lqh+ZmM0VYvAA5BBg8AAADAlJPBAwAAAAdRU7OoixYDJ4MHAAAAYMrJ4AEAADbV3FFHdWKj225rD26dizOJbkat6449B2jtp6/Mn3jfTuxfXvSATuyBL22fLbQz1655LjZRTRYl8DBwMngAAAAAppwMHgAAADiIGl20GD4bPAAAQH8apVCjWxttv8e0KJ9IOdaar7v2/8LPHXlkM37L6+7ViZ3wxvlObMe7/769rNEmtoUHZooNHgAAADioksWM2YSEgXAGDwAAAMCUk8EDAAD0Z60lVn2VYk3Aji8+tRN76bvf1Bz7gm//gU7sqI9+pBMbraMzV+a6JV5LF9n6cq6ye/c9A/tmM8ulJhkN9xaFJDJ4AAAAAKaeDB4AAAA4BGfwMHQ2eAAAgKayc1czXg/s72GyCXTRal1j3Ps3yp6ufc5ZzaH/+Qff04m94PSvao6tCx/txsZ9bGtc1xBKscap+/atCqhjgq1igwcAAAAOokYGD8PnDB4AAACAKSeDBwAAaJZj9VKKNc4kSnvWcY3j/uaYTuyTn9zXGJm871GtUrWFNc81f+/jO7HFG25sDx5wOVbT6vKzGa7QGlUZPAybDB4AAACAKSeDBwAAAA7CGTxMAxs8AACwnbS6NGWTy7H60uhW9exPfLo59Dee3+2YdcZbPrjm666nHGzx+hvWPHbq6JoFg2GDBwAAAA6ipmTRCScMnDsUAAAAYMrJ4AEAAIBD0EWLobPBAwAA28m0teFuaZ2Jk+STv3dmJ/b73/HQ5ti9/3DRhucDGBIbPAAAAHAQumgxDZzBAwAAADDlZPAAAABbbx1lUPXs/9CM7/3XnZ3Y6B8uOewlfWFCrcApWazyIxg2dygAAADAlJPBAwAAAAdRk4zkRzBwNngAAIAtN7d7dzNejjm6E/ua89/XHPues+/biY02tizuNjffDJe5VaV1C5uwFqDJBg8AAAAcgi5aDJ0cMwAAAIApJ4MHAADYXI1yn7kT79Mc+p1vf38n9r8f8YDm2Lpw+4aWtanGdQ0baseu0WIzXFfXwA10+RtVqy5aDJ87FAAAAGDKyeABAACAQxg5g4eBs8EDAABsrka5z1Xnntoc+ifn7O/E6sK1E1/SphtqKRYwtZRoAQAAAEw5GTwAAABwEDXJovwIBs4dCgAAADDlZPAAAACbam7v3k5s33HtsQvXzMB5Ow1lR/u/YnVhYZNXskGr273P7NFC2qQzfO5QAAAAgCkngwcAAAAOoiYZyY9g4GzwAADAUK0uf0mmrr12/cpHdWK/+Iev6cR+6ZGNjzXJaOIr6tfcnj3d4Px8JzS6/fZNWM0mmLL7EWaZDR4AAAA4hMXa3oSEoZBjBgAAADDlZPAAAMBQTVH5y/wZD2rGH3/+33ZiL3nw2Z1YXbhj4mvaCuXIboewxZtu3oKVMEk1JYvyIxg4dygAAADAlJPBAwAAAIcwqvIjGDYbPAAAwPo0uns9523/tzn0N09/aDdY9096RZuv1eEsyeKNN3WDU1Rqt25zqzqELW7NMgAbPAAAAHBQNXEGD4PnDgUAAACYcoPP4CmlXJDkqUmuq7U+YtVjP5PkV5Pcp9Z6fSmlJHllkm9IckeSZ9ZaP7zZawYAgFkwt2dPM/6Ll7yvE2t1xkoyE+VYZffuTqzu27cFKxmg0faoyaopWaztsjwYimnI4HltkievDpZSTk3yxCSXrwg/Jcnpyy/PTvKqTVgfAAAAwJYa/AZPrfW9SW5sPPTrSZ6XpXLIu52b5PV1yQeSHFtKOWkTlgkAAMAMG2Wu9xfYiKm8g0op5ya5qtb6j6seOjnJFSvevnI51rrGs0spF5dSLj4Q6ZUAAADA9Br8GTyrlVL2Jvn5LJVnHbZa6/lJzk+So8vxM9y3EAAADm1u795O7Hf/5a+aY597znd3YvXAZye+pqGo+6f/HCE2ptZksU5lfgTbyNRt8CR5UJLTkvzj0pnKOSXJh0spj01yVZJTV4w9ZTkGAAAAMLOmboOn1vrPSe5799ullH9NcuZyF60Lkzy3lPKGJGclubnWevXWrBQAAIDZUDKKLloM2+A3eEopf5zknCQnlFKuTHJerfXVY4a/LUst0i/NUpv0Z23KIgEAYIhK9z+kZX6+OfTyPzitE/svD/665tjRXbNZjjV31FHN+OjWWzd5JQDrN/gNnlrr0w/x+ANWvF6TPKfvNQEAALB91DiDh+FzhwIAAABMucFn8AAAAIepdpvFzp3eLcVKkiMvPLoTG91118SXNBiN8rXRbbdtwUKYFovyIxg4dygAAADAlJPBAwAAAAdRUzKqumgxbDZ4AACgpVHC0yp5mjaf+5X2f1KP/8YPbPJKttgMfC0BVrLBAwAAAIfgDB6Gzh0KAAAAMOVk8AAAQMuMlvDc52fbH9fijH68MAk1yajKj2DY3KEAAAAAU04GDwAAABxUyWJ00WLYbPAAAGyGGe3ItO20vo7JYL+WR/3NCZ3YrV9z2RasZA0m8bmdm+/GRouHtx6AKWODBwAAAA7CGTxMA3coAAAAwJSzwQMAAACHsLh8Dk+fL4dSSnlyKeUTpZRLSykvaDz+RaWUd5dSPlJK+adSyjf08slgkJRoAQBshoGe0bLdzO3d24mN7rhj7RcY6NexfuWjmvHPv7B7Js386Pr2Rbb6nKhJzOW8HWZYKWU+ye8k+fokVyb5UCnlwlrrJSuG/WKSN9ZaX1VKeViStyV5wKYvli1hgwcAAAAOotYyhDN4Hpvk0lrrZUlSSnlDknOTrNzgqUmOXn79mCT/tqkrZEvZ4AEAAIBhOKGUcvGKt8+vtZ6//PrJSa5Y8diVSc5a9f4vSvJXpZQfS3Jkkq/ra6EMjw0eAIDD1SppKWOe4VU6sqnK7t3N+LrKsba6ZGmMsnNXJ/app+1pjj39Jy9uxpsG8LExhVZ/n8zwbbS4ORk819daz9zA+z89yWtLine truncated
+      "text/plain": [
+       "<Figure size 1152x1728 with 5 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sentence = df.iloc[175, 1]\n",
+    "print(sentence)\n",
+    "model.decoder.max_decoder_steps = 250\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Comparision with https://mycroft.ai/blog/available-voices/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " >  Run-time: 5.965982913970947\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRsDOAwBXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YZzOAwAAAAMACAAHAPz//P8CAAYACAAKAAQAAAD2/+n/7f/0/wEAAwDr/9f/yf/P/+P/6P/m/9j/wv+7/8H/yf/R/83/zf/Z/9b/zP++/7//5P8EABAABQDp/+f/+f8WADYAPAA8ADsAMAAxADMARwBjAGcAXgBUAFEATgBBADMARQBZAFcAOAANAAAA/f/8//v/+P/y/9P/rf+g/63/vP+w/5f/kv+n/6X/mP9+/3z/qP+w/73/vP+o/7r/qf+v/9n/4P8AAPz/3f/j/9z/9f8cABEADAAQAAsAGwAQAPz/EwAsAEIAYgA/ABgAHAAwAI8AtwCEAHoAYwCDALsApQDRAPMA3gDRAHYAawDMAPkAAwGaACEALwBHAFQAQADp/9T/yP+I/1f/D/8U/1T/Of/l/nn+Uv6U/sb+r/6U/nb+b/6O/pL+u/7X/tL+Cf9F/1j/Tv8s/3r/EwBTAEUANABCAJIAqwCoABQBWAFxAVcBIgFsAZQBYAFtAZYBDgI5AqoBIAHyAE4B3gEVAsMBOgHgAKkAuADLANcANAH5AEIAdP/0/n7/OQA4AKX/uf4d/k/+gf7A/uL+iv49/uj9pP36/UX+bf6E/ij+H/5L/of++v7i/p7+wf4J/7D/qP8q/0P/rP9hAEsAmf9d/8T/jgDJAHUAAQAiAGoAQQASADYALgHkAT8BEgCy/38A/wF7AvYBZQHaAEQB/wFDAlUCygG2ARQC6AHBAWQBRgE1AaQAkwDbABEBZQAK/1T+fP5V/5z/Gf93/rz9d/25/dT9Tv6q/vn+bP8N/0n+Kv5m/z8BogIFAuwAAAGWAckCSQMmA/ADAwRVA8UCzgGJAggDIgJLASUApf93/1T+JP1Q/Of6Xvod+pb5Tvna91/2/PW99aT2rPeI9073ufZA9+b4Ifoz+5r8I/6Z/1sAhQAHAm8EYAe2CcYJJwlGCeMKAQ7DD44P2A7EDfMNlw5TDo8OBg7ODF8LQQiBBp0GvAYWBl0Ccf4e/Zf8o/z9+pf4d/cI9u71V/Ut9LDzmvND9TL2avVQ9Mj0+vZ1+Kn4U/id+c/7VfzG+3/6tvr9/MX9tP18/Gv7QPxe+nn4R/gD+pD9ifur9oH0dPUG+nv7jPoy/Dz9Nf5q/e79AQS9CfwLtQriCTAO1xOqFmMWABbUGFIcnRwfGqoYohljGiMXHRMOEvYRgg9lB7IA3v9CABP/Dfgl8WTv4+1D7Rfqa+cH6ZLpMOms5xHnVOs18KnxXvJk83D4tv6n/1z/dgD9BGkLHQzHCYkIaggnCpUITAW1A6ABl/8h+2H1LfLV7y7ulOq75SjkmOQ45Ybiyd6q4BTnxu3X8KLw2/LA93f+SQdPEL8XYhvqGzYeXyUoLn41DDh4NQwzJTFeMaszrDKpL/IoXh+BGeoUORG1CxoBd/km9E/wv+t441Ldstiy1vfVI9YG2GjXMNbV1MLWDN5n5QLtXfGo8wv2efj9/p8HexBnFNYS9xC0EkcXSBiXEzAPjg5DDhQKCwDO9z/0hPC262HlNOAE3pXZztM80JHPTtQK2gXc99zq3QXie+tN9WUAwwraEPAVPBtcItQtDTjmQDlHm0Q7QRtAI0RkTNVLakRAO5kwuyrMJf0f9xo1EfUFcfuj8WfqZ+TN3irZvdM0zyzMG8vUyNjIWcrCzT/UwtZe2dfdZeWL8Hv2K/eB+twBewxnFDcVaRaNF04XBRdlFHcUmxVtEtgMjwIz+bzze+2U6o/lV+FZ3UfUuc1VyTHKCs/Z0dXUB9gB2tjdOOSf7pz6igMTC2YUYCArKdctSTPAOyRI2E48TmFPD1AIULtMDUU5RVZHQEEQNu0k/xmCFdcOnAhMALL1yOqb36rXa9O70MfOG837ymHF48IyxX/LtNI31b7XB93t4gvqe/BW9sj94QGjBpELDBHdFjEYBBYFErUPChCnEtQRJAx8AHTz4e2H7RPuUej23N/U69Adz6DNW82oz/bRedLQ0g3YveE87AP0DvgS/tQInxZvIm8pHy8ONlI+mEWsSyFRElOwUUdPaE54T99MyUU/PVY1JC8bKEkeNBPCCBEBQvqS8Xfold+C2VDUwc7wyxzLmslEx8DFQsiszlHSZ9Sm13rdQ+SG6crvh/fH/gAA2/9FBSYP3RdwFnMOugq0C8IPvA/NCLwDm/0v9znwVej/5yzo+OCm1qfOTdAd1l/U1c8Mz1HU5dsD4UjlLuvk8rz58QEoC3UVwCGqKTMvxDFNN8VDwkvDT2FMH0qiTrZOaE1NSJpCXUDNONMv3Cb7HscajhI1CED9P/SQ8CzrAeMQ2o7TgdKI0k7PSMsMy+nM1c7RzpvPeNcM3/zhduIP40/s6PUd+Xv7yvwwAwoIqAUuBUMH8AuCDOMEPQAlAEMBhv5l9NruJO+/7lDrIOPu3pfgseBy3g3chd644/XlHObV5/7uBfiA/V0BUwZqDe0VpRsNIf4n2i6SNK42GDjMOuw8xj/hQFtAFD8VObUzsy/YLC4rpCXHHQQWww/ACRwCQPvq9sP0U/AW6J3hEN7t3CrbKdl72D/YRtio1mHWq9h+3NbfdeGz4mzlkunQ7LruzvGD9YX3+fj4+iz96f6K/+b+CACqAGn/7v6v/Sj9Qv1B/Ob6C/mo92/3/vdO9x33ifeS9wH5Wvny+uP8r/06APIC+gVtCJgJAAxhDwMSARR6FtoYyRrVG10c6h3qHvweWh4MHq4eqh3mGqsY+RasFQUUShB0DbML6wipBNgApv8m/pX7k/dz9FfzJPLx74Htre2z7UzrhOm66fHqPez+6rDpYesl7ebt2u3I7ifxVPId8h3ybPP29dv31Pcx+Ij5L/oC+r75PPv9/Xn/rf7F/dL9lf7Z/74ANwItA5UCMAKEAgkEkwUZBnEGgwb4BmMHZwhPCcsJrQpZCmQKPgv/CzQMzAuPC+kL/wwBDQ0MiQtKCusJPQoKCk4KhAikBiMG/gQoBLgDqQIZAuYAl/7E/RT92vyj/OH6q/lp+U75GvnJ9wX3Bfg0+E33Vva59g/4OfiW9zT3Kfjm+Lv4wvgk+ST6Mfrm+ZL6a/sc/I/79voW/PX8ev12/Qb9CP61/nj+mP4T/0kAwQA3AL8AygGNAp4CMwKJAkoDLwQTBeMEuQSeBJwEOQVABdEFKgasBXUF3AStBEAFjwVtBZsEbgSGBOsDcgP6AmIDmwPRAl0C9wHLAZoB2wC/AL8AZwDs/2b/cP8P/37+Fv7g/fH9l/1K/c38Qvw4/EH8QfwW/NH7z/vN+3H7a/vn+4r8oPwe/ED84vxM/XX9k/3d/UL+if7I/i7/X/98/6r/zv8cAFMAhgD3ACEBygB7AMQARwFvAVsBaQG1AYQBMgEuAXsB2QHPAdEBygHBAakBmAHLAfkB7AHUAcwBygHVAcQBcgFtAb0B0AG4AU4BCwEOAcQAoQCyAJcAZAACAJn/kv+I/03/BP+x/r/+2v6x/n/+Y/51/mT+Rf56/rv+tP6h/q3+3P4d/zX/av+X/3z/nf/w/wYACAAhAEgAdQBQAD4AXQBkAGAANQAYABwALQAnAPb/s/+e/5T/d/9y/5j/rf9j/wP/C/9l/4z/d/9k/4r/tf+5/7r/0v8BACcAPwBaAHgAmgCnAJMAlQDCAPgALAEaAdkAzwDaAPQAAgH0APsA4QCuAJYAhwCNAH0ATABFADwAHgAAAOD/wv+Q/2D/Yf+G/3v/Sv8n/xL/P/9Z/1f/c/9m/3L/k/+e/9z//v8SACwAAQAOACsATwBzAFIAQQAwACcAMgAiAB8AGgD+/+H/wv/Q/+D/1P+v/4r/of+3/73/vP/A/93/3v/F/87//v8nACMADQAhAFAAXgBMAEgAXQB2AGcAVgBmAHEAcABVADQALAAxAEwARgAfAAEA8//n/8v/xv/O/9z/1v+k/4z/i/+M/5z/pP+n/6X/nf+h/6X/sv/B/9n/5f/m//z/CwAYABgAEAAmAD4ASQBMADoAOQA9AD4ASgA5ACYAJQAhACEAGwASABEAEgAPAAMA7//p//r/BwAIAPr/9f8HAP7/6//q//v/GAAVAP3/7v/0/woAGgAYAA8ACQADAAAAAgAVACMAGAAAAPD/+v8HAAYAAwD4//H/9f/s/+T/2v/Y/+H/4f/k/+H/3//m/+b/6v/y//j/AAAGAA8AFAASABMAGQAjACYAJAAqADMALAAQAP7/AgAWACAAEAAAAO7/4P/b/+H/9//+//H/1P++/87/5f/x//b/7f/q/+3/7v/y//3/CgAVABcAEQAPABoAIgAZAA8ADAAcACoAIgAPAAkAFQAbAAoA7//n//v/CAAEAPT/6//u/+j/4P/f/+f/9P/3/+v/3v/g/+n/7//s/+j/9/8GAAoAAADu//D/AAAQABcAEAALABMAFAAJAP//AgAOABEACQAEAAUA/f/n/9X/3v/7/xYAEwDw/8//z//x/xAAGgATAAcABQAIABEAIgAtADIALQAmACUALQA6AEAAMwAbABMAGgAuADIAHgAHAPP/8P/t/+b/4//m/+3/4P/F/6//rv+//83/0//Y/97/2v/U/9b/6P/8/wAA+//+/wwAJgA0ACYADwD//wsAKQA5ADcAHgAAAO3/6f/w/wEADwAFAOD/tf+q/8P/4f/l/9X/yv/L/83/0P/g//b/BwAKAAoAHwA8AEoASgBOAGQAgACRAJkAmACaAJ0AlQCLAIMAhAB/AGcARgAsACIAGQD8/8v/n/+W/5//m/+E/2D/Rv8z/yz/Ov9X/3D/a/9S/0P/U/94/5b/qP+4/8v/4f/t//j/AQAOACAALQBAAFIAXwBZAEYAPgBKAFkAUgA9ADcAPwBBACwADwACAAAA+f/n/+H/7f/y/+H/wv+x/8DLine truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAa4CAYAAAD7hW7eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XmcpWdZJ/zfXdVb6JAFghCTAAHCEkBfIWzqAC4sIgO8iryAIuKCwytuOMo6RnHXGSUyuERB4shMQFxAB0XZxpUQFgVJWEJYkpCE7EuHXqrOPX9URYpO93OfTlU/S/X3+/mcT9c59133c9XpU+ecus71XHeptQYAAACA6VoYOgAAAAAA1keCBwAAAGDiJHgAAAAAJk6CBwAAAGDiJHgAAAAAJk6CBwAAAGDiJHgAAAAAJk6CBwAAAGDiJHgAAAAAJm7L0AGMwbayve7IzqHDAOAQlW3b2pOWlnoIpGzAnLohoWSh8dlNneM4y7PuJWbd48B0lTmez+o8zyMMojReA2Y7dzTXWPqK7uf4xesWO8f33Hxt9u3ZNccL47Q84Rt21muuXT7sx/nAh/e8vdb6xMN+IDYlCZ4kO7IzjyjfNHQYAKy10P0GMkm2nHxyc87s6mvXH0sjoTFXomlL4yW3zpE0meOPqnJ04wOLPXuba8xuurl7fNeu5hrASDUSOAvbtzeXmO3evVHRcCjmSL4t3OEOneN7Hnn/5hrXvPCWzvGd5x7bOf5vb39V8xhTdM21y3nf2+9+2I+zeOInTzjsB2HTkuABAACADjXJLCpYGTc9eAAAAAAmTgUPAOM0a5/nvvTZSzdknXUb0ylL12zAKWnA5tU41dPpV8MpjVN5F1qn4Cb53A8+sHN88VHXNdc46WXdr5vlxku6j3FT+1TgaapZnud0ahiQCh4AAACAiZPgAQAAAJg4p2gBAABAh5Umy+3dLGFIKngAAAAAJk4FDwDTNU8D5VLWf5xGU9JJmef+2Ew/L8AYzPHcu/txX9M5fsXz2g2wt53XPX7SD17TXKPe3L1xQLnj0a0VmseYKtukM3YqeAAAAAAmTgUPAAAAdKipWVbhysip4AEAAACYOBU8AAAA0GAXLcZOBQ8AAADAxKngAWBzc778l3N/APRu4avu35wz+7GrO8fv9Lq7Ntc47l2f7Bxfvuqq5hpZWOwcLovd45ltzp2mapJlFTyMnAoeAAAAgIlTwQMAAAANevAwdip4AAAAACZOBQ8AAAB0qEmW9bFj5CR4AAAADqaUzuGFo45qLnHaay9qzvnUU+/SOT67/iPNNWZLS8057UWWO4eXb97VOV6XN2eTZZgCCR4AAABokLpi7PTgAQAAAJg4FTwAAADQoaZm2S5ajJwKHgAAAICJU8EDAABsSmXrts7x2cMe0FzjL/7499cdx7f9h+9ozlm6/JLO8YVtW5trlMXFzvG56k8aTaW3fMUJ3d9+9Sb9E7Mmywp4GDkVPAAAAAATt0nTqwAAALAxauyixfip4AEAAACYOBU8AAAA0KlkOd39iWBoEjwAAMDolC3df6pc9+yHNdf4p196Tef4q6//QnONp97r6zvH6549zTWSz8wxp9ts93J70kJ3k+W51O5OwktXXNn49qX1xwDcLhI8AAAA0KEmmdlFi5HTgwcAAABg4lTwAAAAQIMePIydCh4AAACAiVPBAwAAAB1qVPAwfhI8AADAhmrtgPWJX39oc40Pf9tZneNP/8Rdm2s86aSHNOe0zbNL1kjMGjttzbPLVmsNYLQkeAAAAKBhVlXwMG568AAAAABMnAoeAAAA6KAHD1OgggcAAABg4lTwAAAAG6ocdVTn+OLu9ufMT7/3YzrH697PH1JMJKmz9pzSXaVStmzt/v59m7PKpaZkWX0EI+cRCgAAADBxKngAAACgwS5ajJ0KHgAAAICJU8EDAAAAHeyixRRI8AAAwJGi0UB38Y53bK+xuNic8osf/OvO8Zec9sXmGnVpqR0Lh6bW9S+xb+9hPwZw+0jwAAAAQKeS5arDCePmEQoAAAAwcSp4AAAAoENNMlMfwch5hAIAAABMnAoeAAAgSbJ8867mnM/87MObc15y3+516lKjUS+jVbZu656wb/PuNGUXLcZOBQ8AAADAxKngAQAAgA612kWL8fMIBQAAAJg4FTwAAADQMNODh5FTwQMAAAAwcSp4AADgSFFr53B5yP2bS8y2znGY5eV5I2Ji6r7GDmiNxxhw+EjwAAAAQIeaZNkJMIycRygAAADAxKngAQAAgE62SWf8PEIBAAAAJk4FDwAAHCEWT7hz5/h1P7+7ucZ9/t9PNefM6mzumGAKapKZ+ghGziMUAAAAYOJU8AAAAEDDci1DhwCdVPAAAAAATJwKHgAAAOhQU7KsPoKRk+ABAIBNYOGr7t+cc/nPdY+f+Jyrmmss7243YmakSvsUo7Jla+d4XV7uXqAxDBw+EjwAAADQMKsqeBg3j1AAAACAiVPBAwAAAB1qogcPo+cRCgAAADBxKngA2NxaDSVr7SeOsZijweYRd5/AVDR+f6/96uObSxx79lLn+PLVHzukkNh8yrbuJssLW4/q/v4bFzcynNGoKVmuc7yGwoBU8AAAAABMnAoeAAAAaJipj2DkPEIBAAAAJk4FDwAAAHSoNVmu6iMYNwkeADY3DYO/nPsDJmvh6KM7x69+aPv3+74vv6BzfHZIEdGrjWiSP8drQF3qbsS97hiAw0aCBwAAADqVzGIXLcZNjRkAAADAxKngAQAAgA41evAwfh6hAAAAABOnggcAAAAaltVHMHISPAAAMLCFnTubc/b+2fGd4/f7jo8311jetWvumBiZnnanqvsau2gtL3d/f7UXGwxFggcAAAA61JTMql20GDc1ZgAAAAATp4IHAAAAGvTgYew8QgEAAAAmTgUPAACbUtm6rTmn7tvbQyTJ4n1O7Rz/qbe/pbnGy1/yA53jW66/5JBiggNqNEmu3T2Wk356QfeuJplV9RGMm0coAAAAwMSp4AEAAIBOJcuxixbjpoIHAAAAYOJU8AAAAEAHPXiYAgkeAACmqfRzukSrWfMVLzijucbLXviGzvFffeh/aK5x9PXnNees18LOnc05s127DnscDKhu0i7JcASQggQAAICG5dU+PIfz0lJKeWIp5eOllItKKS85wPjdSynvLqV8qJTy4VLKkw7LncEoSfAAAADAyJVSFpO8Jsm3JDk9ybNKKafvN+0VSd5Ua/2aJM9M8lv9RsmQnKIFAAAAHWotY+jB8/AkF9VaL06SUsq5SZ6a5II1c2qSY1a/PjbJ53uNkEFJ8AAAAMD4nZTkkjXXL03yiP3m/EySvyml/HCSnUm+uZ/QGAMJHgAAxmdhsT1nttw5XPft3ZBQbn7K13SO73rkLc01XnvfUxszbjiEiG6/VsPounffBh2o0UtEI18maLmfCp4TSinvX3P97Frr2Yfw/c9K8vpa638rpTwqyf8opTyo1jrb2DAZo0kneEopP57k+7NShvaRJM9LcmKSc5PcOckHkjyn1roxr+4AAABw+Fxdaz3Y1nyXJTllzfWTV29b6/uSPDFJaq3/XErZkeSEJF/Y6EAZn8FPIry9SiknJfmRJGfUWh+UZDErTaR+Jclv1Frvk+S6rDzAAQAA4HapSWYph/3ScH6S00opp5ZStmXl79+37jfnc0m+KUlKKQ9IsiPJVRt7bzBWk03wrNqS5KhSypYkd0hyeZJvTPLm1fFzkjxtoNgAAABgQ9Ral5K8MMnbk1yYld2yPlpKeWUp5Smr034iyQ+UUv41yf9K8j21OifySDHZU7RqrZeVUv5rVjKUX0zyN1k5Jev61Qd+stJ06qQDfX8p5flJnp8kO3KHwx8wAAAAE1X66sHTqdb6tiRv2++2n17z9QVJvq7vuBiH4R+ht1Mp5fisbAl3apKvzEqH8CfO+/211rNrrWfUWs/Ymu2HKUoAAACAw2+yFTxZ2e7t07XWq5KklPKnWclUHldK2bJaxXOgplMAAIxdY4esjbL0jQ9tzrnsyUud4/f9ro9sVDjr09q5KnPsLDbHGnNxRsh0NR4DZcvW7u/ft0GPoZGpSWZ1c/5sbB6TreDJyqlZjyyl3KGUUrLSSOqCJO9O8vTVOc9N8paB4gMAAADoxWQreGqt55VS3pzkg0mWknwoydlJ/neSc0spP79622uHixIAAIDNYHnS9REcCSab4EmSWuuZSc7c7+aLkzx8gHAAAAAABjHpBA8AAAAcbjVFDx5GT4IHAIBNaekdd2/O2VKubM65/7de3Tk+m6ch9MJi9/C2RuPaJLPdu7snbERjY82RaTwG6tK+dX0/cPg4iRAAAABg4lTwAAAAQMNMfQQj5xEKAAAAMHEqeAAAAKBDrcmyJsuMnAQPAABf0mgGnCRpNBVe2LGjvcSePZ3jW069R3ON09/82c7xf/mROzXXyD/+a3PKbAOaxpat3W+7mw2UYSw0UYbRkuABAACABtukM3Z68AAAAABMnAoeAAAA6FBTMqvqIxg3j1AAAACAiVPBAwDAl9TZupdYuHO7uXG98gud4598/onNNXb/5J07x7f8wweaa8yl1Xh6jvusNppKA+O3HD14GDcVPAAAAAATp4IHAAAAOtTYRYvxU8EDAAAAMHEqeAAAAKCTXbQYP49QAAAAgIlTwQMAcKQoc/SPqLU5ZbG1S9Y8x/nq+3UO3/HT7SW2vOuD7UkNZfv29pzGzzPbvXvdccBktH6/208hkzWzixYjp4IHAAAAYOJU8AAAAECHWpNlu2gxcip4AAAAACZOBQ8AAAA02EWLsZPgAQAYgbKl8bZscXHdx1i49z2ac5Yv/GR7obvdpXuNT7Q7JO96+Mmd43d57fnNNWqrIfQ8zZ6Xl5tTZnPMgSNGkeSAsZLgAQAAgA41JTM9eBg56VcAAACAiVPBAwAAAA2zqOBh3FTwAAAAAEycCh4AgBFLine truncated
+      "text/plain": [
+       "<Figure size 1152x1728 with 5 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sentence =  \"It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.\"\n",
+    "model.decoder.max_decoder_steps = 250\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap, figures=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " >  Run-time: 1.3593213558197021\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRqAPAQBXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YXwPAQAAAAEAAQACAAMABQAGAAcACAAKAAsADAANAA4ADwAQABEAEgATABMAFAAUABUAFQAVABUAFQAVABQAFAATABMAEgARABAADwAOAA0ADAALAAkACAAHAAUABAADAAIAAAD///7//f/7//r/+P/3//b/9P/z//L/8f/w/+//7v/t/+z/6//q/+r/6f/p/+j/6P/o/+j/6P/o/+j/6f/p/+n/6v/q/+v/7P/s/+3/7v/u/+//8P/x//L/8//0//X/9//4//r//P/9/wAAAAADAAUABwAKAA0ADwASABUAGAAaAB0AIAAjACUAKAAqACwALgAwADEAMwA0ADUANQA2ADYANgA2ADUANAAzADIAMAAuACwAKgAnACQAIQAdABoAFgASAA4ACQAEAAAA/P/3//L/7P/n/+L/3f/Y/9L/zv/J/8P/v/+6/7b/sv+u/6r/p/+k/6H/n/+e/53/nP+c/5z/nf+e/6D/ov+l/6n/rP+x/7X/uv/A/8X/y//R/9j/3v/l/+z/8//6/wEACAAPABgAHwAnAC8AOABAAEgAUABYAGAAZwBuAHUAfACCAIgAjgCTAJcAmgCeAKAAoQCiAKEAoACeAJsAmACUAI8AiQCCAHsAcwBqAGEAVwBNAEMAOAAtACIAFwAMAAAA9v/r/+D/1f/L/8D/t/+t/6T/mv+R/4n/gP94/3D/aP9g/1j/Uf9K/0P/O/81/y7/J/8h/xv/Fv8R/w7/C/8J/wj/CP8J/wv/D/8U/xv/JP8u/zn/R/9W/2b/eP+L/6D/t//O/+f/AAAbADcAUgBvAIwAqADFAOEA/AAXATEBSQFgAXYBiQGbAaoBtwHCAckBzgHRAdABzQHHAb8BswGlAZUBggFtAVYBPQEiAQcB6gDMAK0AjQBtAE0ALQAMAO3/zv+v/5D/cv9V/zj/Hf8D/+n+0P64/qH+i/52/mL+Tv48/iz+G/4N/v/98/3p/eH92v3V/dL90v3U/dn94P3q/fj9CP4b/jH+Sv5n/ob+qP7N/vX+H/9L/3n/qf/b/w0AQAB1AKkA3gARAUMBdAGjAdEB/AElAksCbAKLAqYCvgLRAuEC7ALyAvUC8wLtAuMC1QLEAq8ClgJ5AloCOAIUAu4BxQGaAW4BQgEVAecAuACKAFwALwADANj/rv+G/1//Of8V//P+0/61/pn+f/5l/k3+Nv4h/g3++v3o/df9xv22/af9mP2K/X39cv1n/Vz9U/1N/Un9R/1I/Uz9Uv1c/Wn9ev2Q/an9xv3n/Qz+NP5i/pT+yv4C/z3/e/+7//7/QQCFAMoADwFTAZYB1gEUAlACiQK+Au8CGQM+A18DegORA6EDqgOsA6oDoQOSA38DZgNJAycD/wLUAqMCcgJAAgsC1QGeAWcBMQH7AMgAlgBnADkADwDo/8P/of+D/2f/Tf82/x//C//6/uj+1v7F/rX+o/6R/n3+Z/5R/jv+I/4K/u790v24/Zv9f/1j/Un9Mv0d/Qn9+Pzs/OP83vzf/OT87vz+/BL9K/1J/Wz9lv3F/fb9K/5j/p/+3/4g/2T/qv/v/zUAegDAAAYBSwGPAdABDQJHAn8CtQLnAhUDPgNhA4ADmgOvA78DyQPQA88DyAO6A6kDkwN4A1kDNAMKA9wCqQJ2Aj8CCALTAZkBXgEhAeMAqABwADwACgDZ/6n/ev9N/yT/AP/e/sD+pf6J/m7+Vv5E/jT+I/4T/gT+9P3j/db9yf2//bX9qP2a/Yn9ff12/W79Z/1f/Vf9Uf1O/U79VP1e/Wr9dv2D/ZT9qf3F/ef9DP4z/ln+gv6v/uD+FP9M/4L/uP/v/yQAXACUAM4ACgFBAXYBqAHbARACQgJxApwCxgLsAg4DLwNNA2gDfwOQA58DpwOqA6cDoAOXA4cDcQNSAy0DCgPjArYCggJNAhMC1QGWAVMBEAHMAI8AVAAXANn/mf9d/yT/8P6+/pT+cP5N/if+BP7p/dX9xv20/aj9nf2S/Yn9f/17/Xr9eP11/W79Zv1h/Vz9WP1U/Uv9Rf0+/Tv9N/06/UD9P/1G/U79Yf12/Yb9o/3K/fj9JP5N/nr+sf75/kP/iP/L/wwAVQCkAPYAQwGTAdwBJwJvAqoC8QIyA28DqgPUA/cDHAQ+BFgEdwSFBIsEjgR2BGUETQQpBBoE9wO7A34DMAP8AsUCewI2AtMBeQEpAdIAiABHAP7/tf9p/xj/2/6b/mH+Ov4S/uz9x/2g/Yn9eP1p/WL9WP1Z/Vj9Vf1N/Ur9WP1j/XL9d/14/Xf9Zf1g/V39X/1o/Vz9U/1A/S79Kv0j/Rn9A/34/Pz8Bv0k/TT9P/1M/WD9k/3M/QH+Of52/sD+HP9w/8X/JACFAO8ARAGcAQICbwLwAlgDqgP5A0YEngTnBCQFZwWoBd4F9AX1BfQF/QUHBvkF3AWhBVgFJQXWBH0EKwTLA3wD/AJjAucBagENAakAKgCw/yn/r/5F/uT9o/16/Tf97/yt/G38Yfxa/GP8dPxq/Hv8hvyb/Mf84vz//Bf9Nf1Y/XD9dP11/Xf9c/1//XD9VP0+/RH95Pyv/ID8dPxn/E38HPzY+6v7sPvO++j7+fsF/D78ePyo/AX9Zv3q/W7+2v5N/9H/ggA7AeIBaAL5AqMDQATgBGwF9AWDBu0GSgeUB+sHWAiOCI4IZwhOCE4IWghBCNUHTAfUBmAG7wVbBb8EMgSPA+cCDgI6AagAIwCX/9P+AP6W/S39y/xX/MH7mvtt+1L7QPvt+gX7MvtE+3j7h/ur++77G/xq/M/8Dv03/TL9L/1h/aP96/3s/ar9bP1M/TL9DP3S/JX8UPz9+7j7R/v/+uH6sfqc+ln6MvpE+kX6mfrT+gn7gPvL+2H8+/yV/XH+MP8pABMBsgGdAoUDkgSrBVQGHAfkB6oIiQkZCoEK4QokC4kLsguyC4wLUQsjC8AKUQqKCe4ILQhlB8sGqQWnBKkDngLIAagAq//6/iP+Wf1e/Gf7yvp7+lP69vmC+T/5Rflb+XT5cPmI+S/6xvoz+3X7ZPvk+4n8EP26/fD9H/49/jb+Rf5V/mv+a/51/gH+S/3J/Dr8Ffzw+2X70/ru+Tj58fjW+Nr4fvgQ+Mz34PdO+KX4D/l4+dX5mPpz+3H8o/2Q/oH/qwAPAn8D1wTiBd4GAAhJCcEK1AttDAsNyA13DgQPHA8gDx0PLg8ED18Otg2wDAIMOgs/CjkJqgdhBv8EuwOEAhIBx/9//ob9WfwH+/b5JPnO+GP4wfcy97v23vZW95H3uvfQ9zX4AvmS+Sn66Pqi+038rPwT/aL9bv7R/tX+2P6L/t7+F/94/gT+QP2+/MT8Ffwm+0z6Pvm6+G74wfeF9yL3Zvb29aP1nPWN9in3gPcj+CL40vj3+Tj7W/0O/wAA/gDMAaEDJAYBCFsJTgpGC9cMeA56D2UQ0hCAEUMSgxJAEuMRlBFKETIR+w/aDlgNuQssC+4JaQh4BigEsgJsARIAzv4D/Vr7KPoN+SX4Nfe99kr25fV+9QT1c/Xx9ZP2EvdE99D3l/hi+YX6YvsA/ND8Y/1P/t3+8/7i/g//hv/7/+b/sf6u/Rz99fzz/Nv7gfp7+Xb41ffW9pT1X/Vi9fH0sPSF88/yiPM49KP1ePaj9qP3SviV+Yz7XP3Q/6sBDANpBOoFQgijCnYM7A2RDxcRZBI2E38TQBQvFTsWVxZLFXoUrxM9E3QSDBGwDzUOiAy7CpsIGgaiBD8DLwGk/xz9SvuH+pj4ivdf9h31TPW39O7zxfOg8yj06PQ39dj1svaS93z4U/kA+g37cPx+/VL+iP7E/t/+bv/Q/77/3P9F/+T+vP0h/Kf7SPtI+0v60vfu9Z70/fRb9ST0xvKf8ZfxGPLd8dfxpfLv85r1FPbm9Wn3zPnz/GT/8f+2AewDrgaiCdgKuwz/DjURQBMuFAoVXhaXF0AYjhg+GEIYTxhYF/QVShRkE64SpBA7Dr0L5gmVCOMFCQOHABX/Zv4t/DD51/YI9gD2CvWb853yZvJx8zbzpPL18kvzrPXk9uX21vcv+Hz5VPsq/Dz9Qv7P/hD/xP6g/kv/kABzAHT/x/3M+7T7xPub+6f6K/jd9Tj0+PNs9OjzifKW8Gfv/e++8BbxB/HK8IHyW/Qt9Vr27PZP+QL9iP+QAeoCeQSUB5EKrAyED9ARshNEFXQVghYyGTEboBvQGikZfBk2Gn4ZRxjVFbsT4hKPEdAOIQwRCT8HvwUeA6kA1f2S+w76nPiD9v/0pvNY8zLzePKI8c/wcvGN8hT0tvSZ9Br1Yvb197f5bPpO+0T8X/0g/m3+m/6w/sH/k/9r/6v+a/2e/Db8B/tq+l75Ovc79pH0jvPj8jbyVvF+8PDuiO57753viPDV8H7wMPIp9Cn2Kfiy+NP6I/5xARIEaAW3BmcKYg6PEF8SghPAFbsYAxovGswarxtuHQseQBxyGk4ZKxkLGYAXRBQIEUYPGw0HC9QISgVTA7oA6v1n/Fn5a/db9rb0E/R9817x3fDt8HvwGvJL8ovyWfQH9MX0WPbJ9rz5BPvb+uX7+/sr/dz+gv6j/pb+RP5m/0T+F/x5+8365Plv+kz4D/a39BDyj/Li8j/xbPAf7grtUu957tvuge9C763xLfOV89T0RfcC+m39Zv/aANkDqgYrCcwMDw/YEc4UyRV6F88YDBtaHXEeRR6sHXMd1BzOHMMbvRqYGUUX8BOHELYOig1kDF8IjQRkAQr/qf0U+wb4zvZ79gL1iPOy7y7vp/As8kbzg/Fj8PHwWvN59Vn22fas9+b4dfrj+lr71fzC/Z7+Yf65/Xj+/f0G/ZP8bfup+0X7jPgY9tz0sfTu9NPz8/AF7//tze327oju3u0z7gvule7K71XxUvPJ9X7Line truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAa4CAYAAAD7hW7eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xu4bXdZH/rvuzckkQByiaY5SSgX4yVGKz5bLtWDUC4G6jH2aUsJVRHRiIdYEI4VL4UYpaDWWy3a5igS2yoirZh6ouFqwapAUBQItxAuSUiAAEJQSLLXes8fa6YuNnvPmWTtNX9jrP35PM981pxjjL3ed4exVzZvvuP3q+4OAAAAAPO1b3QDAAAAAOyMAQ8AAADAzBnwAAAAAMycAQ8AAADAzBnwAAAAAMycAQ8AAADAzBnwAAAAAMycAQ8AAADAzBnwAAAAAMzcnUY3MAXH1fF9Qk4c3QYAQFI1rnb3uNocGwbd3hv3HPd3/S/44s8MqXvz5v4hdT97/ady8yc/M/AH2e74pkec2B/7+Mau13nzX910WXefveuF2JMMeJKckBPz4Hrk6DYA5mffmL88Jkk2d/8vWTBCHX/8sNp9003DanNsqDuN+b8ff/24A0PqJslXfP/bh9T90N984ZC6b3jqbw6pu9s+9vGNvPGy++x6nf2nvOekXS/CnmXAAwAAAEt0ks1sjm4DlrIGDwAAAMDMSfAAAADAUp2NluBh2iR4AAAAAGbOgAcAAABg5jyiBQAAAEtsLbLco9uApSR4AAAAAGZOggcAAABWsE06UyfBAwAAADBzEjwAAACwRKez0dbgYdokeAAAAABmToIHAAAAVrCLFlMnwQMAAAAwcxI8AAAAsEQn2ZDgYeIkeAAAAABmToIHAAAAVrAGD1MnwQMAAAAwcxI8AAAAsEQn2WgJHqZNggcAAABg5iR4ALjjNjdGdwB7Tt900+gWWJeqIWU/+cQHD6mbJBc97xeG1H3We08ZUjdJrn/q6UPq7v/MzUPq1gf3boZgc3QDsMLe/dMHAAAAcIyQ4AEAAIAlOp0Nu2gxcRI8AAAAADMnwQMAAADLdLIhwMPESfAAAAAAzJwEDwAAACzRsYsW0yfBAwAAADBzEjwAAACwVGUjNboJWEqCBwAAAGDmJHgAAABgiU6yaRctJk6CBwAAAGDmJHgAAABgBWvwMHUSPAAAAAAzJ8EDAAAAS3QkeJg+CR4AAACAmZPgAQAAgBU2W4KHaZPgAQAAAJg5CR4AAABYwho8zIEEDwAAAMDMSfAAAADAEp3KhnwEE+cOBQAAAJg5CR4AAIaqO437K+nGQ84aVvsx//H1Q+r+4fUfGlI3SX74IecMqbvv+quH1E2SzWGVx+i+aXQLu8YuWkydBA8AAADAzEnwAAAAwBJ20WIOJHgAAAAAZk6CBwAAAJaqbLR8BNPmDgUAAACYOQkeAAAAWKKTbMpHMHHuUAAAAICZk+ABAACAFeyixdRJ8AAAAADMnAQPAAAALNFtFy2mzx0KAAAAMHMSPAAAALDCpjV4mDgJHgAAAICZM+ABAAAAmDmPaAEAAMASnWRDPoKJc4cCAAAAzJwEDwAAACxlm3Smzx0KAAAAMHMSPAAAALBEJ9mUj2Di3KEAAAAAMyfBAwAwJVWjO1i/B37FsNJf9FMfGFb7FU/9P4fUvfOb3jGkbpIcvOmmYbVhpzb6GPz5zKxI8AAAAADMnAQPAAAALNGpbMhHMHHuUAAAAICZk+ABAACAFTZbPoJpc4cCAAAAzJwEDwAAACzRiTV4mDx3KAAAAMDMSfAAAADAEp3KRtfoNmCp2SZ4qur0qnptVV1RVW+vqqcvjt+rql5ZVe9ZfL3n6F4BAAAAdtNsBzxJDiZ5VnefmeQhSZ5WVWcmeXaSV3f3GUlevfgMAAAAd9hm9u36C3ZitndQd1/X3X++eH9jknckOTXJOUkuXlx2cZJvHdMhAAAAwHrsiTV4quq+SR6Y5A1JTu7u6xanrk9y8hF+zXlJzkuSE3KX3W8SAACAWepONnq2+QiOEbO/Q6vqrkn+W5JndPentp/r7s7Wjnafp7sv6u4D3X3gzjl+DZ0CAAAA7I5ZJ3iq6s7ZGu781+7+74vDH66qU7r7uqo6JclHxnUIAADA/FU2Yxctpm22CZ6qqiS/luQd3f1z205dkuRJi/dPSvJ76+4NAAAAYJ3mnOD5+iTfnuStVfWWxbEfSfKCJC+tqqck+UCSxw/qDwAAgD2gYw0epm+2A57u/uPkiBm5R66zFwAAAICRZjvgAQAAgHXZmO8KJxwj3KEAAAAAMyfBAwAAAEt0KpttFy2mTYIHAAAAYOYkeAAADqfG/JfaOu64IXWT5D0veOCQuk955GuH1E2SP37kfYbV3vfRt6y+aBf0kKowf9bgYercoQAAAAAzJ8EDAAAAS3SSzZaPYNrcoQAAAAAzJ8EDAAAAS1U2Yhctpk2CBwAAAGDmJHgAAABgCWvwMAfuUAAAAICZM+ABAACAFTYW6/Ds5muVqjq7qt5VVVdW1bMPc/4+VfXaqvqLqvqrqnrcrvzDYJIMeAAAAGDiqmp/khcmeWySM5OcW1VnHnLZjyV5aXc/MMkTkvzyertkJGvwAAAAwBLdNYU1eB6U5MruvipJquolSc5JcsW2azrJ3RfvvzDJh9baIUMZ8AAAAMA0nFRVl2/7fFF3X7R4f2qSq7eduybJgw/59RckeUVVfX+SE5M8arcaZXoMeAAAAGCFjfUkeG7o7gM7+PXnJnlxd/9sVT00yX+uqrO6e/Mo9ceEDc+YAQAAACtdm+T0bZ9PWxzb7ilJXpok3f2nSU5IctJaumM4Ax4AAABYopNspnb9tcKbkpxRVferquOytYjyJYdc88Ekj0ySqvqKbA14Pnp0/2kwVQY8AAAAMHHdfTDJ+UkuS/KObO2W9faqurCqvmVx2bOSfE9V/WWS30rynd3dYzpm3azBAwAAAEvVutbgWaq7L01y6SHHnrPt/RVJvn7dfTEN4+9QAAAAAHZEggcAAACW6CSbvXKNHBhKggcAAABg5iR4AAAAYIUN+Qgmzh0KAAAAMHMSPADAdNW49Q72/YOvGFL3n/zWHw2pmyQ/+7KvHVL3dV93jyF1t3xqYG1gLjplDR4mT4IHAAAAYOYMeAAAAABmziNaAAAAsMKmfAQT5w4FAAAAmDkJHgAAAFiiO9mwyDITJ8EDAAAAMHMSPAAAALCCbdKZOgkeAAAAgJmT4AEAAIAlOpXNlo9g2tyhAAAAADMnwQMAAAArbMQaPEybBA8AAADAzEnwAAAAwBIdu2gxfRI8AAAAADMnwQMAAABL2UWL6XOHAgAAAMycBA8AAACssGkXLSZOggcAAABg5iR4AAAAYInuZMMuWkycBA8AAADAzEnwAAAAwAp20WLqDHgAgMm608lfPKz21//Gm4fUfflDzxhSN0nu+6k3DqnbmxtD6gLAXmLAAwAAAEt0KpvW4GHiZMwAAAAAZk6CBwAAAFbYjAQP0ybBAwAAADBzEjwAAACwRCfW4GHyJHgAAAAAZk6CBwAAAFbYbPkIps0dCgAAADBzEjwAAACwTJc1eJg8CR4AAACAmZPgAQAAgCU6yWYkeJg2CR4AAACAmZPgAQAAgBWswcPUSfAAAAAAzJwEDwAAACzRkeBh+iR4AAAAAGZOggcAAABWkOBh6iR4AAAAAGZOggcAAACW6JQED5MnwQMAAAAwcxI8AAAAsMJmJHiYNgMeAGClOv74IXUfdNnVQ+omyesfcu8hdTc/86khdZOk9u8fUrc3h5SF9agxQ4FRf55zcExZwIAHAAAAlmu7aDF91uABAAAAmDkJHgAAAFiiI8HD9EnwAAAAAMycBA8AAACsIMHD1EnwAAAAAMycBA8AAAAs0SkJHiZPggcAAABg5gx4AAAAAGbOI1oAAACwQntEi4mT4AEAAACYOQkeAAAAWGEzEjxMmwQPAAAAwMxJ8AAAAMAS3bFNOpMnwQMAAAAwcxI8AAAAsIJdtJg6CR4AAACAmZPgAQAAgKXKGjxMngQPAAAAwMxJ8AAAAMAK1uBh6gx4AGAm9p1wwrDaV174wCF19537sSF1k2Tzs+8dUnf/3e42pG6SbH7ms8Nqw57VPabswYND6mbMbxeIAQ8AAAAs1Yk1eJg8a/AAAAAAzJwEDwAAACzTw562g9tMggcAAABg5iR4AAAAYIXNWIOHaZPgAQAAAJg5CR4AAABYopO0XbSYOAkeAAAAgJmb7YCnql5UVR+pqrdtO3ZBVV1bVW9ZvB43skcAAAD2gspm7/4LdmK2A54kL05y9mGO/3x3f83idemaewIAAABYu2Fr8FTVlyR5UJJTk1yb5A3d/d7b+uu7+3VVdd/d6Q4AAAD+TvfoDmC5tQ94quqEJL+c5NuT7N92aqOqLk7ytO6+aQclzq+q70hyeZJndfcnjtDHeUnOS5ITcpcdlAMAAAAYa8QjWv8uyb9M8twkX5LkbouvF2Rr6PMzO/jev5LkAUm+Jsl1SX72SBd290XdfaC7D9w5x++gJAAAAHtdd+36C3ZixCNaT0jy4939b7cduyrJ86oqSX4gyb+6I9+4uz986/uq+n+T/P4O+gQAAACYhREDnuOTvPEI596Q5Lg7+o2r6pTuvm7x8Z8keduy6wEAAGCV7kjYMHkjBjyvSvKYxddDPSbJa27LN6mq30ry8CQnVdU12Xrk6+FV9TVJOsn7k3zvUegXAAAAYNJGDHh+Lsl/rqoTk/xOkg8nOTnJ45M8Lsm3VdX9b724u6863Dfp7nMPc/jXjn67AAAAHOs2JXiYuBEDnv+5+Pp9SZ667Xgdcv5W+wMAAADAEY0Y8Dx5QE0AAAC4w7pHdwDLrX3A090Xr7smAAAAwF42IsEDAAAAs2IXLaZuyICnqs5O8s+TnJ7khENOd3d/4/q7AoDboMb95e6dv3zWsNr7/npLine truncated
+      "text/plain": [
+       "<Figure size 1152x1728 with 5 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sentence = \"Be a voice,not an echo.\"  # 'echo' is not in training set. \n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " >  Run-time: 3.3218443393707275\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRkwXAgBXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YSgXAgAuAB8ACwAEAAgAPwAZAPL/CwAhAC4ABADh/97/CwAhAO//9v///xMAEwABAPv/8P9BACUACQArAPr/SgB2ADMAHAACACUAIwD5/xQABgAGAAsAzP/B/+L/4//Y/8j/q//R/+r/0//F/77/CwAAAM//7f/w/xcADwDo/+v/+/8YABQACgABAAcAGwD4/wAALQAjADIAKQAeADwAPAA2ACkAJAA7AEkATQBkAGIAWAA7AD8AiQB0AFwAYQBKAGkAaQBDADkAMQArAO//x/+//77/wP+K/3L/aP9L/0D/Jv/6/iD/IP8w/0n/Ev82/zL/Rv9z/17/gf+4/9L/7v8bAPj/6P8LADAAZQBZAI8AfAB0AK8AaQCIAI8AhgCfAHIAowDJALwAswBwAEwAgAC+ALIAqwC7AJ0AjACAAIkAogCWAJYAcABeAJMAiACKAIUAMgAyADkAJwAhABMA9/+C/6T/iv9C/6//TP8R/w//y/7w/sX+4/7Z/oH+kv5p/o7+pv6s/t7+bv6J/n/+XP4c/yX/Mv89/yP/MP8m/5j/5v8DABMAHgA1AGAA0QDlANwAFgEGATwBvgH9AQUCzAHPAdcBoQG/AQUCEwLjAZsBFAHZABEBCgEAAckApgC9ALAAvQDRALQAmgCZAIoApwC7ANQA2gCgAIAARgApAFcAgABXAOv/b//+/u/+BP/O/rP+h/4V/qL9GP3b/Ov84fzC/Kz8Y/xA/G38efy2/ND89Pxn/bT9Kf7M/l7/uP8AAIUA4gA2AbUB4QHEAdwBNgJaAjYCDALIASIB1gC8AGIAdgBhABoAp/8K/1f/SP87/8H/rP88AFcAfAAPAeoA2QGJAtgCLwNbAy0EhASZBLwE3gR0BbsFvwWvBZcFrQVMBRIFxASLBC4EYAMfA2YC6wEWAcT/L//j/ST9gvxR+w773Pmr+Pj37vYW95j2vvW39ez0ufTB9HX0gvXL9cD1LvZI9kT3Dvih+Fv5TvqV+xj9Pf4k/3AABAEVAj8DhgTLBUMGBgd5B0oILAglCJcIlwgsCXMIwQhLCRAJDwm6B+YHfwiVCAoJpAg/CeQJ3AmxCV4J0gk6ClQKFApPCgoLXwuLCq8IZAf5BuEGbQUXAzUBsP9T/mn7Yvgu9qPzQvHg7UfrBupy6MnmPOSf4qvi/OIJ44DiFOMU5XznfOmt6wPvo/Iu9vX4LvwkANgDZAfvCeMMWBD6EsgU3RSDFaQW3RbdFgoVwRMPEiUP4Qw8CicJ2wdMBc8CnP+s/Xb8k/sH/GX8tP1J/+cANAOTBLEF8QYFCZwMJBBPEyMWAhgDGQIZmxh6GOEXXxazFOsSfRGlDx8MlQe/Aez7x/aT8avsxOdx4+Xf8Nuz17DTrNCkzozMGstgy7fN5dDZ0/DW7Nrn373kxum976v2gP4zBc4KpRBuFi8ctB+3IVYkoyZqKPknTyapJMMhCx6tGCkTuA4BCpgFZQDe+yH5F/aD89fwaO/q7/Dwb/PM9gL7yv9kBPUIJw2+Eb0Wvhs1ILAj8SbtKMMpAip5KS8p4ycPJdQg8RqTFfsQzAuZBWX+l/dw8R/qH+K+2nTUuc8Gy4bGPcO0wAm/X70PvP6877+aw/7GXsoL0ALYCeD+5vPsavNe+9wDtgxlFZ4cXyILJtsoKywTL4oxJjKsMOQt+SkbJjciwR2oF0sQIQnNA2UASfx49+/ypO837qns3OsS7cDvLvRR+Lf8PgJWCPQOEhR6GDgd4yKRKa4ukDEXMkox+zBnMJQvzy36KdMkmR3iFeoOyAilA2/80PL/5yvf29mO1M/NXsX3vW25GrbWtIy0orR5tLey7bLYtnu+pcc9zgTTDdjp323q9fSI/mEG2g2dFBsbsiIOKoMwyjOdMxczxzKXM2c0azJkLgAowyDPGoEVWBFBDdcHBAKs/Pv32PQC8zXyLvO28+D0Z/dM+hv/FQRPCfUOlRNMGR8f/iMXKMQqmS2+L6EwhTCUL9stGit8J6IiTR1iFx8RigqMAm/6svLu6ofjL9xs1dnOH8eFv6O5m7XVs7CyQbFtsK6vg7COsk21Y7uIw4bL5dGv1vfd4ef18RP7YgIECo0S8Ro2Ilwn0SsVME8zczSCNNI1Ojf1Na4wmSpzJzglcSEkG+cUpRAEDFoHZQJp/2T+3Pwo/Ov68fu4/uz/zAGNA2wI2A92FBYYKhrMHFgh8yO3Jl8pRiqJKtknHyUuJLghch7OGJsSyw3qBxUC//q+8hjrvuMe3m3YLdKpzDXGEcDKuiu3BbbjtEG05bPIsz21hLj2vGTB1sXWyp3Rl9kz4qDqbfHs93r/0wfvD1sWdRxXIoInpyvTLYUwwTK/Mwsz4zB1MLQvOSxBJwYjCSHdHg4aXBQGEFYNEQtqCFoGyQVWBYMERgNPA2wFMwitCvML1A3KEAYUuBaxGAUaeBsXHcAdzx37HIUcEhwyGc8UKBGUDnELWAZ8/zb6u/XQ8MnrwuQJ37PZoNSh0AfMRMmDxjfDvcCUvu++8L9QwCLCpMPfxsHKTM7J01vYw93243rpVfBp9mH8EgOfCDMOPxPGF4Ic6yBbJBYnMyg/KCsptimQKlkq2Ci8JqcjRCGFHxIeWBvNFzMViBNbEgUQuw38DD0MeAtACgUKFws8C60LTgvrCwwOwA+kEVYRoRAJEbQQ9hBdEVYRFxGoDm0LlQgnBrwEYgKW/uT64PYg8qXtYOpp6EflCeFr3UXagdfu1ODS+NFz0dHQStC0zzDQONIj1EHWLtiG2s7e/+EV5XHoD+x18Wz26fpn/toAnQQvCacNghFiFJwWnxjbGj8csx3sHkAgHyHmH/4eZh6ZHlUeYhxhGsgYFhjWFhkVRRMDEmURwA9DDlMNQQ36DIELyAqgCowKqApACqAJWAl+CcwJUwkACKIHmweRBocFegQzA3oBBACp/hH93vov+Bn35vQ18n/wsu1A7F/rnemW57DkNOO641XjdeLy4YThAeJv4ofizuN/5VDnD+mQ6brqNu1V8C/0R/aH9kb4k/pi/XUAuAKJBesGMAcYCcwKOgx9DlkQqBAkEDsQlBGfEw8TMhPEE9USixMYE7sSqRI6EYgRZxEdEDAQ1g+QDuYMhQvMCyIMxwoeCTgI2Ae9B/0GigUABT8FMAU5BEsCPwGZAUACswHw/yb+C/0V/X78NPvK+vz5pPjs9g/14/QE9Rr0JvMb8hDx0/C28EbwaPDV7zjv9u808BjxD/Jp8d3xSPJR86r1pfV59u/3E/iJ+WT6sfu0/TP+rP7K/74AjAHJAj0DFAQ8BQEGgQZ9BUEG+wfbBygIYQgsCFwIAwnnCY4J5wgICbEJmAl4CdMJbQnGCcAJqQjhB0kIlwkVCVwH4gbcBqUGgQYHBg8FjwSlBCcElgK/AfkBegEFAVkA+f8H/8D9kf2g/BX89PwR/Rr7XflX+Yz6hPrI+Dz56Plx+N33L/jJ+Eb5R/j096P4tPhC+WT5y/gc+eL5+vor+5b6p/r9+9X9Bv0V/Hz9wf5Y/7r+r/5UAFIAAADkAFcBSAE2AekBXQIWAhgCkAIeAyEC3AGcAzYDAwPtAmAC0AOiAnID5gRkAqsCoQMoBIUEvwOUA94CAwPhBFkF1QNFAhID/ANNA4ADiAMyA88CqwJIAt0BFwLzAZUBEQFRAV4BaQBY/3//uQDs/yv++v19/rr+4P2B/aP90fyf/CT9xPy/+xP9gP3J+477eftp/MH9Rf2G/G/8xvv5+4P9Ev60/Yf9LP2c/cn9zf1P/zT/xP7q/fX+9QCOAIoAEv+x/ngBRAJaAUIBcQBIATABvwCbAgMCoAHXAPn/QAJJAkwAQQE0AZgALgLrAIUAtAExAQ8B2/8rAaoC5AFvAVwA5QBCAdcBmQJOAgwCFAFAAJ8A1AKJA/kB1f/J//UBOAH2/1MAtwHlANv9of6nAEcA3f9I/yD+Yf7k/pMArQD6/eb9F/9+/2H/M/6E/34AvvwF/r//T/4VAHb9OP3O/1T9cP7G/p3+bAAW/Kz8XQCI/0AAI/4e/kP/7/01AMoBIwAk/5b/Jv+k/wgBqAHLABT/xP/bAQUB+f96AQkBP//y/74BWQLcAEr/GwAOAQwDeAIU/7r+t/9DAtsCpf8BACIAVf9JARICugFn/c79swMQA8z+rf5KAYIAvQCJAWUByABz/9QBzgD6/UABqAIPAvb/uP0x/04B1gCZ/5QAcP/N/h3/Uf8q/0z/jP8zAFL+6/xYAeH+PP3g/kIAIQCj/ZL9n/5rAZn/m/42//f+D/+C/qD/8wAuAAYAqP9C/KH/BgOSAKf+p/0+ALsCY//k/S8BHwFd/8v+Zv9WAUoCPQAz/7j/Ef8+AscB/P8jAAT/cwIgAoUA+v4nAOwCKgGRALn/VwGLAQ7/nf72AOwCLwDJ/jv/p/7BACwBBQEhADz+jP57AH4C3f/r/8n+4v+GA3sAOv6W/k4BhAIOAfz+Sf+NAJb/QAIPAN38/f/LAS4Bq/65/vT9CADBAOr+hP+m/qr/mP0T/koDd/9f/Ab/Ov/X/9j+DgHY/x77AwD0AvYA8vxZ/fwAQABeApX/Lf5B//MAcgM0/M/9BQQMAwv/Av1HALQAwgJaAvD7J/1nBQsFW/5J/XwAWwEKAYUD6AEN/gv+6QKLA8z+Q/9CAoICpf7o+44BIASd/pn92/7PAKgBb/9S+y38NwNKAv3/Qvzc+eMC2gKR/QIAlgCzAET+ZPuw/9QFzgNE/Vv9Zv4cAhcCKgBoAvT8If4RBOoBfP0g/6kAUf+VABcBwQFs/In7QAKwA5QBAvvR+/wBygEoAPn+a/+T/Qr/TwLxAAD/of68/kMAwwKhAbj+TfxWAjEFRwGS/439jQBSAY4FZQRG/Vn/t/6BA/QLine truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAa4CAYAAAD7hW7eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XmYpndZJ/rv3Z0NEiKEKOQkIAECgqCimQDiYRM0LgM4ag6LiJhjXMBRcUaCjoAogs5RXAY9ZlyIO4gCEdGILIICISAKIWwhgATDFgIJGJJ01T1/VAWLpruq+32q3+d5qj+f63qvfre7fncnbyfVd32f36+6OwAAAADM166xGwAAAABgGAMeAAAAgJkz4AEAAACYOQMeAAAAgJkz4AEAAACYOQMeAAAAgJkz4AEAAACYOQMeAAAAgJkz4AEAAACYuSPGbmAKjqqj+5gcO3YbwETV7sVn4b2yuo2dAACzUgNKd+0etPT1X3LMwrV9VC9cu+eqq7Ny7WcG/M6n6RsfdGxf9YmVQ77Om996/YXdfeYhX4gdyYAnyTE5Nveurx+7DWCidh93/MK1K5/+zLDFVw/9NxIAwKFRRyz+161dxw37AfT7z/7yhWuv+9IbF6798M/92sK1U3bVJ1byxgtvf8jX2X3Se0485IuwYxnwAAAAwCY6yWoks5k2e/AAAAAAzJwEDwAAAGyqs9ISPEybBA8AAADAzBnwAAAAAMycS7QAAABgE2ubLC9+fDwsgwQPAAAAwMxJ8ABsYeWaa8ZuAQBYVNXCpbuOO27Q0lc/7MsXrn3wf3vdoLUvu3DxtMmd/2DPwrVXX7VzUy6OSWfqJHgAAAAAZk6CBwAAADbR6az0zk0nsTNI8AAAAADMnAQPAAAAbMEpWkydBA8AAADAzEnwAAAAwCY6yYoEDxMnwQMAAAAwcxI8AAAAsAV78DB1BjwAAMB0VQ0qP+KUkxeu/cTXnTJo7Y/fa/HaN3//Vw5a+87vvnTh2tXPXLf4wjd+dvFaYBADHgAAANhEJ1lpCR6mzR48AAAAADMnwQMAAABbWB27AdiCBA8AAADAzEnwAAAAwCY6nRWnaDFxEjwAAAAAMyfBAwAAAJvpZEWAh4mT4AEAAACYOQkeAAAA2ETHKVpMnwEPAAAwWX2frxhU/94zj124dteXXzNo7dv//0cvXFv/9I5Ba6+srAyqX1i7jgnGYsADAAAAm6qspMZuAjZlDx4AAACAmZPgAQAAgE10klVXnzFxEjwAAAAAMyfBAwAAAFuwBw9TJ8EDAAAAMHMSPAAAALCJjgQP0yfBAwAAADBzEjwAAMAhs/vOpw6qf/cTh/1Mevflix99dOoTPzZo7T0f/fjCtb26Mmhttt9qS/AwbRI8AAAAADM36wRPVb0/ybVJVpLs6e7Tq+qEJM9Pcock709yVndfPVaPAAAAzJs9eJiDnZDgeVB3f1V3n77++Nwkr+ju05K8Yv0xAAAAwI416wTPfjw8yQPX75+f5NVJnjxWMwAAAMxbp7KyI/IR7GRz/4R2kr+tqjdX1Tnrz92mu69cv//hJLcZpzUAAACA5Zh7gufruvtDVfUlSV5eVe/c+GJ3d1Xtc9v89YHQOUlyTG5+6DsFAABgtpyixdTNOsHT3R9a//WjSV6U5IwkH6mqk5Jk/deP7qf2vO4+vbtPPzJHL6tlAAAAgG032wFPVR1bVbe46X6Sb0hySZILkjxu/W2PS/KScToEAABgJ7jpFK1DfYMh5nyJ1m2SvKiqkrXfxx93999U1cVJXlBVZyf5QJKzRuwRAAAA4JCb7YCnuy9P8pX7eP6qJF+//I4AAGCHOuOeC5d+7W9fPGjplXPOGFRfb/mnhWv3XH/9oLXZSSorPdsLYDhM+IQCAAAAzNxsEzwAAACwDJ1kVT6CifMJBQAAAJg5CR4AAADYglOumDoJHgAAAICZk+ABAACATXQ7RYvp8wkFAAAAmDkJHgAAANjCqj14mDgDHgAA2Olq2F9MH/Dbb1y49rX3+5JBa9cN7xpU39dfP6h+tgb+O19Yj7Ms4BItAAAAgNmT4AEAAIBNdJIV+QgmzicUAAAAYOYkeAAAAGBTjkln+nxCAQAAAGZOggcAAAA20UlW5SOYOJ9QAAAAgJmT4AEAAIAtrHSN3QJsSoIHAAAAYOYkeAAAZqiOPGpQfd94wzZ1wtLs2r1w6cdefOdBS7/2vlcMqF4dtPauWxw3qH7lhgGf9e5Ba49qzr1PUKeyIh/BxPmEAgAAAMycBA8AAABsYbXlI5g2n1AAAACAmZPgAQAAgE10Yg8eJs8nFAAAAGDmJHgAAABgE53KStfYbcCmJHgAAAAAZk6CBwAAALawKh/BxBnwAACMpQbE/Xt1+/pgOXbtHlT+4R++98K1u/+6B629et1lC9fWkcP+yrH68asG1R+2Bv33ZdjnBRiHAQ8AAABsojtZaQkeps0nFAAAAGDmJHgAAABgU5XVOEWLaZPgAQAAAJg5CR4AAADYRMcePEyfTygAAADAzEnwAAAAwBZW5COYOJ9QAAAAgJmT4AEAAIBNdCqr7RQtps2ABwBgLN2Ll+7Zs42NcMBq8b/gXfW9Zwxa+trTVhauPe2JFw1ae4i+fvG+WVwdddQ4tZ92kQiMxYAHAAAAtmAPHqbOJxQAAABg5iR4AAAAYBOdZLXlI5g2n1AAAACAmZPgAQAAgE1VVuIULaZNggcAAABg5gx4AAAAYBM37cFzqG9bqaozq+pdVXVZVZ27j9dvX1Wvqqq3VNVbq+qbD8U/D6bJgAcAAAAmrqp2J3lukm9Kcvckj6qqu+/1tv+R5AXdfa8kj0zyG8vtkjHZgwcAAA5U98KlV52xZ9DSX/Zjly5cuzpoZeaob7hhnHUH/BmZugnswXNGksu6+/Ikqao/TfLwJBv/49BJjl+//0VJ/m2pHTIqAx4AAACYhhOr6k0bHp/X3eet3z85yQc3vHZFknvvVf/0JH9bVT+c5NgkDzlUjTI9BjwAAACwie46oD1ytsHHu/v0AfWPSvK87v6lqrpvkj+oqnt0tyDfYcAePAAAADB9H0pyuw2PT1l/bqOzk7wgSbr79UmOSXLiUrpjdBI8AAAAsIWV5SR4NnNxktOq6tSsDXYemeTRe73nX5N8fZLnVdXdsjbg+dhSu2Q0o39CAQAAgM11954kT0xyYZJ3ZO20rLdX1TOq6mHrb/vxJN9XVf+S5E+SfE/v5J2v+TwSPAAAALCJTrI6/ila6e6XJXnZXs89dcP9S5Pcb9l9MQ0SPAAAAAAzJ8EDAAAAm6op7MEDm/IJBQAAAJg5CR4AAA4bdeRRg+ofe8l7F6794zO/dNDafbNjFi/+zGcGrc0MDdhXt6+/fpR1p6yTrPb4e/DAZiR4AAAAAGZOggcAAAC2sCIfwcT5hAIAAADMnAQPAAAAbKJT9uBh8iR4AAAAAGbOgAcAAABg5lyiBQAAAFtYlY9g4nxCAQAAAGZOggcAgHmpxTc6/eRZXz1o6T88604L1/YV7x20dt94w6B6OGC7di9eu7J9bUxJd7Jik2UmToIHAAAAYOYkeAAAAGALjkln6iR4AAAAAGZOggcAAAA20amstnwE0+YTCgAAADBzEjwAAACwhZXYg4dpk+ABAAAAmDkJHgAAANhExylaTJ8EDwAAAMDMSfAAALBcu3YPKt996xMWrr3+rE8OWnv1T94zoHhl0NqwND6r++AULabPJxQAAABg5iR4AAAAYAurTtFi4iR4AAAAAGZOggcAAAA20Z2sOEWLiZPgAQAAAJg5CR4AAADYglO0mDqfUAAAAICZk+ABAACATXQqq/bgYeIMeAAAODi7dg8rP+rIQfXf/pq3LVz75197t0Frr6yuDKoHgEPFgAcAAAC2sBoJHqZt9nvwVNXuqnpLVb10/fGpVXVRVV1WVc+vqqPG7hEAAADgUJr9gCfJjyR5x4bHv5DkOd195yRXJzl7lK4AAADYETrJatchv8EQsx7wVNUpSb4lyW+vP64kD07ywvW3nJ/kEeN0BwAAALAcc9+D51eS/ESSW6w/vnWST3b3nvXHVyQ5eYzGAAAA2DlWe9b5CA4Ds/2EVtW3Jvlod795wfpzqupNVfWmG3P9NncHAAAAsDxzTvDcL8nDquqbkxyT5Pgkv5rkllV1xHqK55QkH9pXcXefl+S8JDm+TujltAwAAMDs2COHGZhtgqe7n9Ldp3T3HZI8Mskru/sxSV6V5DvW3/a4JC8ZqUUAAACApZjtgGcTT07ypKq6LGt78vzOyP0AAAAwY51kNXXIbzDEnC/R+pzufnWSV6/fvzzJGWP2AwAwdXX00QvX7rrFcYPW/p7XvWlQ/flnPmjh2pVPfmDQ2gAwVTtiwAMAAACHkj14mLqdeIkWAAAAwGFFggcAAAA20ZHgYfokeAAAAABmToIHAAAAtiDBw9RJ8AAAAADMnAQPAAAAbKJTEjxMngQPAAAAwMxJ8AAAHIZ23fH2C9f+1F89f9DazzzzOwfVr1z+3kH1s7Vr9+K1qyvb1wccplYjwcO0SfAAAAAAzJwEDwAAAGymnaLF9EnwAAAAAMycBA8AAABsoiPBw/RJ8AAAAADMnAQPAAAAbEGCh6mT4AEAAACYOQkeAAAA2ESnJHiYPAkeAAAAgJmT4AEAGMuu3YvX9uqgpU/4nY8tXPvMr3rgoLVXrnnvoPrD1urK2B1wuKgBSZXevjaAg2PAAwAAAFtol2gxcS7RAgAAAJg5CR4AAADYwmokeJg2CR4AAACAmZPgAQAAgE10xzHpTJ4EDwAAAMDMSfAAAADAFpyixdRJ8AAAAADMnAQPAMCCdt385oPq6+ijF679gTdeNGjt3/y2hy1cu3rNOwetzQhqYPKge3v6YB78+96HsgcPkyfBAwAAADBzEjwAAACwBXvwMHUSPAAAAAAzJ8EDAAAAm+jEHjxMngQPAAAAwMxJ8AALine truncated
+      "text/plain": [
+       "<Figure size 1152x1728 with 5 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sentence = \"The human voice is the most perfect instrument of all.\"\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " >  Run-time: 3.3077054023742676\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRqS7AABXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YYC7AAAAAAAAAQADAAQAAwACAAIAAQAAAP3/+//5//j/+v///wMACgARABkAIQAnACoAKQAlAB4AFwAPAAUA/f/z/+r/4v/b/9b/0//V/9v/5v/2/wYAGAApADcAQQBGAEYAQwA8ADMAKAAdABMACQABAPr/8//t/+b/4f/g/+L/6f/y//v/AAACAAIAAAD8//f/9v/3//v/AgALABQAHAAgACIAHwAbABQACwACAPj/7//m/9z/0f/E/7r/tv+5/8X/1P/l//j/CwAeAC0ANQA0AC0AIgAXAA0ABQD9//T/6P/a/83/xf/F/83/3f/z/wsAJQA+AFQAZgByAHQAagBWADsAHwAEAOz/1P+8/6j/mP+O/4r/jv+e/7r/3v8EAC0AVgCCAKsAzADdANwAygCuAIcAVAAVAND/i/9O/x7//f7s/uz+9/4J/yD/Qf9w/7L/AABOAJMAxQDmAPcA+QDoAMQAjQBMAAsA0v+h/3v/Xv9J/z3/Of89/0v/Zv+S/9D/GQBlAKcA2gD+ABEBEgEAAd8AtgCOAGoATAAuAA0A6f++/5T/bv9T/0j/T/9i/3n/kv+s/8j/4//6/woAGgAvAEkAYwB4AIQAgwB0AFMAHwDa/47/S/8c/wT/AP8V/0L/hP/M/w8ASQB3AJ8AwQDZAN8AzQCoAHUAOwD7/7n/ev9E/xv/BP8H/yT/W/+l//v/VQCuAAABRwF6AZIBjQFwAUAB/QCkADcAxv9d/wn/zP6j/ov+i/6m/tb+EP9N/5D/4f9AAKQA9wAsAUEBPwEtAQcBxgBwABQAwf99/0X/E//l/sL+sv66/tL+7/4L/yv/Uv99/6r/2/8OAD8AYABlAEsAHgD8//n/HgBlALsAEwFoAbUB7QEFAvoB0AGTAVABDQHAAFkA2f9H/7b+Mv7L/Y79gP2f/er9Zv4R/97/pgBNAcIBEAJQApACwwK/AmkCxAHzABgASv+X/gz+tf2b/cD9Ff5+/uP+MP9e/23/Z/9f/1z/X/9i/2X/bP9v/2H/Pf8O/+T+1v7+/mb/AgC2AGUB8QE3Ah0CpwH6AE8A1f+f/6j/2P8RAEMAawB9AHMAUgAqAAwAAQAOADsAiQDsAFgBtQHaAaYBJwGaAEIAMABJAGEAWQAyAAMA5P/R/8f/yv/q/yEAWAB6AIwApgDBALcAYwDF/wz/gv5T/mf+e/5m/if+2P2D/Tf9C/0Q/Uv9qf0Q/nr+//7I/9cA8gHGAjIDYAOOA84D8QOtA+sC0wHDAAEAm/9c//j+Qv5C/UH8mvuI+//7wvyR/TH+jv7J/in/4f/RAKEBAwLtAZ8BcQGMAb0BtQFhAQ4BFwF/AegB+QGsAUIB9gDVANIA4QAAATUBdAGWAWsB7ABPAL//N/+v/lf+j/6Q/zAB7wIyBJ0EQARyA3cCTAHD/839pfvN+c747fgC+pH7Iv1Z/g//Tv9u/+b/2ADyAa8CwAJQAtwB0gEaAvsB2gDq/gj96vuQ+5L7m/um++f7l/yn/bj+df/w/2oA3gAgAUkBsAFRApsCFQLWAJH/5v7s/jP/LP/n/uf+ev9eABMBbAGFAY4BtAH7AV4C3gJ4A+cDwwMdA4wChgIHA8UDggT6BBMFCQUaBRMFjAR7Aw0CWACH/hL9WvwS/J77pfpE+f73l/eY+IP6HPzQ/E/9sf4aAZgD9QR+BKECowCf/6L/1f+x/1z/N/87/wn/a/5W/SL8Yvt6+yr8tvzc/PX8a/1X/pD/8gD7AR0CdwHjADYBTwJNA0gD2wFw//v8avv1+kT7/fvp/Lj9Cv4M/lX+J/8NAHEAXwBqAFEBeQNNBk8IMghSBiUExQJ3Ai0D0QS5BiwILQkbCgULjQt1C4AKbwiyBUIDoQGCAIn/m/4//ez6/fdu9bTzb/Jx8RTxxvHi82n3j/ul/o7/vP6a/U79GP7I/74BhwMZBYkGpgetB1kGAARAAar+Xfxn+rf4l/eJ93T4rPlF+g76b/kI+bj5z/sF/3cCZAV/B1EI+Ac0B9IG3wZgBowEEAF1/Dv43fWX9W32oPcU+bD6dfwI/68CDgZVB2IG7wTiBBoH3QoJDpsOxwyGCg0JTAjuBxUIpwh4Cf0KCA2FDloOgwxVCaQEN/91+j/3B/UU86nxtfAG8FHvgO6c7YTsSewE7hbyvfcN/QUBZwPEBJIFlAV8BdMFtgbPB3MIdQjqBlwDbP4C+WH0UvGh8AjyEfQx9qv4vftj/ov/oP9R/2P/qQD0A/sIfw2qD/4O1wseBxQCWf4H/OT5cvdf9cL06/V1+Ij7Ef7E/1MBDgMbBDEEcwSFBj0KzA3KD7kPsA7/DTkOoQ7nDcUM6Az4DpERvhJREo0QcA2uCEkCbvtC9efwgO4V7SzsgOs769Xqo+n254DmA+ep6jzx5PhJ//UD8QZaCFYIeAfRBnYGegbtBpcHxQdoBjgDJ/699wfxgOuM6KLoBOwN8tb4p/15/2sA+gIMB/MJDAp2CBUI6gr7D3ATXRF7CtECp/2A+pf3QvUL9E30QvVz9iL4t/pU/2IEngbGBKYBEQKbBm4MwRDaEjwUZxUVFqsVfxS0FHcWvRg9Gj0a3hj7FGEPEAm2AYP5IPE86/jnxOVf5PjiYuEa31ndw91J397hbeYM7qv3NAAuBgIJngnJCVQK+wrNCkoKeQp1CtEI0QRA/8L5xfRq8KLr9OYl5fjnXO/I93D+EQLdArYDRAZkCqwNkw6hDq4OCA9aD4cPCQ82CyYDxvis8N7uuvIe+BD60/eF9Vr3Af3lAfoCVQJ7A78GMgosDZ8R8BfgHdsghR+8G0AZTRtTIR4mASZDIV4aQxNJCwwCZPfU7CLlOeCw3HLZkNep13fX/dVa1PjTV9b+3EPoFPXk/psFWAuBD7QQ8A6oDDwLQgoeC7QM+wsQCGgCrP1d+CPxXeqO5fLj0eXl64712P4oBrIK6AtmCtII+QqjDoUPcQ1hDC4PdhLAEd0Kc/9N9ILulu+O8h3zrfG/8bD03fcb+/3+HwNABmQHmAgxC3gQBxlpIsgoPCmdJv4lkihlLLku1S76K5wkoxmyDYkC0vft7KniDdrR03PQzM/Az8/NisooyGHIEMyg0x/fkOwp+bIDXAs4D88Pvw5UDfALJQtPDMEOlQ9uDeMIjgKS+sDxvurT5qflhed47Xv3GgIcCtMNJw5DDUsM6QzrDZIOWw+hEPASPRNcD2wHRP1W9ELuYOsP6+3rCu3+7Rvv3PGs9l78PgEbBMAFPghPDjkYySLCKuYvvDOuNSc1oDStNoQ5dDiPMaomkxnsCzP/VPPy5v3Zy88jym3HpsUQxGfCHsBhvqS/YMVIzzrdxewK+kID8gjNDY8QZxByDtULvgv7DbQRYRP6D18JBAEU+fzxq+wq6hHqL+519qgARgkUDoMQfxDiDtAN9g1lD00QdBFqEygTLA9FBx7+1PW+7oPqXOhZ5/vmcOfz6W/tlfFk9oL73wCSBZcKBBHcGUcl9S9pNzo6ojrWOxk++0AgQS49TDXfKqcfIRIIA6jz7uV62trPSsdtwfa+fL4bvVW6Nrf7t5a+u8nX1kPj5u77+KgB9Qi6DSQPWg2tC4YMZQ8QEn8SVxDMCwEGLADy+j/2XPM+9Nz4o//uBSML7g7SEDURRhA3D30OWQ4uD90PZQ8IDGIFyv3d9vfxme0t6f7leuSJ5VfnmenF7IjxYfjN/h0ERAnMEG0bbCZwLxU1vzi+O4c+70BAQrRCY0HxPNozpCaNF8QIJ/sW7bbe19FByUHFfcKQv8y78LfvtCq0mrhlwfHMLdn95OLvlfjx/4wFPQjaB6wG9gf5CxoQkRJIEucPPgyLBwUDvv4Q/Wr+AQKMBmYKLw7iEHsSpRIXEQ4Pgw2UDRsOFQ0oCnoF3//k+Xb0YfDm7PDpaefA5RblAeaV6fHuo/Tp+bH/0gYaD8AXaSBPKPouMjTNN3Q6dzwUP5RBjEISQJY4Ci0mH94RqwXB+PvqLN7y1DTPrspTxuDBbb1cueK2fLe+uyXDFc1L2J3iCevJ8dv3w/yG//cApwLSBWwKPA+uEuYSSxCHDdILfApyCAgH4AczCscMQA+XEcESKBKIENsOCg1qC3IK+wixBXAA0Pvt+Mz2svSo8drtzumO58Xo0evk7hLyt/bd/PsC9QizD0MXyh7pJEspFyyNLnAyIDfhOrw7FzrXNjMx0yiwHbgRJAY7+0LxuOd/3xrZmNTh0NjLp8XKv4C84rw2wLfFPcyg073bk+Nh6kvvFfNb9ir5WPxGAIwF7graDvEQLxG9EBAQsw+RDwMPaw5kDr0PyhE9E1cTGxIVELoNWAsVCZoGhANMAHz9b/t4+Rz3tfSI8rzwHe817s/uOPHO9M34DP3QAYAHgw0VE6EXahtsH8Yj8Sf6Ku4shS7dL0MwoS6zKt4kvh0BFvcNnQUB/Yv0Ou0u56/hGdyA1q/Rvs2/yg/JKMkGywjO5NGp1uzbBeFy5V7pUu128e31QPod/rYBXgUWCQgMtg1+DswOzQ6cDo4O9A6yD34QCBH5EDcQBw+iDd4LnQkoB/AEEwNXAbT/Q/4J/br7E/ol+Jz2Kvbh9nn4SPpB/Hz+cwGCBeQJhQ3HD8MRtxSXGDwcxR55IOIhCiNPIyYimx9VHNsYEBWJEEcLyQV4AEj79vV38A3rM+ZQ4kjfvNxy2rrY+tcQ2OXYTdpB3JneJ+FJ5ObnmevD7lLx7fOs9ov5Dvwr/j8ATgKJBF8Glgc/CKIIXQkNCocKuAr4CroLagy1DHUM3AsiC/0JtghyBycG7gQQBB4EngSoBA0EfAOhAzIEnATXBF0Fowa5CCALTQ0aD8oQhBK8Ex4U7hPgE1kU3BQAFUsUtBKEENwN5QpHBy8DF/9m+zDLine truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAa4CAYAAAD7hW7eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xu8ZmdZH/zfNZMjIQFCKEbCISByKCrYiCB9BVEwRSv0U2uhSqlSUy20KFRBbQVRK7ZVtJb6NgqSnkSgCikvFjmWWikSSkQI4RRQEiHhECCBZDKz9/X+sXd0O8zsnbln5llr7fl+P5/nM8+z1rr2fe2913OYa1/rvqu7AwAAAMBy7Zk6AQAAAACOjgIPAAAAwMIp8AAAAAAsnAIPAAAAwMIp8AAAAAAsnAIPAAAAwMIp8AAAAAAsnAIPAAAAwMIp8AAAAAAs3ElTJzAHp9SpfVrOmDoNYEXqlFPGAtfXBuN6LG7VauWBgwZ/nqO/htFv76TBt9i1sfOsDwyenwAktdr3srWzbzcU14Np9h3G3iP2fHbvEcfsu/EzOXDzF1b94eC4+9ZvOqM//Znj/177znfve113X3jcB2JXUuBJclrOyNfXN0+dBrAiJ513z6G4vuHGsbibbh6Ky57VNlnW6IfbPYNxNfj9jRZABuNq75F/uE2S3PWcsbjPfHYobO3TnxkbD4DUqaeOBQ7+Eeczf/OvjQ138lBY9n3b54biTn/VHY445n3//YVDY83dpz+zlj983T2O+zh7z/3g4AcIUOABAACAbXWS9axPnQZsyxw8AAAAAAungwcAAAC21VlrHTzMmw4eAAAAgIVT4AEAAABYOJdoAQAAwDY2JlkeWzUNVkUHDwAAAMDC6eABAACAHVgmnbnTwQMAAACwcDp4gBPOgY/+6WoHbNdrn5BuuGEszvkCMKxOPmUo7sZvf/BQ3OfO3zsUd8bHxzpBPvnwsbj7Pb+G4vZ85Iojjjnp8zcPjTV3nc6a92hmTgcPAAAAwMLp4AEAAIAdWEWLudPBAwAAALBwOngAAABgG51kTQcPM6eDBwAAAGDhdPAAAADADszBw9zp4AEAAABYOB08AAAAsI1OstY6eJg3HTwAAAAAC6eDBwAAAHawPnUCsAMdPAAAAAALp4MHOPG4fppVcJ4Bu0nVUNjeO5w1FPfp73jgUNzTfvwVQ3E/864HDMWd9abbDcXd4TffMRR3x98a+z302tpQ3Poppxz5WOu7s8+l01mzihYzp4MHAAAAYOF08AAAAMB2OlnTwMPM6eABAAAAWDgdPAAAALCNjlW0mD8dPAAAAAALp4MHAAAAtlVZy9gqZrAqOngAAAAAFk4HDwAAAGyjk6xbRYuZ08EDAAAAsHA6eAAAAGAH5uBh7nTwAAAAACycDh6Yuxr8S0G7SBgAdq09e4fC9t73/KG4m/7d/qG4Hzv/tUNx/+w9Xz4U91sXfsNQ3H2u++BQ3PpNNw3FjeoDq/181/v2DQTtzs+gHR08zJ8OHgAAAICF08EDAAAAO1hvHTzM2+I7eKpqb1W9q6pes/n4/Kp6e1V9qKp+q6pOmTpHAAAAgONp8QWeJM9I8r4tj38+yQu7+yuSXJ/kqZNkBQAAwK5w6xw8x/sGR2PRBZ6qOi/JtyX59c3HleTRSV65ecglSZ4wTXYAAAAAq7H0OXh+KcmPJjlz8/Gdk3y2uw9sPr46yd0OFVhVFyW5KElOy+2Oc5oAAAAsVaeytuz+CE4Aiz1Dq+rbk1zX3e8cie/ui7v7gu6+4OSceoyzAwAAAFidJXfwPCLJd1TV45KcluSsJL+c5I5VddJmF895Sa6ZMEcAAAB2AatoMXeL7eDp7h/r7vO6+15JnpjkTd393UnenOQ7Nw97SpJXT5QiAAAAwEosuYPncJ6d5GVV9TNJ3pXkxRPnAwAAwILduooWzNmuKPB091uSvGXz/lVJHjplPgAAAACrtCsKPLCrdU+dAQAwM3sf8BVDcR/7tjsPxZ356+tDcb/0lm8aijvvwLVDcQeu/9xQXNbXxuI4gVTWerEznHCCcIYCAAAALJwOHgAAANhGJ1nXH8HMOUMBAAAAFk4HDwAAAOzAKlrMnQ4eAAAAgIXTwQMAAADb6LaKFvPnDAUAAABYOB08AAAAsIN1c/Awczp4AAAAABZOgQcAAABg4VyiBQAAE6mTxj6Of/przx6K27NvKCx3ePXlQ3FrN988NiDzUAOXJPWxT2MOOsma/ghmzhkKAAAAsHA6eAAAAGBblkln/pyhAAAAAAungwcAAAC20UnW9Ucwc85QAAAAgIXTwQMAAAA7WOuBVcVghXTwAAAAACycDh4AAADYRqeypj+CmXOGAgAAACycDh4AAADYwXrrj2DenKEAAAAAC6eDBwCAedqzdyyu149tHrdpzB4Ku+aHH3qME9nePf7bnw3FHdi37xhnskvU4KpKg+fLsME8a+/Ac3DF39qqdGIOHmbPGQoAAACwcDp4AAAAYBudyloPdmzBiujgAQAAAFg4HTwAAACwg3X9EcycMxQAAABg4XTwAAAAwDa6k7XWH8G8OUMBAAAAFk4HDwAAAGyrsh6raDFvOngAAAAAFk4HDwAAAGyjYw4e5s8ZCgAAALBwOngAADi+amzeitozFtcHeiguSfY8+IFDcT/4ilcNxT3jDQ8dinvgv75uKG79zz4xFJce/5nuaqM/l8HnxPB4g3F94MBA0NBQi7CmP4KZc4YCAAAALJwOHgAAANhGp7LeVtFi3nTwAAAAACycDh4AAADYgTl4mDtnKAAAAMDC6eABAACAbXSS9dYfwbw5QwEAAAAWTgcPAAAAbKuyFqtoMW86eAAAAAAWTgcPAAAAbMMcPCyBMxQAAABg4XTwAABwfHWPhR04cIwT2dkn/9odhuL+w6MeNRR3v0/90VBcn37aUNz6zTcPxbFwe/aOxfX6QMzYUEtgDh7mTgcPAAAAwMLp4AEAAIBtdJc5eJg9ZygAAADAwi22wFNVp1XVH1bVH1XVe6vqpza3v7SqPlJVl2/eHjx1rgAAACzbWu857redVNWFVfX+qvpQVT3nEPvvUVVvrqp3VdW7q+pxx+WHwSwt+RKtfUke3d03VtXJSX6/qn53c9+PdPcrJ8wNAAAAjpmq2pvkRUkek+TqJO+oqku7+4oth/3zJC/v7l+tqgcmeW2Se608WSax2AJPd3eSGzcfnrx528VztgMAADCFTrI+/SpaD03yoe6+Kkmq6mVJHp9ka4Gnk5y1ef8OSf5spRkyqcVeopVsVDCr6vIk1yV5fXe/fXPXz262o72wqk49TOxFVXVZVV22P/tWljMAAAAcxjm3/j9183bRln13S/KxLY+v3ty21fOSfE9VXZ2N7p1/clyzZVYW28GTJN29luTBVXXHJL9TVQ9K8mNJPpHklCQXJ3l2kucfIvbizf05q87W+QMAAMBh1G2aI+cY+FR3X3AU8U9K8tLu/oWqeniS/1RVD+ru9WOUHzO26A6eW3X3Z5O8OcmF3f3x3rAvyW9ko40NAAAAluyaJHff8vi8zW1bPTXJy5Oku9+W5LQk56wkOya32AJPVd1ls3MnVXV6NiaaurKqzt3cVkmekOQ902UJAADA0nWS9a7jftvBO5Lct6rOr6pTkjwxyaUHHfOnSb45SarqAdko8Hzy2P40mKslX6J1bpJLNmcS35ONmcJfU1Vvqqq7JKkklyf5gSmTBAAAgKPV3Qeq6ulJXpdkb5KXdPd7q+r5SS7r7kuTPCvJr1XVD2ejLvUPNhco4gSw2AJPd787yUMOsf3RE6QDcOKqwRUlfNYAdlAnjX1UHY1Lkju9/6ahuLXrPjUUVyeP5bp+081DccxD7d07FNdra2PjDZ5nGRlvF7+9r83gApjufm02Jk/euu0nt9y/IskjVp0X8zD9GQoAAADAUVlsBw8AAACsQuc2zZEDk9LBAwAAALBwCjwAAAAAC+cSLQAAANjBuv4IZs4ZCgAAALBwOngAAABgG93JmkmWmTkdPAAAAAALp4MHAAAAdmCZdOZOBw8AAADAwungAQAAgG10KuutP4J5U+AB4Oh0T50BsCo1eHnCil8n+q/eZzj2prueOhR3xv5bhuL6wP6hOGZiz96hsF5bGxtv8LlUg8/drpGChsuYYCoKPAAAALCDNcUrZk6PGQAAAMDC6eABAACAbXSsosX86eABAAAAWDgdPAAAALAtq2gxf85QAAAAgIXTwQMAAAA7WLeKFjOngwcAAABg4XTwAAAAwDa6kzWraDFzOngAAAAAFk4HDwAAAOzAKlrMnQIPwG1Vg2253cc2D4CjNfp6tmJ1+ulDce//3tsPj3n/f/GBobi10QG9R8zD4HNiz2mnDsWt33TTUNyq7Tn9tCOOqbVlvL7AbqTAAwAAANvoVNbNwcPM6TEDAAAAWDgdPAAAALCD9ejgYd508AAAAAAsnA4eAAAA2EYn5uBh9nTwAAAAACycDh4AAADYwXrrj2DenKEAAAAAC6eDBwAAALbTZQ4eZk8HDwAAAMDC6eABAACAbXSS9ejgYd4UeABuq+6pMwA4NmqwibvXj20eO/j0337QUNw9fvfA8Jhr118/HDukBv/D6D1pFtZv3jcWuOLf3/ot+8cCB+J6bbWvE8BfUOABAACAHZiDh7kzBw8AAADAwungAQAAgG10dPAwfzp4AAAAABZOBw8AAADsQAcPc6eDBwAAAGDhdPAAAADANjqlg4fZ08EDAAAAsHA6eAAAAGAH69HBw7zp4AEAAABYOB08AAAAsJ22ihbzp8ADAHCi6fXBuB4K++4rrx6Ke+7v7x+Ku98PXD4UlyQ5abUfj/vAgZWOx2EMntvptbG4PXvH4tYHxxuNAxZFgQcAAAC20dHBw/yZgwcAAABg4RbbwVNVpyV5a5JTs/F9vLK7n1tV5yd5WZI7J3lnkid39y3TZQoAAMDS6eBh7pbLine truncated
+      "text/plain": [
+       "<Figure size 1152x1728 with 5 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sentence = \"I'm sorry Dave. I'm afraid I can't do that.\"\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " >  Run-time: 3.859889507293701\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <audio controls=\"controls\" >\n",
+       "                    <source src=\"data:audio/wav;base64,UklGRuTaAABXQVZFZm10IBAAAAABAAEAIE4AAECcAAACABAAZGF0YcDaAAD+//v/+f/3//T/8P/u/+//7//v/+7/6//s/+3/7//y//P/9v/4//r//f8AAAMABgAHAAkACgAOABMAFQAXABYAFgAWABUAFwAYABgAGAAWABQAEQAPAAwACgAJAAUAAwAAAPz/+P/0//H/7v/q/+f/5v/n/+b/5P/h/+H/4f/i/+X/6P/s//D/8//2//3/AgAHAAsADgAVABwAIgAnACsALQAuAC4ALAArACsAKwApACQAHQAVABAACAAAAPn/9P/w/+j/4f/a/9T/0v/P/8z/yv/M/83/z//S/9X/2f/g/+b/7v/2//z/BQAOABYAHgAjACwAMwA7AEEARgBKAEoASQBGAEcASABDAD0AMgAqACQAGQARAAYA/P/y/+X/2//V/9P/z//G/7v/sf+v/7L/t/+8/77/wP++/7//xv/Q/9r/3//h/+b/7f/y//X/8//y//P/9v/7/wEAAgD+//f/7//x//f/AgALAAsACQAFAAgAEwAZACAAJAArADYAOABAAEUASQBOAEkASgBRAFgAXABWAFMAUABOAEsAQQA4ADEAKwApACUAHQASAAMA9//u/+r/6//v/+r/3//Q/8D/v//C/8n/zv/I/8r/wf+3/7b/tf++/8L/v/+7/7v/xP/H/8H/uv+6/8L/x//P/9T/1P/Y/9P/0P/Z/+T/8f/2//X/+v/+/wMACwAJAAUACgAQACAALAAsACgAHAAYABUAFQAhAC8ANQAtABkAEgAWABQADwAIABAAHAAlACIAGAAbABcAFQAdACYARABTAFEAUgBDAFUAaQBtAHYAaABmAGsAcQB3AHUAbQBTAD8ANgAwADIAJQAGAO7/3v/e/9n/yP+1/53/iP97/3P/eP+B/3f/a/9e/1z/b/97/4b/h/+E/5j/rf/C/8n/xf/Y/+v/AgAGAAAAFwAbABgAGgAWAC4APQAsACMAJQAoAB0A8v/T/9r/5f/c/7H/hf9y/1T/Jv/7/uH+5/7Z/rL+oP6d/rb+vf60/tD++/46/23/pP8GAF0ArADtADgBqwEXAnECvQIXA2wDsQPQA94DAwQWBB8E9QO2A48DSgP/ApMCJALHAU8BxwAdAJP/Hv+X/hn+jf0s/d38cvwh/Mv7ofuT+2H7U/tX+4D7x/v8+zr8iPzW/CP9hP3f/VX+y/4d/4r/3P85AKUA3wAgATkBRgFtAXwBlgGmAZsBkQF2AUcBIwHxALcAlgBjAEsAPgAfACYABADa/7j/lf/P/xEAUQCLAJYA0AAGATkBlAHYATsClALTAioDbwOtA98D8AMHBBoEIwQjBP4D2AOPAzUD0wJFAuEBVwHJAFMAm/8D/0j+iv0W/Yj8Ivyr+xP7uvph+ir6JPoV+i36R/pI+pD66fpO+8v7Hfyg/Eb9x/1P/sj+S//u/2sA1gA6AZcB5gERAhgCLgJcAocCnAJwAj0C6wGMATsB2QDMAMAAkwBeAOj/pP9j/xj/C//w/uX+9v72/jL/jP+1/+X/CQAjAJkAFgGpAXwC3wJcA7ED1wOKBPUEbgXtBe0FVQaJBo8GzgZwBjkG8wVaBREFYgS5AzQDNgJSARIAw/7Y/af8pfuT+m/5vvjt9yb3b/an9Ub1CvXw9Cr1g/UV9rD2Lff299/47fkJ+wT8TP2U/v7/RwFpAqMDaQQmBaAFIAbxBnEH7gcKCMoHfAfSBicGlgX/BGEEiANjAlMBYAB+/7b+vf3Q/Nj78Ppz+kX6iPqb+mb6QPpE+uv60fvQ/On9zv64/70A9gGJAyUFcQZnBwII1gj/CSQLCwxDDDwMFgzOC48LEAuACo0JDghbBsQEgQM8ApcAd/5S/Gr6qfhA98X1ZvRW8wvyM/Fk8Lzv1u/M70HwzvBj8aTyr/Py9Cv2i/eC+U77V/3a/nMAmgL9A94FAweyB0QJogl9CikLIAsBDJAL8wpGCtUIQAhFBzsGewUQBMgCKQFe/wj+oPys+7/61flB+af4S/j+98z31PdT+Af5Jfqq+838Qf5t/1YAJwKwA7YFAggqCbUKsQtqDL0NQA7lDnkPSQ8nD3IOZA2lDIQL+QnUB18FRgOOAav/Mv2H+vL3o/Wk85Lx1e+F7h/tH+wD65LqEOt46yXsZ+wL7cju0fAo8yv19/ZW+Y/7vf0bAJ4CYgV0B9sI5wmUC3gNXw78DtoOEA+8D/AOSQ52DfYLUQtKCXAH0QbtBLgDlwHh/gD+LPy9+tP5+feM9wf3Mva79p/2vfac97v3G/nZ+vT7Kv6q//wAWAMJBUgHlgmBCjYMnA1/DuEPLRDCEJwRDBFaEAgPwQ3uDAEL6AhxBmsE9wHz/vT7Dfni9v/zMfGr7vXsnOu76Qfo3Obe5h7nH+eu56vok+px7O/tYPAd8wf2p/gN+wb+jwFwBLsGHglIC7sNWg8yEL8RxRJ8E5UTRxI3ErIReRBFD/sM5AuJCj0IwAUiAzwBmv/J/Zb7OPq6+Ev3OPYD9TP1nPXK9eT1C/YX97r41vqC/EL+SADqAW4EbgafCCgLMgwHDkcPURAwEiQS3BKNEvEQgBCUDtsNtAzBCXcHGQQ6Aer+fPuI+Ez14/FN76bsgOq26ALnSeXf4zHjBeMH5OLkB+ZZ5+josOtZ7k/xffTR9xr7ev73AAUEAgjQCvENARA6Ea4TxBQDFl0XsRYIF6gVGhTrEygScxGBD6QMdQofB9AEZwJPAN/+NfyA+jj41fYo9j30Y/TQ82X0gvUw9cv28/fP+e37pvwz/5sBMAQIBwYIuAqLDOANcA9bD3YRxBJgEgsSrg8XD9cOZgwiC/kHwgTiAkz+wPt3+Xr1wPMM71brT+pO55Pmp+S24STimOER4t3jzOMx5qbnY+hl7FXvW/Sx+CT6NP6YABcEvwivCjsPXBLDE6QWzhY0GPUZ5BgwGesY1xerF70VehP1EeUOKQwGCdkFewRvApP/0vwC+kj4yvZK9I7zbvOA89/zTvMb9IP1k/Yc+LX5+PtY/tAAGgN+BYgI1AmKC3ENFg6dEAUR9hCpElIQCxDdDiAMXwzTCNsFOAOH/1X9WPmT9b3yQe967KDpBed65V/j+OA54DHgaeH44mLjUeTa5d3oV+uJ73vzmfZO+3f9TAEJBqUJ7A15ELESPBUoGAwaJBuAHPsb/RzOG/EZERqlF4IW2BPmDyUOSguRCJMFugE//7z8CvqR9+f1kvQJ80XybPGe8dvyg/Jc81z1PPZ0+Y37Z/26ALUBKAQfB50J0gtKDZ8O0g6AEA8QWg9cEMMNoQ0ADMcHMAfDAhP/av3O96H1GfO47l/tHOlQ5uXkxOHU4FLgk+Ax4lfjyeK149jlh+g87R3wYPQn+ZX7o/8BAuoG4wxMD1cT8BR7F9sbvxzqHfUerh6nHjYe+BtOG58a/RatFHsQLA35C3AIfQVsAbb8Ovrm9xv2+/T88uTxMvBc7zfvufC/8lHzEPWK9Z/4wvtL/gYB+AHmBNcGmAgEC7kMRQ+pDwkP+A0RDVkNvAs9CgUHhASjA+f/Yfyv+P7zTfKN7hvqMOnz5qfmjeT34Lrg5OBK4irj4OKf5c7pGe1p7w/xcfWA+nf+lgFdBBYLOBCFEmIUWxVkGm0dyx3tHpMf6CBUIGAdCxzVG4AZChY/EqMPjQ5pCxwGUQIv/3b8fvnd9Qz1RvRe8n7wKe8Y8AzyRfEU8Wjz3vQY+X76xvuF/2YAVgJrBFEGPQp7C2gKzwsDC0cMHQyFCfsJWAfgBEMDZ/8p/gX7Pvbn9Krwxe6z7FHpvugz5iDkDuRQ5KTkHeUu5QbnAOqY7C/vnPIV9tD5Rv2Y/10EjQmMDc0QzxK3FQQZkRodHG8dth6wH/wd7BzLHDobSRmjFf4RCxDADYkLTwgIBWIBGv6M+nv3iPfx9cL0ePMu8Y7yNvNy8vnzdfQ19Xn3Rfjj+4v/xv+hAQgCogPxBpgGeAdUCIoIiwn1B8sGmwWPA6YCOv+f/Jj7DPld9yf0qfC+7ivsOuqG6CvoGOlr6VLoB+fy5xvpBux87hDxcvUD+Pn5cvxAAJkEKgjiCpQN1hAuFEIWMBgJGmcaGhptGfwZwBqdGZ8Y2hWtErwRIw+nDD4KmwcjBrACagDc/TT8EvzR+DX4Efcu9sL2wvTt9M71WPbS+Af57flt/KP85v1E/k3/sAIyA/sD9gRiBOwF1gQnAyMDyQGaATQAjP0J/Ub74/eU9d7zgPKA8W7vXe2l7Rft/Ovq6+zr7+xb7obv7/Cb8/H1GPcr+sD8af9WA3kE9wZiCrkLjQ/4EIwRuxMXE0kUxhS9FG0VLxRHEwYSyRACEEYOiQxMCgAJCwdsBY4E7ABoASAAgf1T/Xn70Pu/+qD5GfoG+hT6Fvo9+hP6YftW/E38iP0p/if/Xf72/T3/5P76APcAVv/r/w/+xf0f/R77GPyQ+pT5i/hq9yf3TvUX9BTzofMd8xD0FfUT9Jb1ufVT9TD3evgK+nD7+fyh/sv/1QHQAqQEjgahBi4JMArQCCALtQz/C9AMRA1cDBkOAw0EC18MiwruCQwJlAfXCBwI5AcHBa8CzwNnAbsBHwHhAFsBT//w/q78UP3n/SH8EP2c/D39JP6M+2P9D/2M+/f8efpa/e7+hf1K/hn8sf36+8z6Of0E+8T83fp1+Qb7yPjf+gT5GfcQ+S34w/is+Pr5Wvr2+ND6NPqS+2r8E/2M//f9cgDVAEEBvgIKAkoDlAN+BaIFxwZpB7UGBgiGBkEHRQinBq4ImgetBuIHRwbvBpgE3AQuBYUD9gNjAgkE8AE9AOUBhQAU/xUB4wBn/b3+i/42/2b+Lf0lAYH9m/2O/1/8hQCD/k7+vP+v/AsA8P0e/dr+pf0V/if90vy7/I/+ePtv+uP9QPrn/HT8h/r2/LPLine truncated
+       "                    Your browser does not support the audio element.\n",
+       "                </audio>\n",
+       "              "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.Audio object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sentence = \"This cake is great. It's so delicious and moist.\"\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Comparison with https://keithito.github.io/audio-samples/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sentence = \"Generative adversarial network or variational auto-encoder.\"\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sentence = \"Scientists at the CERN laboratory say they have discovered a new particle.\"\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sentence = \"here’s a way to measure the acute emotional intelligence that has never gone out of style.\"\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sentence = \"President Trump met with other leaders at the Group of 20 conference.\"\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sentence = \"The buses aren't the problem, they actually provide a solution.\"\n",
+    "align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/synthesis.py

@@ -40,8 +40,12 @@ def visualize(alignment, spectrogram, stop_tokens, CONFIG):
     plt.plot(range(len(stop_tokens)), list(stop_tokens))
 
     plt.subplot(3, 1, 3)
-    librosa.display.specshow(spectrogram.T, sr=CONFIG.sample_rate,
-                             hop_length=hop_length, x_axis="time", y_axis="linear")
+    librosa.display.specshow(
+        spectrogram.T,
+        sr=CONFIG.sample_rate,
+        hop_length=hop_length,
+        x_axis="time",
+        y_axis="linear")
     plt.xlabel("Time", fontsize=label_fontsize)
     plt.ylabel("Hz", fontsize=label_fontsize)
     plt.tight_layout()

requirements.txt

@@ -1,10 +1,12 @@
+numpy==1.14.3
+lws
 torch>=0.4.0
-librosa
-inflect
-unidecode
+librosa==0.5.1
+Unidecode==0.4.20
 tensorboard
 tensorboardX
-matplotlib
+matplotlib==2.0.2
 Pillow
 flask
-scipy
+scipy==0.19.0
+lws

server/README.md

@@ -1,9 +1,9 @@
 ## TTS example web-server
 Steps to run:
-1. Download one of the models given on the main page.
-2. Checkout the corresponding commit history. 
-2. Set paths and other options in the file ```server/conf.json```.
-3. Run the server ```python server/server.py -c conf.json```. (Requires Flask)
+1. Download one of the models given on the main page. Click [here](https://drive.google.com/drive/folders/1Q6BKeEkZyxSGsocK2p_mqgzLwlNvbHFJ?usp=sharing) for the lastest model.
+2. Checkout the corresponding commit history or use ```server``` branch if you like to use the latest model.
+2. Set the paths and the other options in the file ```server/conf.json```.
+3. Run the server ```python server/server.py -c server/conf.json```. (Requires Flask)
 4. Go to ```localhost:[given_port]``` and enjoy.
 
-Note that the audio quality on browser is slightly worse due to the encoder quantization. 
+For high quality results, please use the library versions shown in the ```requirements.txt``` file.

server/conf.json

@@ -1,5 +1,5 @@
 {
-    "model_path":"/home/erogol/projects/models/LJSpeech/May-22-2018_03_24PM-e6112f7",
+    "model_path":"../models/May-22-2018_03_24PM-e6112f7",
     "model_name":"checkpoint_272976.pth.tar",
     "model_config":"config.json",
     "port": 5002,

server/server.py

@@ -2,12 +2,11 @@
 import argparse
 from synthesizer import Synthesizer
 from TTS.utils.generic_utils import load_config
-from flask import (Flask, Response, request,
-                  render_template, send_file)
+from flask import Flask, Response, request, render_template, send_file
 
 parser = argparse.ArgumentParser()
-parser.add_argument('-c', '--config_path', type=str,
-                    help='path to config file for training')
+parser.add_argument(
+    '-c', '--config_path', type=str, help='path to config file for training')
 args = parser.parse_args()
 
 config = load_config(args.config_path)
@@ -16,17 +15,19 @@ synthesizer = Synthesizer()
 synthesizer.load_model(config.model_path, config.model_name,
                        config.model_config, config.use_cuda)
 
+
 @app.route('/')
 def index():
     return render_template('index.html')
 
+
 @app.route('/api/tts', methods=['GET'])
 def tts():
     text = request.args.get('text')
     print(" > Model input: {}".format(text))
     data = synthesizer.tts(text)
-    return send_file(data,  
-                     mimetype='audio/wav')
+    return send_file(data, mimetype='audio/wav')
+
 
 if __name__ == '__main__':
-    app.run(debug=True, host='0.0.0.0', port=config.port)
+    app.run(debug=True, host='0.0.0.0', port=config.port)

server/synthesizer.py

@@ -13,39 +13,44 @@ from matplotlib import pylab as plt
 
 
 class Synthesizer(object):
-
     def load_model(self, model_path, model_name, model_config, use_cuda):
         model_config = os.path.join(model_path, model_config)
-        self.model_file = os.path.join(model_path, model_name)        
+        self.model_file = os.path.join(model_path, model_name)
         print(" > Loading model ...")
         print(" | > model config: ", model_config)
         print(" | > model file: ", self.model_file)
         config = load_config(model_config)
         self.config = config
         self.use_cuda = use_cuda
-        self.model = Tacotron(config.embedding_size, config.num_freq, config.num_mels, config.r)
-        self.ap = AudioProcessor(config.sample_rate, config.num_mels, config.min_level_db,
-                                 config.frame_shift_ms, config.frame_length_ms, config.preemphasis,
-                                 config.ref_level_db, config.num_freq, config.power, griffin_lim_iters=60)  
+        self.model = Tacotron(config.embedding_size, config.num_freq,
+                              config.num_mels, config.r)
+        self.ap = AudioProcessor(
+            config.sample_rate,
+            config.num_mels,
+            config.min_level_db,
+            config.frame_shift_ms,
+            config.frame_length_ms,
+            config.preemphasis,
+            config.ref_level_db,
+            config.num_freq,
+            config.power,
+            griffin_lim_iters=60)
         # load model state
         if use_cuda:
             cp = torch.load(self.model_file)
         else:
-            cp = torch.load(self.model_file, map_location=lambda storage, loc: storage)
+            cp = torch.load(
+                self.model_file, map_location=lambda storage, loc: storage)
         # load the model
         self.model.load_state_dict(cp['model'])
         if use_cuda:
             self.model.cuda()
-        self.model.eval()       
-    
+        self.model.eval()
+
     def save_wav(self, wav, path):
         wav *= 32767 / max(1e-8, np.max(np.abs(wav)))
-        # sf.write(path, wav.astype(np.int32), self.config.sample_rate, format='wav')
-        # wav = librosa.util.normalize(wav.astype(np.float), norm=np.inf, axis=None)
-        # wav = wav / wav.max()
-        # sf.write(path, wav.astype('float'), self.config.sample_rate, format='ogg')
-        scipy.io.wavfile.write(path, self.config.sample_rate, wav.astype(np.int16))
-        # librosa.output.write_wav(path, wav.astype(np.int16), self.config.sample_rate, norm=True)
+        librosa.output.write_wav(path, wav.astype(np.int16),
+                                 self.config.sample_rate)
 
     def tts(self, text):
         text_cleaner = [self.config.text_cleaner]
@@ -54,14 +59,15 @@ class Synthesizer(object):
             if len(sen) < 3:
                 continue
             sen = sen.strip()
-            sen +='.'
+            sen += '.'
             print(sen)
             sen = sen.strip()
             seq = np.array(text_to_sequence(text, text_cleaner))
-            chars_var = torch.from_numpy(seq).unsqueeze(0)
+            chars_var = torch.from_numpy(seq).unsqueeze(0).long()
             if self.use_cuda:
                 chars_var = chars_var.cuda()
-            mel_out, linear_out, alignments, stop_tokens = self.model.forward(chars_var)
+            mel_out, linear_out, alignments, stop_tokens = self.model.forward(
+                chars_var)
             linear_out = linear_out[0].data.cpu().numpy()
             wav = self.ap.inv_spectrogram(linear_out.T)
             # wav = wav[:self.ap.find_endpoint(wav)]

setup.py

@@ -25,7 +25,6 @@ else:
 
 
 class build_py(setuptools.command.build_py.build_py):
-
     def run(self):
         self.create_version_file()
         setuptools.command.build_py.build_py.run(self)
@@ -40,7 +39,6 @@ class build_py(setuptools.command.build_py.build_py):
 
 
 class develop(setuptools.command.develop.develop):
-
     def run(self):
         build_py.create_version_file()
         setuptools.command.develop.develop.run(self)
@@ -50,8 +48,11 @@ def create_readme_rst():
     global cwd
     try:
         subprocess.check_call(
-            ["pandoc", "--from=markdown", "--to=rst", "--output=README.rst",
-             "README.md"], cwd=cwd)
+            [
+                "pandoc", "--from=markdown", "--to=rst", "--output=README.rst",
+                "README.md"
+            ],
+            cwd=cwd)
         print("Generated README.rst from README.md using pandoc.")
     except subprocess.CalledProcessError:
         pass
@@ -59,30 +60,31 @@ def create_readme_rst():
         pass
 
 
-setup(name='TTS',
-      version=version,
-      url='https://github.com/mozilla/TTS',
-      description='Text to Speech with Deep Learning',
-      packages=find_packages(),
-      cmdclass={
-          'build_py': build_py,
-          'develop': develop,
-      },
-      install_requires=[
-          "numpy",
-          "scipy",
-          "librosa",
-          "torch >= 0.4.0",
-          "unidecode",
-          "tensorboardX",
-          "matplotlib",
-          "Pillow",
-          "flask",
-      ],
-      extras_require={
-          "bin": [
-              "tqdm",
-              "tensorboardX",
-              "requests",
-          ],
-      })
+setup(
+    name='TTS',
+    version=version,
+    url='https://github.com/mozilla/TTS',
+    description='Text to Speech with Deep Learning',
+    packages=find_packages(),
+    cmdclass={
+        'build_py': build_py,
+        'develop': develop,
+    },
+    setup_requires=["numpy==1.14.3"],
+    install_requires=[
+        "scipy==0.19.0",
+        "torch == 0.4.0",
+        "librosa==0.5.1",
+        "unidecode==0.4.20",
+        "tensorboardX",
+        "matplotlib==2.0.2",
+        "Pillow",
+        "flask",
+        "lws",
+    ],
+    extras_require={
+        "bin": [
+            "tqdm",
+            "requests",
+        ],
+    })

tests/generic_utils_text.py

@@ -8,19 +8,17 @@ OUT_PATH = '/tmp/test.pth.tar'
 
 
 class ModelSavingTests(unittest.TestCase):
-
     def save_checkpoint_test(self):
         # create a dummy model
         model = Prenet(128, out_features=[256, 128])
         model = T.nn.DataParallel(layer)
 
         # save the model
-        save_checkpoint(model, None, 100,
-                        OUTPATH, 1, 1)
+        save_checkpoint(model, None, 100, OUTPATH, 1, 1)
 
         # load the model to CPU
-        model_dict = torch.load(MODEL_PATH, map_location=lambda storage,
-                                loc: storage)
+        model_dict = torch.load(
+            MODEL_PATH, map_location=lambda storage, loc: storage)
         model.load_state_dict(model_dict['model'])
 
     def save_best_model_test(self):
@@ -29,11 +27,9 @@ class ModelSavingTests(unittest.TestCase):
         model = T.nn.DataParallel(layer)
 
         # save the model
-        best_loss = save_best_model(model, None, 0,
-                                    100, OUT_PATH,
-                                    10, 1)
+        best_loss = save_best_model(model, None, 0, 100, OUT_PATH, 10, 1)
 
         # load the model to CPU
-        model_dict = torch.load(MODEL_PATH, map_location=lambda storage,
-                                loc: storage)
+        model_dict = torch.load(
+            MODEL_PATH, map_location=lambda storage, loc: storage)
         model.load_state_dict(model_dict['model'])

tests/layers_tests.py

@@ -2,11 +2,11 @@ import unittest
 import torch as T
 
 from TTS.layers.tacotron import Prenet, CBHG, Decoder, Encoder
-from TTS.layers.losses import L1LossMasked, _sequence_mask
+from TTS.layers.losses import L1LossMasked
+from TTS.utils.generic_utils import sequence_mask
 
 
 class PrenetTests(unittest.TestCase):
-
     def test_in_out(self):
         layer = Prenet(128, out_features=[256, 128])
         dummy_input = T.rand(4, 128)
@@ -18,7 +18,6 @@ class PrenetTests(unittest.TestCase):
 
 
 class CBHGTests(unittest.TestCase):
-
     def test_in_out(self):
         layer = CBHG(128, K=6, projections=[128, 128], num_highways=2)
         dummy_input = T.rand(4, 8, 128)
@@ -31,7 +30,6 @@ class CBHGTests(unittest.TestCase):
 
 
 class DecoderTests(unittest.TestCase):
-
     def test_in_out(self):
         layer = Decoder(in_features=256, memory_dim=80, r=2)
         dummy_input = T.rand(4, 8, 256)
@@ -48,7 +46,6 @@ class DecoderTests(unittest.TestCase):
 
 
 class EncoderTests(unittest.TestCase):
-
     def test_in_out(self):
         layer = Encoder(128)
         dummy_input = T.rand(4, 8, 128)
@@ -62,7 +59,6 @@ class EncoderTests(unittest.TestCase):
 
 
 class L1LossMaskedTests(unittest.TestCase):
-
     def test_in_out(self):
         layer = L1LossMasked()
         dummy_input = T.ones(4, 8, 128).float()
@@ -79,7 +75,7 @@ class L1LossMaskedTests(unittest.TestCase):
         dummy_input = T.ones(4, 8, 128).float()
         dummy_target = T.zeros(4, 8, 128).float()
         dummy_length = (T.arange(5, 9)).long()
-        mask = ((_sequence_mask(dummy_length).float() - 1.0)
-                * 100.0).unsqueeze(2)
+        mask = (
+            (sequence_mask(dummy_length).float() - 1.0) * 100.0).unsqueeze(2)
         output = layer(dummy_input + mask, dummy_target, dummy_length)
         assert output.item() == 1.0, "1.0 vs {}".format(output.data[0])

tests/loader_tests.py

@@ -4,37 +4,46 @@ import numpy as np
 
 from torch.utils.data import DataLoader
 from TTS.utils.generic_utils import load_config
-from TTS.datasets.LJSpeech import LJSpeechDataset
+from TTS.utils.audio import AudioProcessor
+from TTS.datasets import LJSpeech, Kusal
 
 file_path = os.path.dirname(os.path.realpath(__file__))
 c = load_config(os.path.join(file_path, 'test_config.json'))
 
 
 class TestLJSpeechDataset(unittest.TestCase):
-
     def __init__(self, *args, **kwargs):
         super(TestLJSpeechDataset, self).__init__(*args, **kwargs)
         self.max_loader_iter = 4
+        self.ap = AudioProcessor(
+            sample_rate=c.sample_rate,
+            num_mels=c.num_mels,
+            min_level_db=c.min_level_db,
+            frame_shift_ms=c.frame_shift_ms,
+            frame_length_ms=c.frame_length_ms,
+            ref_level_db=c.ref_level_db,
+            num_freq=c.num_freq,
+            power=c.power,
+            preemphasis=c.preemphasis,
+            min_mel_freq=c.min_mel_freq,
+            max_mel_freq=c.max_mel_freq)
 
     def test_loader(self):
-        dataset = LJSpeechDataset(os.path.join(c.data_path_LJSpeech, 'metadata.csv'),
-                                  os.path.join(c.data_path_LJSpeech, 'wavs'),
-                                  c.r,
-                                  c.sample_rate,
-                                  c.text_cleaner,
-                                  c.num_mels,
-                                  c.min_level_db,
-                                  c.frame_shift_ms,
-                                  c.frame_length_ms,
-                                  c.preemphasis,
-                                  c.ref_level_db,
-                                  c.num_freq,
-                                  c.power
-                                  )
+        dataset = LJSpeech.MyDataset(
+            os.path.join(c.data_path_LJSpeech),
+            os.path.join(c.data_path_LJSpeech, 'metadata.csv'),
+            c.r,
+            c.text_cleaner,
+            ap=self.ap,
+            min_seq_len=c.min_seq_len)
 
-        dataloader = DataLoader(dataset, batch_size=2,
-                                shuffle=True, collate_fn=dataset.collate_fn,
-                                drop_last=True, num_workers=c.num_loader_workers)
+        dataloader = DataLoader(
+            dataset,
+            batch_size=2,
+            shuffle=True,
+            collate_fn=dataset.collate_fn,
+            drop_last=True,
+            num_workers=c.num_loader_workers)
 
         for i, data in enumerate(dataloader):
             if i == self.max_loader_iter:
@@ -57,25 +66,22 @@ class TestLJSpeechDataset(unittest.TestCase):
             assert mel_input.shape[2] == c.num_mels
 
     def test_padding(self):
-        dataset = LJSpeechDataset(os.path.join(c.data_path_LJSpeech, 'metadata.csv'),
-                                  os.path.join(c.data_path_LJSpeech, 'wavs'),
-                                  1,
-                                  c.sample_rate,
-                                  c.text_cleaner,
-                                  c.num_mels,
-                                  c.min_level_db,
-                                  c.frame_shift_ms,
-                                  c.frame_length_ms,
-                                  c.preemphasis,
-                                  c.ref_level_db,
-                                  c.num_freq,
-                                  c.power
-                                  )
+        dataset = LJSpeech.MyDataset(
+            os.path.join(c.data_path_LJSpeech),
+            os.path.join(c.data_path_LJSpeech, 'metadata.csv'),
+            1,
+            c.text_cleaner,
+            ap=self.ap,
+            min_seq_len=c.min_seq_len)
 
         # Test for batch size 1
-        dataloader = DataLoader(dataset, batch_size=1,
-                                shuffle=False, collate_fn=dataset.collate_fn,
-                                drop_last=True, num_workers=c.num_loader_workers)
+        dataloader = DataLoader(
+            dataset,
+            batch_size=1,
+            shuffle=False,
+            collate_fn=dataset.collate_fn,
+            drop_last=True,
+            num_workers=c.num_loader_workers)
 
         for i, data in enumerate(dataloader):
             if i == self.max_loader_iter:
@@ -100,9 +106,13 @@ class TestLJSpeechDataset(unittest.TestCase):
             assert mel_lengths[0] == mel_input[0].shape[0]
 
         # Test for batch size 2
-        dataloader = DataLoader(dataset, batch_size=2,
-                                shuffle=False, collate_fn=dataset.collate_fn,
-                                drop_last=False, num_workers=c.num_loader_workers)
+        dataloader = DataLoader(
+            dataset,
+            batch_size=2,
+            shuffle=False,
+            collate_fn=dataset.collate_fn,
+            drop_last=False,
+            num_workers=c.num_loader_workers)
 
         for i, data in enumerate(dataloader):
             if i == self.max_loader_iter:
@@ -132,16 +142,157 @@ class TestLJSpeechDataset(unittest.TestCase):
             assert mel_lengths[idx] == mel_input[idx].shape[0]
 
             # check the second itme in the batch
-            assert mel_input[1-idx, -1].sum() == 0
-            assert linear_input[1-idx, -1].sum() == 0
-            assert stop_target[1-idx, -1] == 1
+            assert mel_input[1 - idx, -1].sum() == 0
+            assert linear_input[1 - idx, -1].sum() == 0
+            assert stop_target[1 - idx, -1] == 1
             assert len(mel_lengths.shape) == 1
 
             # check batch conditions
             assert (mel_input * stop_target.unsqueeze(2)).sum() == 0
             assert (linear_input * stop_target.unsqueeze(2)).sum() == 0
 
-            
+
+class TestKusalDataset(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(TestKusalDataset, self).__init__(*args, **kwargs)
+        self.max_loader_iter = 4
+        self.ap = AudioProcessor(
+            sample_rate=c.sample_rate,
+            num_mels=c.num_mels,
+            min_level_db=c.min_level_db,
+            frame_shift_ms=c.frame_shift_ms,
+            frame_length_ms=c.frame_length_ms,
+            ref_level_db=c.ref_level_db,
+            num_freq=c.num_freq,
+            power=c.power,
+            preemphasis=c.preemphasis,
+            min_mel_freq=c.min_mel_freq,
+            max_mel_freq=c.max_mel_freq)
+
+    def test_loader(self):
+        dataset = Kusal.MyDataset(
+            os.path.join(c.data_path_Kusal),
+            os.path.join(c.data_path_Kusal, 'prompts.txt'),
+            c.r,
+            c.text_cleaner,
+            ap=self.ap,
+            min_seq_len=c.min_seq_len)
+
+        dataloader = DataLoader(
+            dataset,
+            batch_size=2,
+            shuffle=True,
+            collate_fn=dataset.collate_fn,
+            drop_last=True,
+            num_workers=c.num_loader_workers)
+
+        for i, data in enumerate(dataloader):
+            if i == self.max_loader_iter:
+                break
+            text_input = data[0]
+            text_lengths = data[1]
+            linear_input = data[2]
+            mel_input = data[3]
+            mel_lengths = data[4]
+            stop_target = data[5]
+            item_idx = data[6]
+
+            neg_values = text_input[text_input < 0]
+            check_count = len(neg_values)
+            assert check_count == 0, \
+                " !! Negative values in text_input: {}".format(check_count)
+            # TODO: more assertion here
+            assert linear_input.shape[0] == c.batch_size
+            assert mel_input.shape[0] == c.batch_size
+            assert mel_input.shape[2] == c.num_mels
+
+    def test_padding(self):
+        dataset = Kusal.MyDataset(
+            os.path.join(c.data_path_Kusal),
+            os.path.join(c.data_path_Kusal, 'prompts.txt'),
+            1,
+            c.text_cleaner,
+            ap=self.ap,
+            min_seq_len=c.min_seq_len)
+
+        # Test for batch size 1
+        dataloader = DataLoader(
+            dataset,
+            batch_size=1,
+            shuffle=False,
+            collate_fn=dataset.collate_fn,
+            drop_last=True,
+            num_workers=c.num_loader_workers)
+
+        for i, data in enumerate(dataloader):
+            if i == self.max_loader_iter:
+                break
+            text_input = data[0]
+            text_lengths = data[1]
+            linear_input = data[2]
+            mel_input = data[3]
+            mel_lengths = data[4]
+            stop_target = data[5]
+            item_idx = data[6]
+
+            # check the last time step to be zero padded
+            assert mel_input[0, -1].sum() == 0
+            # assert mel_input[0, -2].sum() != 0
+            assert linear_input[0, -1].sum() == 0
+            # assert linear_input[0, -2].sum() != 0
+            assert stop_target[0, -1] == 1
+            assert stop_target[0, -2] == 0
+            assert stop_target.sum() == 1
+            assert len(mel_lengths.shape) == 1
+            assert mel_lengths[0] == mel_input[0].shape[0]
+
+        # Test for batch size 2
+        dataloader = DataLoader(
+            dataset,
+            batch_size=2,
+            shuffle=False,
+            collate_fn=dataset.collate_fn,
+            drop_last=False,
+            num_workers=c.num_loader_workers)
+
+        for i, data in enumerate(dataloader):
+            if i == self.max_loader_iter:
+                break
+            text_input = data[0]
+            text_lengths = data[1]
+            linear_input = data[2]
+            mel_input = data[3]
+            mel_lengths = data[4]
+            stop_target = data[5]
+            item_idx = data[6]
+
+            if mel_lengths[0] > mel_lengths[1]:
+                idx = 0
+            else:
+                idx = 1
+
+            # check the first item in the batch
+            assert mel_input[idx, -1].sum() == 0
+            assert mel_input[idx, -2].sum() != 0, mel_input
+            assert linear_input[idx, -1].sum() == 0
+            assert linear_input[idx, -2].sum() != 0
+            assert stop_target[idx, -1] == 1
+            assert stop_target[idx, -2] == 0
+            assert stop_target[idx].sum() == 1
+            assert len(mel_lengths.shape) == 1
+            assert mel_lengths[idx] == mel_input[idx].shape[0]
+
+            # check the second itme in the batch
+            assert mel_input[1 - idx, -1].sum() == 0
+            assert linear_input[1 - idx, -1].sum() == 0
+            assert stop_target[1 - idx, -1] == 1
+            assert len(mel_lengths.shape) == 1
+
+            # check batch conditions
+            assert (mel_input * stop_target.unsqueeze(2)).sum() == 0
+            assert (linear_input * stop_target.unsqueeze(2)).sum() == 0
+
+
 # class TestTWEBDataset(unittest.TestCase):
 
 #     def __init__(self, *args, **kwargs):
@@ -212,7 +363,7 @@ class TestLJSpeechDataset(unittest.TestCase):
 #         for i, data in enumerate(dataloader):
 #             if i == self.max_loader_iter:
 #                 break
-                
+
 #             text_input = data[0]
 #             text_lengths = data[1]
 #             linear_input = data[2]
@@ -272,4 +423,4 @@ class TestLJSpeechDataset(unittest.TestCase):
 
 #             # check batch conditions
 #             assert (mel_input * stop_target.unsqueeze(2)).sum() == 0
-#             assert (linear_input * stop_target.unsqueeze(2)).sum() == 0
+#             assert (linear_input * stop_target.unsqueeze(2)).sum() == 0

tests/tacotron_tests.py

@@ -19,50 +19,51 @@ c = load_config(os.path.join(file_path, 'test_config.json'))
 
 
 class TacotronTrainTest(unittest.TestCase):
-    
     def test_train_step(self):
         input = torch.randint(0, 24, (8, 128)).long().to(device)
         mel_spec = torch.rand(8, 30, c.num_mels).to(device)
         linear_spec = torch.rand(8, 30, c.num_freq).to(device)
-        mel_lengths = torch.randint(20, 30, (8,)).long().to(device)
+        mel_lengths = torch.randint(20, 30, (8, )).long().to(device)
         stop_targets = torch.zeros(8, 30, 1).float().to(device)
-        
+
         for idx in mel_lengths:
             stop_targets[:, int(idx.item()):, 0] = 1.0
-            
-        stop_targets = stop_targets.view(input.shape[0], stop_targets.size(1) // c.r, -1)
+
+        stop_targets = stop_targets.view(input.shape[0],
+                                         stop_targets.size(1) // c.r, -1)
         stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float()
-        
+
         criterion = L1LossMasked().to(device)
         criterion_st = nn.BCELoss().to(device)
-        model = Tacotron(c.embedding_size,
-                         c.num_freq,
-                         c.num_mels,
+        model = Tacotron(c.embedding_size, c.num_freq, c.num_mels,
                          c.r).to(device)
         model.train()
         model_ref = copy.deepcopy(model)
         count = 0
-        for param, param_ref in zip(model.parameters(), model_ref.parameters()):
+        for param, param_ref in zip(model.parameters(),
+                                    model_ref.parameters()):
             assert (param - param_ref).sum() == 0, param
             count += 1
         optimizer = optim.Adam(model.parameters(), lr=c.lr)
         for i in range(5):
-            mel_out, linear_out, align, stop_tokens = model.forward(input, mel_spec)
+            mel_out, linear_out, align, stop_tokens = model.forward(
+                input, mel_spec)
             assert stop_tokens.data.max() <= 1.0
             assert stop_tokens.data.min() >= 0.0
             optimizer.zero_grad()
-            loss = criterion(mel_out, mel_spec, mel_lengths) 
+            loss = criterion(mel_out, mel_spec, mel_lengths)
             stop_loss = criterion_st(stop_tokens, stop_targets)
-            loss = loss + criterion(linear_out, linear_spec, mel_lengths) + stop_loss
+            loss = loss + criterion(linear_out, linear_spec,
+                                    mel_lengths) + stop_loss
             loss.backward()
             optimizer.step()
         # check parameter changes
         count = 0
-        for param, param_ref in zip(model.parameters(), model_ref.parameters()):
-            # ignore pre-higway layer since it works conditional 
-            if count not in [145, 59]:
-                assert (param != param_ref).any(), "param {} with shape {} not updated!! \n{}\n{}".format(count, param.shape, param, param_ref)
-            count += 1
-            
-        
-        
+        for param, param_ref in zip(model.parameters(),
+                                    model_ref.parameters()):
+            # ignore pre-higway layer since it works conditional
+            # if count not in [145, 59]:
+            assert (param != param_ref).any(
+            ), "param {} with shape {} not updated!! \n{}\n{}".format(
+                count, param.shape, param, param_ref)
+            count += 1

tests/test_config.json

@@ -9,6 +9,8 @@
     "ref_level_db": 20,
     "hidden_size": 128,
     "embedding_size": 256,
+    "min_mel_freq": null,
+    "max_mel_freq": null,
     "text_cleaner": "english_cleaners",
 
     "epochs": 2000,
@@ -27,8 +29,9 @@
     "num_loader_workers": 4,
 
     "save_step": 200,
-    "data_path_LJSpeech": "/data/shared/KeithIto/LJSpeech-1.0",
-    "data_path_TWEB": "/data/shared/BibleSpeech",
+    "data_path_LJSpeech": "C:/Users/erogol/Data/LJSpeech-1.1",
+    "data_path_Kusal": "C:/Users/erogol/Data/Kusal",
     "output_path": "result",
+    "min_seq_len": 0,
     "log_dir": "/home/erogol/projects/TTS/logs/"
     }

train.py

@@ -1,67 +1,44 @@
 import os
 import sys
 import time
-import datetime
 import shutil
 import torch
-import signal
 import argparse
 import importlib
-import pickle
 import traceback
 import numpy as np
 
 import torch.nn as nn
 from torch import optim
-from torch import onnx
 from torch.utils.data import DataLoader
-from torch.optim.lr_scheduler import ReduceLROnPlateau
 from tensorboardX import SummaryWriter
 
-from utils.generic_utils import (Progbar, remove_experiment_folder,
-                                 create_experiment_folder, save_checkpoint,
-                                 save_best_model, load_config, lr_decay,
-                                 count_parameters, check_update, get_commit_hash)
-from utils.model import get_param_size
+from utils.generic_utils import (
+    synthesis, remove_experiment_folder, create_experiment_folder,
+    save_checkpoint, save_best_model, load_config, lr_decay, count_parameters,
+    check_update, get_commit_hash, sequence_mask)
 from utils.visual import plot_alignment, plot_spectrogram
-from datasets.LJSpeech import LJSpeechDataset
 from models.tacotron import Tacotron
 from layers.losses import L1LossMasked
+from utils.audio import AudioProcessor
+
 
 torch.manual_seed(1)
+torch.set_num_threads(4)
 use_cuda = torch.cuda.is_available()
 
-parser = argparse.ArgumentParser()
-parser.add_argument('--restore_path', type=str,
-                    help='Folder path to checkpoints', default=0)
-parser.add_argument('--config_path', type=str,
-                    help='path to config file for training',)
-parser.add_argument('--debug', type=bool, default=False,
-                    help='do not ask for git has before run.')
-args = parser.parse_args()
 
-# setup output paths and read configs
-c = load_config(args.config_path)
-_ = os.path.dirname(os.path.realpath(__file__))
-OUT_PATH = os.path.join(_, c.output_path)
-OUT_PATH = create_experiment_folder(OUT_PATH, c.model_name, args.debug)
-CHECKPOINT_PATH = os.path.join(OUT_PATH, 'checkpoints')
-shutil.copyfile(args.config_path, os.path.join(OUT_PATH, 'config.json'))
-
-# setup tensorboard
-LOG_DIR = OUT_PATH
-tb = SummaryWriter(LOG_DIR)
-
-
-def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st, epoch):
+def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st,
+          scheduler, ap, epoch):
     model = model.train()
     epoch_time = 0
     avg_linear_loss = 0
     avg_mel_loss = 0
     avg_stop_loss = 0
-    print(" | > Epoch {}/{}".format(epoch, c.epochs))
-    progbar = Progbar(len(data_loader.dataset) / c.batch_size)
+    avg_step_time = 0
+    print(" | > Epoch {}/{}".format(epoch, c.epochs), flush=True)
     n_priority_freq = int(3000 / (c.sample_rate * 0.5) * c.num_freq)
+    batch_n_iter = int(len(data_loader.dataset) / c.batch_size)
     for num_iter, data in enumerate(data_loader):
         start_time = time.time()
 
@@ -74,41 +51,38 @@ def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st,
         stop_targets = data[5]
 
         # set stop targets view, we predict a single stop token per r frames prediction
-        stop_targets = stop_targets.view(text_input.shape[0], stop_targets.size(1) // c.r, -1)
+        stop_targets = stop_targets.view(text_input.shape[0],
+                                         stop_targets.size(1) // c.r, -1)
         stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float()
 
         current_step = num_iter + args.restore_step + \
             epoch * len(data_loader) + 1
 
         # setup lr
-        current_lr = lr_decay(c.lr, current_step, c.warmup_steps)
-        current_lr_st = lr_decay(c.lr, current_step, c.warmup_steps)
-
-        for params_group in optimizer.param_groups:
-            params_group['lr'] = current_lr
-
-        for params_group in optimizer_st.param_groups:
-            params_group['lr'] = current_lr_st
-
+        scheduler.step()
         optimizer.zero_grad()
         optimizer_st.zero_grad()
 
         # dispatch data to GPU
         if use_cuda:
             text_input = text_input.cuda()
+            text_lengths = text_lengths.cuda()
             mel_input = mel_input.cuda()
             mel_lengths = mel_lengths.cuda()
             linear_input = linear_input.cuda()
             stop_targets = stop_targets.cuda()
 
+        # compute mask for padding
+        mask = sequence_mask(text_lengths)
+
         # forward pass
-        mel_output, linear_output, alignments, stop_tokens =\
-            model.forward(text_input, mel_input)
+        mel_output, linear_output, alignments, stop_tokens = torch.nn.parallel.data_parallel(
+            model, (text_input, mel_input, mask))
 
         # loss computation
         stop_loss = criterion_st(stop_tokens, stop_targets)
         mel_loss = criterion(mel_output, mel_input, mel_lengths)
-        linear_loss = 0.5 * criterion(linear_output, linear_input, mel_lengths) \
+        linear_loss = 0.5 * criterion(linear_output, linear_input, mel_lengths)\
             + 0.5 * criterion(linear_output[:, :, :n_priority_freq],
                               linear_input[:, :, :n_priority_freq],
                               mel_lengths)
@@ -116,16 +90,16 @@ def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st,
 
         # backpass and check the grad norm for spec losses
         loss.backward(retain_graph=True)
-        grad_norm, skip_flag = check_update(model, 0.5, 100)
+        grad_norm, skip_flag = check_update(model, 1)
         if skip_flag:
             optimizer.zero_grad()
-            print(" | > Iteration skipped!!")
+            print(" | > Iteration skipped!!", flush=True)
             continue
         optimizer.step()
 
         # backpass and check the grad norm for stop loss
         stop_loss.backward()
-        grad_norm_st, skip_flag = check_update(model.module.decoder.stopnet, 0.5, 100)
+        grad_norm_st, skip_flag = check_update(model.decoder.stopnet, 0.5)
         if skip_flag:
             optimizer_st.zero_grad()
             print(" | | > Iteration skipped fro stopnet!!")
@@ -135,29 +109,20 @@ def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st,
         step_time = time.time() - start_time
         epoch_time += step_time
 
-        # update
-        # progbar.update(num_iter+1, values=[('total_loss', loss.item()),
-        #                                    ('linear_loss', linear_loss.item()),
-        #                                    ('mel_loss', mel_loss.item()),
-        #                                    ('stop_loss', stop_loss.item()),
-        #                                    ('grad_norm', grad_norm.item()),
-        #                                    ('grad_norm_st', grad_norm_st.item())])
-
         if current_step % c.print_step == 0:
-            print(" | | > Step:{}  GlobalStep:{}  TotalLoss:{:.5f}  LinearLoss:{:.5f}  "\
-                  "MelLoss:{:.5f}  StopLoss:{:.5f}  GradNorm:{:.5f}  "\
-                  "GradNormST:{:.5f}  StepTime:{:.2f}".format(num_iter, current_step,
-                                             loss.item(),
-                                             linear_loss.item(),
-                                             mel_loss.item(),
-                                             stop_loss.item(),
-                                             grad_norm.item(),
-                                             grad_norm_st.item(),
-                                             step_time))
+            print(
+                " | | > Step:{}/{}  GlobalStep:{}  TotalLoss:{:.5f}  LinearLoss:{:.5f}  "
+                "MelLoss:{:.5f}  StopLoss:{:.5f}  GradNorm:{:.5f}  "
+                "GradNormST:{:.5f}  StepTime:{:.2f}".format(
+                    num_iter, batch_n_iter, current_step, loss.item(),
+                    linear_loss.item(), mel_loss.item(), stop_loss.item(),
+                    grad_norm.item(), grad_norm_st.item(), step_time),
+                flush=True)
 
         avg_linear_loss += linear_loss.item()
         avg_mel_loss += mel_loss.item()
         avg_stop_loss += stop_loss.item()
+        avg_step_time += step_time
 
         # Plot Training Iter Stats
         tb.add_scalar('TrainIterLoss/TotalLoss', loss.item(), current_step)
@@ -173,50 +138,51 @@ def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st,
         if current_step % c.save_step == 0:
             if c.checkpoint:
                 # save model
-                save_checkpoint(model, optimizer, linear_loss.item(),
-                                OUT_PATH, current_step, epoch)
+                save_checkpoint(model, optimizer, optimizer_st,
+                                linear_loss.item(), OUT_PATH, current_step,
+                                epoch)
 
             # Diagnostic visualizations
             const_spec = linear_output[0].data.cpu().numpy()
             gt_spec = linear_input[0].data.cpu().numpy()
 
-            const_spec = plot_spectrogram(const_spec, data_loader.dataset.ap)
-            gt_spec = plot_spectrogram(gt_spec, data_loader.dataset.ap)
-            tb.add_image('Visual/Reconstruction', const_spec, current_step)
-            tb.add_image('Visual/GroundTruth', gt_spec, current_step)
+            const_spec = plot_spectrogram(const_spec, ap)
+            gt_spec = plot_spectrogram(gt_spec, ap)
+            tb.add_figure('Visual/Reconstruction', const_spec, current_step)
+            tb.add_figure('Visual/GroundTruth', gt_spec, current_step)
 
             align_img = alignments[0].data.cpu().numpy()
             align_img = plot_alignment(align_img)
-            tb.add_image('Visual/Alignment', align_img, current_step)
+            tb.add_figure('Visual/Alignment', align_img, current_step)
 
             # Sample audio
             audio_signal = linear_output[0].data.cpu().numpy()
-            data_loader.dataset.ap.griffin_lim_iters = 60
-            audio_signal = data_loader.dataset.ap.inv_spectrogram(
-                audio_signal.T)
+            ap.griffin_lim_iters = 60
+            audio_signal = ap.inv_spectrogram(audio_signal.T)
             try:
-                tb.add_audio('SampleAudio', audio_signal, current_step,
-                             sample_rate=c.sample_rate)
+                tb.add_audio(
+                    'SampleAudio',
+                    audio_signal,
+                    current_step,
+                    sample_rate=c.sample_rate)
             except:
-                # print("\n > Error at audio signal on TB!!")
-                # print(audio_signal.max())
-                # print(audio_signal.min())
                 pass
 
     avg_linear_loss /= (num_iter + 1)
     avg_mel_loss /= (num_iter + 1)
     avg_stop_loss /= (num_iter + 1)
     avg_total_loss = avg_mel_loss + avg_linear_loss + avg_stop_loss
+    avg_step_time /= (num_iter + 1)
 
     # print epoch stats
-    print(" | | > EPOCH END -- GlobalStep:{}  AvgTotalLoss:{:.5f}  "\
-          "AvgLinearLoss:{:.5f}  AvgMelLoss:{:.5f}  "\
-          "AvgStopLoss:{:.5f}  EpochTime:{:.2f}".format(current_step,
-                                                       avg_total_loss,
-                                                       avg_linear_loss,
-                                                       avg_mel_loss,
-                                                       avg_stop_loss,
-                                                       epoch_time))
+    print(
+        " | | > EPOCH END -- GlobalStep:{}  AvgTotalLoss:{:.5f}  "
+        "AvgLinearLoss:{:.5f}  AvgMelLoss:{:.5f}  "
+        "AvgStopLoss:{:.5f}  EpochTime:{:.2f}  "
+        "AvgStepTime:{:.2f}".format(current_step, avg_total_loss,
+                                    avg_linear_loss, avg_mel_loss,
+                                    avg_stop_loss, epoch_time, avg_step_time),
+        flush=True)
 
     # Plot Training Epoch Stats
     tb.add_scalar('TrainEpochLoss/TotalLoss', avg_total_loss, current_step)
@@ -229,161 +195,204 @@ def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st,
     return avg_linear_loss, current_step
 
 
-def evaluate(model, criterion, criterion_st, data_loader, current_step):
+def evaluate(model, criterion, criterion_st, data_loader, ap, current_step):
     model = model.eval()
     epoch_time = 0
     avg_linear_loss = 0
     avg_mel_loss = 0
     avg_stop_loss = 0
     print(" | > Validation")
-    # progbar = Progbar(len(data_loader.dataset) / c.batch_size)
+    test_sentences = [
+        "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
+        "Be a voice, not an echo.",
+        "I'm sorry Dave. I'm afraid I can't do that.",
+        "This cake is great. It's so delicious and moist."
+    ]
     n_priority_freq = int(3000 / (c.sample_rate * 0.5) * c.num_freq)
     with torch.no_grad():
-        for num_iter, data in enumerate(data_loader):
-            start_time = time.time()
+        if data_loader is not None:
+            for num_iter, data in enumerate(data_loader):
+                start_time = time.time()
 
-            # setup input data
-            text_input = data[0]
-            text_lengths = data[1]
-            linear_input = data[2]
-            mel_input = data[3]
-            mel_lengths = data[4]
-            stop_targets = data[5]
+                # setup input data
+                text_input = data[0]
+                text_lengths = data[1]
+                linear_input = data[2]
+                mel_input = data[3]
+                mel_lengths = data[4]
+                stop_targets = data[5]
 
-            # set stop targets view, we predict a single stop token per r frames prediction
-            stop_targets = stop_targets.view(text_input.shape[0], stop_targets.size(1) // c.r, -1)
-            stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float()
+                # set stop targets view, we predict a single stop token per r frames prediction
+                stop_targets = stop_targets.view(text_input.shape[0],
+                                                 stop_targets.size(1) // c.r,
+                                                 -1)
+                stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float()
 
-            # dispatch data to GPU
-            if use_cuda:
-                text_input = text_input.cuda()
-                mel_input = mel_input.cuda()
-                mel_lengths = mel_lengths.cuda()
-                linear_input = linear_input.cuda()
-                stop_targets = stop_targets.cuda()
+                # dispatch data to GPU
+                if use_cuda:
+                    text_input = text_input.cuda()
+                    mel_input = mel_input.cuda()
+                    mel_lengths = mel_lengths.cuda()
+                    linear_input = linear_input.cuda()
+                    stop_targets = stop_targets.cuda()
 
-            # forward pass
-            mel_output, linear_output, alignments, stop_tokens =\
-                model.forward(text_input, mel_input)
+                # forward pass
+                mel_output, linear_output, alignments, stop_tokens =\
+                    model.forward(text_input, mel_input)
 
-            # loss computation
-            stop_loss = criterion_st(stop_tokens, stop_targets)
-            mel_loss = criterion(mel_output, mel_input, mel_lengths)
-            linear_loss = 0.5 * criterion(linear_output, linear_input, mel_lengths) \
-                + 0.5 * criterion(linear_output[:, :, :n_priority_freq],
-                                  linear_input[:, :, :n_priority_freq],
-                                  mel_lengths)
-            loss = mel_loss + linear_loss + stop_loss
+                # loss computation
+                stop_loss = criterion_st(stop_tokens, stop_targets)
+                mel_loss = criterion(mel_output, mel_input, mel_lengths)
+                linear_loss = 0.5 * criterion(linear_output, linear_input, mel_lengths) \
+                    + 0.5 * criterion(linear_output[:, :, :n_priority_freq],
+                                    linear_input[:, :, :n_priority_freq],
+                                    mel_lengths)
+                loss = mel_loss + linear_loss + stop_loss
 
-            step_time = time.time() - start_time
-            epoch_time += step_time
+                step_time = time.time() - start_time
+                epoch_time += step_time
 
-            # update
-            # progbar.update(num_iter+1, values=[('total_loss', loss.item()),
-            #                                    ('linear_loss', linear_loss.item()),
-            #                                    ('mel_loss', mel_loss.item()),
-            #                                    ('stop_loss', stop_loss.item())])
-            if num_iter % c.print_step == 0:
-                print(" | | > TotalLoss: {:.5f}   LinearLoss: {:.5f}   MelLoss:{:.5f}  "\
-                      "StopLoss: {:.5f}  ".format(loss.item(),
-                                                          linear_loss.item(),
-                                                          mel_loss.item(),
-                                                          stop_loss.item()))
+                if num_iter % c.print_step == 0:
+                    print(
+                        " | | > TotalLoss: {:.5f}   LinearLoss: {:.5f}   MelLoss:{:.5f}  "
+                        "StopLoss: {:.5f}  ".format(loss.item(),
+                                                    linear_loss.item(),
+                                                    mel_loss.item(),
+                                                    stop_loss.item()),
+                        flush=True)
 
-            avg_linear_loss += linear_loss.item()
-            avg_mel_loss += mel_loss.item()
-            avg_stop_loss += stop_loss.item()
+                avg_linear_loss += linear_loss.item()
+                avg_mel_loss += mel_loss.item()
+                avg_stop_loss += stop_loss.item()
 
-    # Diagnostic visualizations
-    idx = np.random.randint(mel_input.shape[0])
-    const_spec = linear_output[idx].data.cpu().numpy()
-    gt_spec = linear_input[idx].data.cpu().numpy()
-    align_img = alignments[idx].data.cpu().numpy()
+            # Diagnostic visualizations
+            idx = np.random.randint(mel_input.shape[0])
+            const_spec = linear_output[idx].data.cpu().numpy()
+            gt_spec = linear_input[idx].data.cpu().numpy()
+            align_img = alignments[idx].data.cpu().numpy()
 
-    const_spec = plot_spectrogram(const_spec, data_loader.dataset.ap)
-    gt_spec = plot_spectrogram(gt_spec, data_loader.dataset.ap)
-    align_img = plot_alignment(align_img)
+            const_spec = plot_spectrogram(const_spec, ap)
+            gt_spec = plot_spectrogram(gt_spec, ap)
+            align_img = plot_alignment(align_img)
 
-    tb.add_image('ValVisual/Reconstruction', const_spec, current_step)
-    tb.add_image('ValVisual/GroundTruth', gt_spec, current_step)
-    tb.add_image('ValVisual/ValidationAlignment', align_img, current_step)
+            tb.add_figure('ValVisual/Reconstruction', const_spec, current_step)
+            tb.add_figure('ValVisual/GroundTruth', gt_spec, current_step)
+            tb.add_figure('ValVisual/ValidationAlignment', align_img,
+                         current_step)
 
-    # Sample audio
-    audio_signal = linear_output[idx].data.cpu().numpy()
-    data_loader.dataset.ap.griffin_lim_iters = 60
-    audio_signal = data_loader.dataset.ap.inv_spectrogram(audio_signal.T)
-    try:
-        tb.add_audio('ValSampleAudio', audio_signal, current_step,
-                     sample_rate=c.sample_rate)
-    except:
-        # print(" | > Error at audio signal on TB!!")
-        # print(audio_signal.max())
-        # print(audio_signal.min())
-        pass
+            # Sample audio
+            audio_signal = linear_output[idx].data.cpu().numpy()
+            ap.griffin_lim_iters = 60
+            audio_signal = ap.inv_spectrogram(audio_signal.T)
+            try:
+                tb.add_audio(
+                    'ValSampleAudio',
+                    audio_signal,
+                    current_step,
+                    sample_rate=c.sample_rate)
+            except:
+                # sometimes audio signal is out of boundaries
+                pass
 
-    # compute average losses
-    avg_linear_loss /= (num_iter + 1)
-    avg_mel_loss /= (num_iter + 1)
-    avg_stop_loss /= (num_iter + 1)
-    avg_total_loss = avg_mel_loss + avg_linear_loss + avg_stop_loss
+            # compute average losses
+            avg_linear_loss /= (num_iter + 1)
+            avg_mel_loss /= (num_iter + 1)
+            avg_stop_loss /= (num_iter + 1)
+            avg_total_loss = avg_mel_loss + avg_linear_loss + avg_stop_loss
 
-    # Plot Learning Stats
-    tb.add_scalar('ValEpochLoss/TotalLoss', avg_total_loss, current_step)
-    tb.add_scalar('ValEpochLoss/LinearLoss', avg_linear_loss, current_step)
-    tb.add_scalar('ValEpochLoss/MelLoss', avg_mel_loss, current_step)
-    tb.add_scalar('ValEpochLoss/Stop_loss', avg_stop_loss, current_step)
+            # Plot Learning Stats
+            tb.add_scalar('ValEpochLoss/TotalLoss', avg_total_loss,
+                          current_step)
+            tb.add_scalar('ValEpochLoss/LinearLoss', avg_linear_loss,
+                          current_step)
+            tb.add_scalar('ValEpochLoss/MelLoss', avg_mel_loss, current_step)
+            tb.add_scalar('ValEpochLoss/Stop_loss', avg_stop_loss,
+                          current_step)
 
+    # test sentences
+    ap.griffin_lim_iters = 60
+    for idx, test_sentence in enumerate(test_sentences):
+        wav, linear_spec, alignments = synthesis(model, ap, test_sentence,
+                                                     use_cuda, c.text_cleaner)
+        try:
+            wav_name = 'TestSentences/{}'.format(idx)
+            tb.add_audio(
+                wav_name, wav, current_step, sample_rate=c.sample_rate)
+        except:
+            print(" !! Error as creating Test Sentence -", idx)
+            pass
+        align_img = alignments[0].data.cpu().numpy()
+        linear_spec = plot_spectrogram(linear_spec, ap)
+        align_img = plot_alignment(align_img)
+        tb.add_figure('TestSentences/{}_Spectrogram'.format(idx), linear_spec,
+                     current_step)
+        tb.add_figure('TestSentences/{}_Alignment'.format(idx), align_img,
+                     current_step)
     return avg_linear_loss
 
 
 def main(args):
+    dataset = importlib.import_module('datasets.' + c.dataset)
+    Dataset = getattr(dataset, 'MyDataset')
+    audio = importlib.import_module('utils.' + c.audio_processor)
+    AudioProcessor = getattr(audio, 'AudioProcessor')
+
+    print(" > LR scheduler: {} ", c.lr_scheduler)
+    try:
+        scheduler = importlib.import_module('torch.optim.lr_scheduler')
+        scheduler = getattr(scheduler, c.lr_scheduler)
+    except:
+        scheduler = importlib.import_module('utils.generic_utils')
+        scheduler = getattr(scheduler, c.lr_scheduler)
+
+    ap = AudioProcessor(
+        sample_rate=c.sample_rate,
+        num_mels=c.num_mels,
+        min_level_db=c.min_level_db,
+        frame_shift_ms=c.frame_shift_ms,
+        frame_length_ms=c.frame_length_ms,
+        ref_level_db=c.ref_level_db,
+        num_freq=c.num_freq,
+        power=c.power,
+        preemphasis=c.preemphasis,
+        min_mel_freq=c.min_mel_freq,
+        max_mel_freq=c.max_mel_freq)
 
     # Setup the dataset
-    train_dataset = LJSpeechDataset(os.path.join(c.data_path, 'metadata_train.csv'),
-                                    os.path.join(c.data_path, 'wavs'),
-                                    c.r,
-                                    c.sample_rate,
-                                    c.text_cleaner,
-                                    c.num_mels,
-                                    c.min_level_db,
-                                    c.frame_shift_ms,
-                                    c.frame_length_ms,
-                                    c.preemphasis,
-                                    c.ref_level_db,
-                                    c.num_freq,
-                                    c.power,
-                                    min_seq_len=c.min_seq_len
-                                    )
+    train_dataset = Dataset(
+        c.data_path,
+        c.meta_file_train,
+        c.r,
+        c.text_cleaner,
+        ap=ap,
+        min_seq_len=c.min_seq_len)
 
-    train_loader = DataLoader(train_dataset, batch_size=c.batch_size,
-                              shuffle=False, collate_fn=train_dataset.collate_fn,
-                              drop_last=False, num_workers=c.num_loader_workers,
-                              pin_memory=True)
+    train_loader = DataLoader(
+        train_dataset,
+        batch_size=c.batch_size,
+        shuffle=False,
+        collate_fn=train_dataset.collate_fn,
+        drop_last=False,
+        num_workers=c.num_loader_workers,
+        pin_memory=True)
 
-    val_dataset = LJSpeechDataset(os.path.join(c.data_path, 'metadata_val.csv'),
-                                  os.path.join(c.data_path, 'wavs'),
-                                  c.r,
-                                  c.sample_rate,
-                                  c.text_cleaner,
-                                  c.num_mels,
-                                  c.min_level_db,
-                                  c.frame_shift_ms,
-                                  c.frame_length_ms,
-                                  c.preemphasis,
-                                  c.ref_level_db,
-                                  c.num_freq,
-                                  c.power
-                                  )
+    if c.run_eval:
+        val_dataset = Dataset(
+            c.data_path, c.meta_file_val, c.r, c.text_cleaner, ap=ap)
 
-    val_loader = DataLoader(val_dataset, batch_size=c.eval_batch_size,
-                            shuffle=False, collate_fn=val_dataset.collate_fn,
-                            drop_last=False, num_workers=4,
-                            pin_memory=True)
+        val_loader = DataLoader(
+            val_dataset,
+            batch_size=c.eval_batch_size,
+            shuffle=False,
+            collate_fn=val_dataset.collate_fn,
+            drop_last=False,
+            num_workers=4,
+            pin_memory=True)
+    else:
+        val_loader = None
 
-    model = Tacotron(c.embedding_size,
-                     c.num_freq,
-                     c.num_mels,
-                     c.r)
+    model = Tacotron(c.embedding_size, ap.num_freq, c.num_mels, c.r)
+    print(" | > Num output units : {}".format(ap.num_freq), flush=True)
 
     optimizer = optim.Adam(model.parameters(), lr=c.lr)
     optimizer_st = optim.Adam(model.decoder.stopnet.parameters(), lr=c.lr)
@@ -394,29 +403,32 @@ def main(args):
     if args.restore_path:
         checkpoint = torch.load(args.restore_path)
         model.load_state_dict(checkpoint['model'])
-        optimizer = optim.Adam(model.parameters(), lr=c.lr)
+        if use_cuda:
+            model = model.cuda()
+            criterion.cuda()
+            criterion_st.cuda()
         optimizer.load_state_dict(checkpoint['optimizer'])
+        optimizer_st.load_state_dict(checkpoint['optimizer_st'])
         for state in optimizer.state.values():
             for k, v in state.items():
                 if torch.is_tensor(v):
                     state[k] = v.cuda()
-        print(" > Model restored from step %d" % checkpoint['step'])
+        print(
+            " > Model restored from step %d" % checkpoint['step'], flush=True)
         start_epoch = checkpoint['step'] // len(train_loader)
         best_loss = checkpoint['linear_loss']
-        start_epoch = 0
         args.restore_step = checkpoint['step']
-        optimizer_st = optim.Adam(model.decoder.stopnet.parameters(), lr=c.lr)
     else:
         args.restore_step = 0
-        print("\n > Starting a new training")
-
-    if use_cuda:
-        model = nn.DataParallel(model.cuda())
-        criterion.cuda()
-        criterion_st.cuda()
+        print("\n > Starting a new training", flush=True)
+        if use_cuda:
+            model = model.cuda()
+            criterion.cuda()
+            criterion_st.cuda()
 
+    scheduler = StepLR(optimizer, step_size=c.decay_step, gamma=c.lr_decay)
     num_params = count_parameters(model)
-    print(" | > Model has {} parameters".format(num_params))
+    print(" | > Model has {} parameters".format(num_params), flush=True)
 
     if not os.path.exists(CHECKPOINT_PATH):
         os.mkdir(CHECKPOINT_PATH)
@@ -425,16 +437,50 @@ def main(args):
         best_loss = float('inf')
 
     for epoch in range(0, c.epochs):
-        train_loss, current_step = train(
-            model, criterion, criterion_st, train_loader, optimizer, optimizer_st, epoch)
-        val_loss = evaluate(model, criterion, criterion_st, val_loader, current_step)
-        print(" | > Train Loss: {:.5f}   Validation Loss: {:.5f}".format(train_loss, val_loss))
-        best_loss = save_best_model(model, optimizer, val_loss,
-                                    best_loss, OUT_PATH,
-                                    current_step, epoch)
+        train_loss, current_step = train(model, criterion, criterion_st,
+                                         train_loader, optimizer, optimizer_st,
+                                         scheduler, ap, epoch)
+        val_loss = evaluate(model, criterion, criterion_st, val_loader, ap,
+                            current_step)
+        print(
+            " | > Train Loss: {:.5f}   Validation Loss: {:.5f}".format(
+                train_loss, val_loss),
+            flush=True)
+        best_loss = save_best_model(model, optimizer, train_loss, best_loss,
+                                    OUT_PATH, current_step, epoch)
 
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--restore_path',
+        type=str,
+        help='Folder path to checkpoints',
+        default=0)
+    parser.add_argument(
+        '--config_path',
+        type=str,
+        help='path to config file for training',
+    )
+    parser.add_argument(
+        '--debug',
+        type=bool,
+        default=False,
+        help='do not ask for git has before run.')
+    args = parser.parse_args()
+
+    # setup output paths and read configs
+    c = load_config(args.config_path)
+    _ = os.path.dirname(os.path.realpath(__file__))
+    OUT_PATH = os.path.join(_, c.output_path)
+    OUT_PATH = create_experiment_folder(OUT_PATH, c.model_name, args.debug)
+    CHECKPOINT_PATH = os.path.join(OUT_PATH, 'checkpoints')
+    shutil.copyfile(args.config_path, os.path.join(OUT_PATH, 'config.json'))
+
+    # setup tensorboard
+    LOG_DIR = OUT_PATH
+    tb = SummaryWriter(LOG_DIR)
+
     try:
         main(args)
     except KeyboardInterrupt:

utils/audio.py

@@ -1,122 +1,152 @@
-import os
-import librosa
-import pickle
-import copy
-import numpy as np
-from scipy import signal
-
-_mel_basis = None
-
-
-class AudioProcessor(object):
-
-    def __init__(self, sample_rate, num_mels, min_level_db, frame_shift_ms,
-                 frame_length_ms, preemphasis, ref_level_db, num_freq, power,
-                 griffin_lim_iters=None):
-        self.sample_rate = sample_rate
-        self.num_mels = num_mels
-        self.min_level_db = min_level_db
-        self.frame_shift_ms = frame_shift_ms
-        self.frame_length_ms = frame_length_ms
-        self.preemphasis = preemphasis
-        self.ref_level_db = ref_level_db
-        self.num_freq = num_freq
-        self.power = power
-        self.griffin_lim_iters = griffin_lim_iters
-
-    def save_wav(self, wav, path):
-        wav *= 32767 / max(0.01, np.max(np.abs(wav)))
-        librosa.output.write_wav(path, wav.astype(np.float), self.sample_rate, norm=True)
-
-    def _linear_to_mel(self, spectrogram):
-        global _mel_basis
-        if _mel_basis is None:
-            _mel_basis = self._build_mel_basis()
-        return np.dot(_mel_basis, spectrogram)
-
-    def _build_mel_basis(self, ):
-        n_fft = (self.num_freq - 1) * 2
-        return librosa.filters.mel(self.sample_rate, n_fft, n_mels=self.num_mels)
-
-    def _normalize(self, S):
-        return np.clip((S - self.min_level_db) / -self.min_level_db, 0, 1)
-
-    def _denormalize(self, S):
-        return (np.clip(S, 0, 1) * -self.min_level_db) + self.min_level_db
-
-    def _stft_parameters(self, ):
-        n_fft = (self.num_freq - 1) * 2
-        hop_length = int(self.frame_shift_ms / 1000.0 * self.sample_rate)
-        win_length = int(self.frame_length_ms / 1000.0 * self.sample_rate)
-        return n_fft, hop_length, win_length
-
-    def _amp_to_db(self, x):
-        return 20 * np.log10(np.maximum(1e-5, x))
-
-    def _db_to_amp(self, x):
-        return np.power(10.0, x * 0.05)
-
-    def apply_preemphasis(self, x):
-        return signal.lfilter([1, -self.preemphasis], [1], x)
-
-    def apply_inv_preemphasis(self, x):
-        return signal.lfilter([1], [1, -self.preemphasis], x)
-
-    def spectrogram(self, y):
-        D = self._stft(self.apply_preemphasis(y))
-        S = self._amp_to_db(np.abs(D)) - self.ref_level_db
-        return self._normalize(S)
-
-    def inv_spectrogram(self, spectrogram):
-        '''Converts spectrogram to waveform using librosa'''
-        S = self._denormalize(spectrogram)
-        S = self._db_to_amp(S + self.ref_level_db)  # Convert back to linear
-        # Reconstruct phase
-        return self.apply_inv_preemphasis(self._griffin_lim(S ** self.power))
-
-#     def _griffin_lim(self, S):
-#         '''librosa implementation of Griffin-Lim
-#         Based on https://github.com/librosa/librosa/issues/434
-#         '''
-#         angles = np.exp(2j * np.pi * np.random.rand(*S.shape))
-#         S_complex = np.abs(S).astype(np.complex)
-#         y = self._istft(S_complex * angles)
-#         for i in range(self.griffin_lim_iters):
-#             angles = np.exp(1j * np.angle(self._stft(y)))
-#             y = self._istft(S_complex * angles)
-#         return y
-    
-    def _griffin_lim(self, S):
-        '''Applies Griffin-Lim's raw.
-        '''
-        S_best = copy.deepcopy(S)
-        for i in range(self.griffin_lim_iters):
-            S_t = self._istft(S_best)
-            est = self._stft(S_t)
-            phase = est / np.maximum(1e-8, np.abs(est))
-            S_best = S * phase
-        S_t = self._istft(S_best)
-        y = np.real(S_t)
-        return y
-
-    def melspectrogram(self, y):
-        D = self._stft(self.apply_preemphasis(y))
-        S = self._amp_to_db(self._linear_to_mel(np.abs(D))) - self.ref_level_db
-        return self._normalize(S)
-
-    def _stft(self, y):
-        n_fft, hop_length, win_length = self._stft_parameters()
-        return librosa.stft(y=y, n_fft=n_fft, hop_length=hop_length, win_length=win_length)
-
-    def _istft(self, y):
-        _, hop_length, win_length = self._stft_parameters()
-        return librosa.istft(y, hop_length=hop_length, win_length=win_length, window='hann')
-
-    def find_endpoint(self, wav, threshold_db=-40, min_silence_sec=0.8):
-        window_length = int(self.sample_rate * min_silence_sec)
-        hop_length = int(window_length / 4)
-        threshold = self._db_to_amp(threshold_db)
-        for x in range(hop_length, len(wav) - window_length, hop_length):
-            if np.max(wav[x:x + window_length]) < threshold:
-                return x + hop_length
-        return len(wav)
+import os
+import librosa
+import pickle
+import copy
+import numpy as np
+from scipy import signal
+
+_mel_basis = None
+
+
+class AudioProcessor(object):
+    def __init__(self,
+                 sample_rate,
+                 num_mels,
+                 min_level_db,
+                 frame_shift_ms,
+                 frame_length_ms,
+                 ref_level_db,
+                 num_freq,
+                 power,
+                 preemphasis,
+                 min_mel_freq,
+                 max_mel_freq,
+                 griffin_lim_iters=None):
+
+        print(" > Setting up Audio Processor...")
+        self.sample_rate = sample_rate
+        self.num_mels = num_mels
+        self.min_level_db = min_level_db
+        self.frame_shift_ms = frame_shift_ms
+        self.frame_length_ms = frame_length_ms
+        self.ref_level_db = ref_level_db
+        self.num_freq = num_freq
+        self.power = power
+        self.preemphasis = preemphasis
+        self.min_mel_freq = min_mel_freq
+        self.max_mel_freq = max_mel_freq
+        self.griffin_lim_iters = griffin_lim_iters
+        self.n_fft, self.hop_length, self.win_length = self._stft_parameters()
+        if preemphasis == 0:
+            print(" | > Preemphasis is deactive.")
+
+    def save_wav(self, wav, path):
+        wav *= 32767 / max(0.01, np.max(np.abs(wav)))
+        librosa.output.write_wav(path, wav.astype(np.int16), self.sample_rate)
+
+    def _linear_to_mel(self, spectrogram):
+        global _mel_basis
+        if _mel_basis is None:
+            _mel_basis = self._build_mel_basis()
+        return np.dot(_mel_basis, spectrogram)
+
+    def _build_mel_basis(self, ):
+        n_fft = (self.num_freq - 1) * 2
+        return librosa.filters.mel(
+            self.sample_rate, n_fft, n_mels=self.num_mels)
+        #    fmin=self.min_mel_freq, fmax=self.max_mel_freq)
+
+    def _normalize(self, S):
+        return np.clip((S - self.min_level_db) / -self.min_level_db, 0, 1)
+
+    def _denormalize(self, S):
+        return (np.clip(S, 0, 1) * -self.min_level_db) + self.min_level_db
+
+    def _stft_parameters(self, ):
+        n_fft = (self.num_freq - 1) * 2
+        hop_length = int(self.frame_shift_ms / 1000.0 * self.sample_rate)
+        win_length = int(self.frame_length_ms / 1000.0 * self.sample_rate)
+        print(" | > fft size: {}, hop length: {}, win length: {}".format(
+            n_fft, hop_length, win_length))
+        return n_fft, hop_length, win_length
+
+    def _amp_to_db(self, x):
+        min_level = np.exp(self.min_level_db / 20 * np.log(10))
+        return 20 * np.log10(np.maximum(min_level, x))
+
+    def _db_to_amp(self, x):
+        return np.power(10.0, x * 0.05)
+
+    def apply_preemphasis(self, x):
+        if self.preemphasis == 0:
+            raise RuntimeError(" !! Preemphasis is applied with factor 0.0. ")
+        return signal.lfilter([1, -self.preemphasis], [1], x)
+
+    def apply_inv_preemphasis(self, x):
+        if self.preemphasis == 0:
+            raise RuntimeError(" !! Preemphasis is applied with factor 0.0. ")
+        return signal.lfilter([1], [1, -self.preemphasis], x)
+
+    def spectrogram(self, y):
+        if self.preemphasis != 0:
+            D = self._stft(self.apply_preemphasis(y))
+        else:
+            D = self._stft(y)
+        S = self._amp_to_db(np.abs(D)) - self.ref_level_db
+        return self._normalize(S)
+
+    def inv_spectrogram(self, spectrogram):
+        '''Converts spectrogram to waveform using librosa'''
+        S = self._denormalize(spectrogram)
+        S = self._db_to_amp(S + self.ref_level_db)  # Convert back to linear
+        # Reconstruct phase
+        if self.preemphasis != 0:
+            return self.apply_inv_preemphasis(self._griffin_lim(S**self.power))
+        else:
+            return self._griffin_lim(S**self.power)
+
+    # def _griffin_lim(self, S):
+    #     '''Applies Griffin-Lim's raw.
+    #     '''
+    #     S_best = copy.deepcopy(S)
+    #     for i in range(self.griffin_lim_iters):
+    #         S_t = self._istft(S_best)
+    #         est = self._stft(S_t)
+    #         phase = est / np.maximum(1e-8, np.abs(est))
+    #         S_best = S * phase
+    #     S_t = self._istft(S_best)
+    #     y = np.real(S_t)
+    #     return y
+
+    def _griffin_lim(self, S):
+        angles = np.exp(2j * np.pi * np.random.rand(*S.shape))
+        S_complex = np.abs(S).astype(np.complex)
+        y = self._istft(S_complex * angles)
+        for i in range(self.griffin_lim_iters):
+            angles = np.exp(1j * np.angle(self._stft(y)))
+            y = self._istft(S_complex * angles)
+        return y
+
+    def melspectrogram(self, y):
+        if self.preemphasis != 0:
+            D = self._stft(self.apply_preemphasis(y))
+        else:
+            D = self._stft(y)
+        S = self._amp_to_db(self._linear_to_mel(np.abs(D))) - self.ref_level_db
+        return self._normalize(S)
+
+    def _stft(self, y):
+        return librosa.stft(
+            y=y, n_fft=self.n_fft, hop_length=self.hop_length, win_length=self.win_length)
+
+    def _istft(self, y):
+        return librosa.istft(y, hop_length=self.hop_length, win_length=self.win_length)
+
+    def find_endpoint(self, wav, threshold_db=-40, min_silence_sec=0.8):
+        window_length = int(self.sample_rate * min_silence_sec)
+        hop_length = int(window_length / 4)
+        threshold = self._db_to_amp(threshold_db)
+        for x in range(hop_length, len(wav) - window_length, hop_length):
+            if np.max(wav[x:x + window_length]) < threshold:
+                return x + hop_length
+        return len(wav)

utils/audio_lws.py

@@ -0,0 +1,151 @@
+import os
+import sys
+import librosa
+import pickle
+import copy
+import numpy as np
+from scipy import signal
+import lws
+
+_mel_basis = None
+
+
+class AudioProcessor(object):
+    def __init__(
+            self,
+            sample_rate,
+            num_mels,
+            min_level_db,
+            frame_shift_ms,
+            frame_length_ms,
+            ref_level_db,
+            num_freq,
+            power,
+            preemphasis,
+            min_mel_freq,
+            max_mel_freq,
+            griffin_lim_iters=None,
+    ):
+        print(" > Setting up Audio Processor...")
+        self.sample_rate = sample_rate
+        self.num_mels = num_mels
+        self.min_level_db = min_level_db
+        self.frame_shift_ms = frame_shift_ms
+        self.frame_length_ms = frame_length_ms
+        self.ref_level_db = ref_level_db
+        self.num_freq = num_freq
+        self.power = power
+        self.min_mel_freq = min_mel_freq
+        self.max_mel_freq = max_mel_freq
+        self.griffin_lim_iters = griffin_lim_iters
+        self.preemphasis = preemphasis
+        self.n_fft, self.hop_length, self.win_length = self._stft_parameters()
+        if preemphasis == 0:
+            print(" | > Preemphasis is deactive.")
+
+    def save_wav(self, wav, path):
+        wav *= 32767 / max(0.01, np.max(np.abs(wav)))
+        librosa.output.write_wav(
+            path, wav.astype(np.int16), self.sample_rate)
+
+    def _stft_parameters(self, ):
+        n_fft = int((self.num_freq - 1) * 2)
+        hop_length = int(self.frame_shift_ms / 1000.0 * self.sample_rate)
+        win_length = int(self.frame_length_ms / 1000.0 * self.sample_rate)
+        if n_fft % hop_length != 0:
+            hop_length = n_fft / 8
+            print(" | > hop_length is set to default ({}).".format(hop_length))
+        if n_fft % win_length != 0:
+            win_length = n_fft / 2
+            print(" | > win_length is set to default ({}).".format(win_length))
+        print(" | > fft size: {}, hop length: {}, win length: {}".format(
+            n_fft, hop_length, win_length))
+        return int(n_fft), int(hop_length), int(win_length)
+
+    def _lws_processor(self):
+        try:
+            return lws.lws(
+                self.win_length,
+                self.hop_length,
+                fftsize=self.n_fft,
+                mode="speech")
+        except:
+            raise RuntimeError(
+                " !! WindowLength({}) is not multiple of HopLength({}).".
+                format(self.win_length, self.hop_length))
+
+    def _amp_to_db(self, x):
+        min_level = np.exp(self.min_level_db / 20 * np.log(10))
+        return 20 * np.log10(np.maximum(min_level, x))
+
+    def _db_to_amp(self, x):
+        return np.power(10.0, x * 0.05)
+
+    def _normalize(self, S):
+        return np.clip((S - self.min_level_db) / -self.min_level_db, 0, 1)
+
+    def _denormalize(self, S):
+        return (np.clip(S, 0, 1) * -self.min_level_db) + self.min_level_db
+
+    def apply_preemphasis(self, x):
+        if self.preemphasis == 0:
+            raise RuntimeError(" !! Preemphasis is applied with factor 0.0. ")
+        return signal.lfilter([1, -self.preemphasis], [1], x)
+
+    def apply_inv_preemphasis(self, x):
+        if self.preemphasis == 0:
+            raise RuntimeError(" !! Preemphasis is applied with factor 0.0. ")
+        return signal.lfilter([1], [1, -self.preemphasis], x)
+
+    def spectrogram(self, y):
+        f = open(os.devnull, 'w')
+        old_out = sys.stdout
+        sys.stdout = f
+        if self.preemphasis:
+            D = self._lws_processor().stft(self.apply_preemphasis(y)).T
+        else:
+            D = self._lws_processor().stft(y).T
+        S = self._amp_to_db(np.abs(D)) - self.ref_level_db
+        sys.stdout = old_out
+        return self._normalize(S)
+
+    def inv_spectrogram(self, spectrogram):
+        '''Converts spectrogram to waveform using librosa'''
+        f = open(os.devnull, 'w')
+        old_out = sys.stdout
+        sys.stdout = f
+        S = self._denormalize(spectrogram)
+        S = self._db_to_amp(S + self.ref_level_db)  # Convert back to linear
+        processor = self._lws_processor()
+        D = processor.run_lws(S.astype(np.float64).T**self.power)
+        y = processor.istft(D).astype(np.float32)
+        # Reconstruct phase
+        sys.stdout = old_out
+        if self.preemphasis:
+            return self.apply_inv_preemphasis(y)
+        return y
+
+    def _linear_to_mel(self, spectrogram):
+        global _mel_basis
+        if _mel_basis is None:
+            _mel_basis = self._build_mel_basis()
+        return np.dot(_mel_basis, spectrogram)
+
+    def _build_mel_basis(self, ):
+        return librosa.filters.mel(
+            self.sample_rate, self.n_fft, n_mels=self.num_mels)
+
+
+#                                    fmin=self.min_mel_freq, fmax=self.max_mel_freq)
+
+    def melspectrogram(self, y):
+        f = open(os.devnull, 'w')
+        old_out = sys.stdout
+        sys.stdout = f
+        if self.preemphasis:
+            D = self._lws_processor().stft(self.apply_preemphasis(y)).T
+        else:
+            D = self._lws_processor().stft(y).T
+        S = self._amp_to_db(self._linear_to_mel(np.abs(D))) - self.ref_level_db
+        sys.stdout = old_out
+        return self._normalize(S)

utils/data.py

@@ -4,9 +4,8 @@ import numpy as np
 def _pad_data(x, length):
     _pad = 0
     assert x.ndim == 1
-    return np.pad(x, (0, length - x.shape[0]),
-                  mode='constant',
-                  constant_values=_pad)
+    return np.pad(
+        x, (0, length - x.shape[0]), mode='constant', constant_values=_pad)
 
 
 def prepare_data(inputs):
@@ -17,8 +16,10 @@ def prepare_data(inputs):
 def _pad_tensor(x, length):
     _pad = 0
     assert x.ndim == 2
-    x = np.pad(x, [[0, 0], [0, length - x.shape[1]]],
-               mode='constant', constant_values=_pad)
+    x = np.pad(
+        x, [[0, 0], [0, length - x.shape[1]]],
+        mode='constant',
+        constant_values=_pad)
     return x
 
 
@@ -32,7 +33,8 @@ def prepare_tensor(inputs, out_steps):
 def _pad_stop_target(x, length):
     _pad = 1.
     assert x.ndim == 1
-    return np.pad(x, (0, length - x.shape[0]), mode='constant', constant_values=_pad)
+    return np.pad(
+        x, (0, length - x.shape[0]), mode='constant', constant_values=_pad)
 
 
 def prepare_stop_target(inputs, out_steps):
@@ -44,6 +46,7 @@ def prepare_stop_target(inputs, out_steps):
 
 def pad_per_step(inputs, pad_len):
     timesteps = inputs.shape[-1]
-    return np.pad(inputs, [[0, 0], [0, 0],
-                           [0, pad_len]],
-                  mode='constant', constant_values=0.0)
+    return np.pad(
+        inputs, [[0, 0], [0, 0], [0, pad_len]],
+        mode='constant',
+        constant_values=0.0)

utils/generic_utils.py

@@ -10,6 +10,7 @@ import subprocess
 import numpy as np
 from collections import OrderedDict
 from torch.autograd import Variable
+from utils.text import text_to_sequence
 
 
 class AttrDict(dict):
@@ -27,22 +28,26 @@ def load_config(config_path):
 def get_commit_hash():
     """https://stackoverflow.com/questions/14989858/get-the-current-git-hash-in-a-python-script"""
     try:
-        subprocess.check_output(['git', 'diff-index', '--quiet', 'HEAD'])   # Verify client is clean
+        subprocess.check_output(['git', 'diff-index', '--quiet',
+                                 'HEAD'])  # Verify client is clean
     except:
-        raise RuntimeError(" !! Commit before training to get the commit hash.")
-    commit = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode().strip()
+        raise RuntimeError(
+            " !! Commit before training to get the commit hash.")
+    commit = subprocess.check_output(['git', 'rev-parse', '--short',
+                                      'HEAD']).decode().strip()
     print(' > Git Hash: {}'.format(commit))
     return commit
 
 
 def create_experiment_folder(root_path, model_name, debug):
     """ Create a folder with the current date and time """
-    date_str = datetime.datetime.now().strftime("%B-%d-%Y_%I:%M%p")
+    date_str = datetime.datetime.now().strftime("%B-%d-%Y_%I+%M%p")
     if debug:
         commit_hash = 'debug'
-    else: 
+    else:
         commit_hash = get_commit_hash()
-    output_folder = os.path.join(root_path, date_str + '-' + model_name + '-' + commit_hash)
+    output_folder = os.path.join(
+        root_path, date_str + '-' + model_name + '-' + commit_hash)
     os.makedirs(output_folder, exist_ok=True)
     print(" > Experiment folder: {}".format(output_folder))
     return output_folder
@@ -51,7 +56,7 @@ def create_experiment_folder(root_path, model_name, debug):
 def remove_experiment_folder(experiment_path):
     """Check folder if there is a checkpoint, otherwise remove the folder"""
 
-    checkpoint_files = glob.glob(experiment_path+"/*.pth.tar")
+    checkpoint_files = glob.glob(experiment_path + "/*.pth.tar")
     if len(checkpoint_files) < 1:
         if os.path.exists(experiment_path):
             shutil.rmtree(experiment_path)
@@ -78,32 +83,37 @@ def _trim_model_state_dict(state_dict):
     return new_state_dict
 
 
-def save_checkpoint(model, optimizer, model_loss, out_path,
+def save_checkpoint(model, optimizer, optimizer_st, model_loss, out_path,
                     current_step, epoch):
     checkpoint_path = 'checkpoint_{}.pth.tar'.format(current_step)
     checkpoint_path = os.path.join(out_path, checkpoint_path)
     print(" | | > Checkpoint saving : {}".format(checkpoint_path))
 
-    new_state_dict = _trim_model_state_dict(model.state_dict())
-    state = {'model': new_state_dict,
-             'optimizer': optimizer.state_dict(),
-             'step': current_step,
-             'epoch': epoch,
-             'linear_loss': model_loss,
-             'date': datetime.date.today().strftime("%B %d, %Y")}
+    new_state_dict = model.state_dict()
+    state = {
+        'model': new_state_dict,
+        'optimizer': optimizer.state_dict(),
+        'optimizer_st': optimizer_st.state_dict(),
+        'step': current_step,
+        'epoch': epoch,
+        'linear_loss': model_loss,
+        'date': datetime.date.today().strftime("%B %d, %Y")
+    }
     torch.save(state, checkpoint_path)
 
 
 def save_best_model(model, optimizer, model_loss, best_loss, out_path,
                     current_step, epoch):
     if model_loss < best_loss:
-        new_state_dict = _trim_model_state_dict(model.state_dict())
-        state = {'model': new_state_dict,
-                 'optimizer': optimizer.state_dict(),
-                 'step': current_step,
-                 'epoch': epoch,
-                 'linear_loss': model_loss,
-                 'date': datetime.date.today().strftime("%B %d, %Y")}
+        new_state_dict = model.state_dict()
+        state = {
+            'model': new_state_dict,
+            'optimizer': optimizer.state_dict(),
+            'step': current_step,
+            'epoch': epoch,
+            'linear_loss': model_loss,
+            'date': datetime.date.today().strftime("%B %d, %Y")
+        }
         best_loss = model_loss
         bestmodel_path = 'best_model.pth.tar'
         bestmodel_path = os.path.join(out_path, bestmodel_path)
@@ -113,16 +123,13 @@ def save_best_model(model, optimizer, model_loss, best_loss, out_path,
     return best_loss
 
 
-def check_update(model, grad_clip, grad_top):
+def check_update(model, grad_clip):
     r'''Check model gradient against unexpected jumps and failures'''
     skip_flag = False
     grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip)
     if np.isinf(grad_norm):
         print(" | > Gradient is INF !!")
         skip_flag = True
-    elif grad_norm > grad_top:
-        print(" | > Gradient is above the top limit !!")
-        skip_flag = True
     return grad_norm, skip_flag
 
 
@@ -135,19 +142,18 @@ def lr_decay(init_lr, global_step, warmup_steps):
     return lr
 
 
-def create_attn_mask(N, T, g=0.05):
-    r'''creating attn mask for guided attention
-    TODO: vectorize'''
-    M = np.zeros([N, T])
-    for t in range(T):
-        for n in range(N):
-            val = 20 * np.exp(-pow((n/N)-(t/T), 2.0)/g)
-            M[n, t] = val
-    e_x = np.exp(M - np.max(M))
-    M = e_x / e_x.sum(axis=0) # only difference
-    M = torch.FloatTensor(M).t().cuda()
-    M = torch.stack([M]*32)
-    return M
+class AnnealLR(torch.optim.lr_scheduler._LRScheduler):
+    def __init__(self, optimizer, warmup_steps=0.1):
+        self.warmup_steps = float(warmup_steps)
+        super(AnnealLR, self).__init__(optimizer, last_epoch)
+
+    def get_lr(self):
+        return [
+            base_lr * self.warmup_steps**0.5 * torch.min([
+                self.last_epoch * self.warmup_steps**-1.5, self.last_epoch**
+                -0.5
+            ]) for base_lr in self.base_lrs
+        ]
 
 
 def mk_decay(init_mk, max_epoch, n_epoch):
@@ -159,142 +165,27 @@ def count_parameters(model):
     return sum(p.numel() for p in model.parameters() if p.requires_grad)
 
 
-class Progbar(object):
-    """Displays a progress bar.
-    Args:
-        target: Total number of steps expected, None if unknown.
-        interval: Minimum visual progress update interval (in seconds).
-    """
+# from https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1
+def sequence_mask(sequence_length, max_len=None):
+    if max_len is None:
+        max_len = sequence_length.data.max()
+    batch_size = sequence_length.size(0)
+    seq_range = torch.arange(0, max_len).long()
+    seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
+    if sequence_length.is_cuda:
+        seq_range_expand = seq_range_expand.cuda()
+    seq_length_expand = (sequence_length.unsqueeze(1)
+                         .expand_as(seq_range_expand))
+    return seq_range_expand < seq_length_expand
 
-    def __init__(self, target, width=30, verbose=1, interval=0.05):
-        self.width = width
-        self.target = target
-        self.sum_values = {}
-        self.unique_values = []
-        self.start = time.time()
-        self.last_update = 0
-        self.interval = interval
-        self.total_width = 0
-        self.seen_so_far = 0
-        self.verbose = verbose
-        self._dynamic_display = ((hasattr(sys.stdout, 'isatty') and
-                                  sys.stdout.isatty()) or
-                                 'ipykernel' in sys.modules)
 
-    def update(self, current, values=None, force=False):
-        """Updates the progress bar.
-        # Arguments
-            current: Index of current step.
-            values: List of tuples (name, value_for_last_step).
-                The progress bar will display averages for these values.
-            force: Whether to force visual progress update.
-        """
-        values = values or []
-        for k, v in values:
-            if k not in self.sum_values:
-                self.sum_values[k] = [v * (current - self.seen_so_far),
-                                      current - self.seen_so_far]
-                self.unique_values.append(k)
-            else:
-                self.sum_values[k][0] += v * (current - self.seen_so_far)
-                self.sum_values[k][1] += (current - self.seen_so_far)
-        self.seen_so_far = current
-
-        now = time.time()
-        info = ' - %.0fs' % (now - self.start)
-        if self.verbose == 1:
-            if (not force and (now - self.last_update) < self.interval and
-                    self.target is not None and current < self.target):
-                return
-
-            prev_total_width = self.total_width
-            if self._dynamic_display:
-                sys.stdout.write('\b' * prev_total_width)
-                sys.stdout.write('\r')
-            else:
-                sys.stdout.write('\n')
-
-            if self.target is not None:
-                numdigits = int(np.floor(np.log10(self.target))) + 1
-                barstr = '%%%dd/%d [' % (numdigits, self.target)
-                bar = barstr % current
-                prog = float(current) / self.target
-                prog_width = int(self.width * prog)
-                if prog_width > 0:
-                    bar += ('=' * (prog_width - 1))
-                    if current < self.target:
-                        bar += '>'
-                    else:
-                        bar += '='
-                bar += ('.' * (self.width - prog_width))
-                bar += ']'
-            else:
-                bar = '%7d/Unknown' % current
-
-            self.total_width = len(bar)
-            sys.stdout.write(bar)
-
-            if current:
-                time_per_unit = (now - self.start) / current
-            else:
-                time_per_unit = 0
-            if self.target is not None and current < self.target:
-                eta = time_per_unit * (self.target - current)
-                if eta > 3600:
-                    eta_format = '%d:%02d:%02d' % (
-                        eta // 3600, (eta % 3600) // 60, eta % 60)
-                elif eta > 60:
-                    eta_format = '%d:%02d' % (eta // 60, eta % 60)
-                else:
-                    eta_format = '%ds' % eta
-
-                info = ' - ETA: %s' % eta_format
-
-            if time_per_unit >= 1:
-                info += ' %.0fs/step' % time_per_unit
-            elif time_per_unit >= 1e-3:
-                info += ' %.0fms/step' % (time_per_unit * 1e3)
-            else:
-                info += ' %.0fus/step' % (time_per_unit * 1e6)
-
-            for k in self.unique_values:
-                info += ' - %s:' % k
-                if isinstance(self.sum_values[k], list):
-                    avg = np.mean(
-                        self.sum_values[k][0] / max(1, self.sum_values[k][1]))
-                    if abs(avg) > 1e-3:
-                        info += ' %.4f' % avg
-                    else:
-                        info += ' %.4e' % avg
-                else:
-                    info += ' %s' % self.sum_values[k]
-
-            self.total_width += len(info)
-            if prev_total_width > self.total_width:
-                info += (' ' * (prev_total_width - self.total_width))
-
-            if self.target is not None and current >= self.target:
-                info += '\n'
-
-            sys.stdout.write(info)
-            sys.stdout.flush()
-
-        elif self.verbose == 2:
-            if self.target is None or current >= self.target:
-                for k in self.unique_values:
-                    info += ' - %s:' % k
-                    avg = np.mean(
-                        self.sum_values[k][0] / max(1, self.sum_values[k][1]))
-                    if avg > 1e-3:
-                        info += ' %.4f' % avg
-                    else:
-                        info += ' %.4e' % avg
-                info += '\n'
-
-                sys.stdout.write(info)
-                sys.stdout.flush()
-
-        self.last_update = now
-
-    def add(self, n, values=None):
-        self.update(self.seen_so_far + n, values)
+def synthesis(model, ap, text, use_cuda, text_cleaner):
+    text_cleaner = [text_cleaner]
+    seq = np.array(text_to_sequence(text, text_cleaner))
+    chars_var = torch.from_numpy(seq).unsqueeze(0)
+    if use_cuda:
+        chars_var = chars_var.cuda().long()
+    _, linear_out, alignments, _ = model.forward(chars_var)
+    linear_out = linear_out[0].data.cpu().numpy()
+    wav = ap.inv_spectrogram(linear_out.T)
+    return wav, linear_out, alignments

utils/model.py

@@ -1,9 +0,0 @@
-
-def get_param_size(model):
-    params = 0
-    for p in model.parameters():
-        tmp = 1
-        for x in p.size():
-            tmp *= x
-        params += tmp
-    return params

utils/text/__init__.py

@@ -4,7 +4,6 @@ import re
 from utils.text import cleaners
 from utils.text.symbols import symbols
 
-
 # Mappings from symbol to numeric ID and vice versa:
 _symbol_to_id = {s: i for i, s in enumerate(symbols)}
 _id_to_symbol = {i: s for i, s in enumerate(symbols)}

utils/text/cleaners.py

@@ -14,31 +14,31 @@ import re
 from unidecode import unidecode
 from .numbers import normalize_numbers
 
-
 # Regular expression matching whitespace:
 _whitespace_re = re.compile(r'\s+')
 
 # List of (regular expression, replacement) pairs for abbreviations:
-_abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [
-    ('mrs', 'misess'),
-    ('mr', 'mister'),
-    ('dr', 'doctor'),
-    ('st', 'saint'),
-    ('co', 'company'),
-    ('jr', 'junior'),
-    ('maj', 'major'),
-    ('gen', 'general'),
-    ('drs', 'doctors'),
-    ('rev', 'reverend'),
-    ('lt', 'lieutenant'),
-    ('hon', 'honorable'),
-    ('sgt', 'sergeant'),
-    ('capt', 'captain'),
-    ('esq', 'esquire'),
-    ('ltd', 'limited'),
-    ('col', 'colonel'),
-    ('ft', 'fort'),
-]]
+_abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1])
+                  for x in [
+                      ('mrs', 'misess'),
+                      ('mr', 'mister'),
+                      ('dr', 'doctor'),
+                      ('st', 'saint'),
+                      ('co', 'company'),
+                      ('jr', 'junior'),
+                      ('maj', 'major'),
+                      ('gen', 'general'),
+                      ('drs', 'doctors'),
+                      ('rev', 'reverend'),
+                      ('lt', 'lieutenant'),
+                      ('hon', 'honorable'),
+                      ('sgt', 'sergeant'),
+                      ('capt', 'captain'),
+                      ('esq', 'esquire'),
+                      ('ltd', 'limited'),
+                      ('col', 'colonel'),
+                      ('ft', 'fort'),
+                  ]]
 
 
 def expand_abbreviations(text):

utils/text/cmudict.py

@@ -1,17 +1,16 @@
 # -*- coding: utf-8 -*-
 
-
 import re
 
-
 valid_symbols = [
-    'AA', 'AA0', 'AA1', 'AA2', 'AE', 'AE0', 'AE1', 'AE2', 'AH', 'AH0', 'AH1', 'AH2',
-    'AO', 'AO0', 'AO1', 'AO2', 'AW', 'AW0', 'AW1', 'AW2', 'AY', 'AY0', 'AY1', 'AY2',
-    'B', 'CH', 'D', 'DH', 'EH', 'EH0', 'EH1', 'EH2', 'ER', 'ER0', 'ER1', 'ER2', 'EY',
-    'EY0', 'EY1', 'EY2', 'F', 'G', 'HH', 'IH', 'IH0', 'IH1', 'IH2', 'IY', 'IY0', 'IY1',
-    'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG', 'OW', 'OW0', 'OW1', 'OW2', 'OY', 'OY0',
-    'OY1', 'OY2', 'P', 'R', 'S', 'SH', 'T', 'TH', 'UH', 'UH0', 'UH1', 'UH2', 'UW',
-    'UW0', 'UW1', 'UW2', 'V', 'W', 'Y', 'Z', 'ZH'
+    'AA', 'AA0', 'AA1', 'AA2', 'AE', 'AE0', 'AE1', 'AE2', 'AH', 'AH0', 'AH1',
+    'AH2', 'AO', 'AO0', 'AO1', 'AO2', 'AW', 'AW0', 'AW1', 'AW2', 'AY', 'AY0',
+    'AY1', 'AY2', 'B', 'CH', 'D', 'DH', 'EH', 'EH0', 'EH1', 'EH2', 'ER', 'ER0',
+    'ER1', 'ER2', 'EY', 'EY0', 'EY1', 'EY2', 'F', 'G', 'HH', 'IH', 'IH0',
+    'IH1', 'IH2', 'IY', 'IY0', 'IY1', 'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG',
+    'OW', 'OW0', 'OW1', 'OW2', 'OY', 'OY0', 'OY1', 'OY2', 'P', 'R', 'S', 'SH',
+    'T', 'TH', 'UH', 'UH0', 'UH1', 'UH2', 'UW', 'UW0', 'UW1', 'UW2', 'V', 'W',
+    'Y', 'Z', 'ZH'
 ]
 
 _valid_symbol_set = set(valid_symbols)
@@ -27,8 +26,10 @@ class CMUDict:
         else:
             entries = _parse_cmudict(file_or_path)
         if not keep_ambiguous:
-            entries = {word: pron for word,
-                       pron in entries.items() if len(pron) == 1}
+            entries = {
+                word: pron
+                for word, pron in entries.items() if len(pron) == 1
+            }
         self._entries = entries
 
     def __len__(self):

utils/text/numbers.py

@@ -8,61 +8,45 @@ _ordinal_re = re.compile(r'([0-9]+)(st|nd|rd|th)')
 _number_re = re.compile(r'[0-9]+')
 
 _units = [
-  '',
-  'one',
-  'two',
-  'three',
-  'four',
-  'five',
-  'six',
-  'seven',
-  'eight',
-  'nine',
-  'ten',
-  'eleven',
-  'twelve',
-  'thirteen',
-  'fourteen',
-  'fifteen',
-  'sixteen',
-  'seventeen',
-  'eighteen',
-  'nineteen'
+    '', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
+    'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen',
+    'seventeen', 'eighteen', 'nineteen'
 ]
 
 _tens = [
-  '',
-  'ten',
-  'twenty',
-  'thirty',
-  'forty',
-  'fifty',
-  'sixty',
-  'seventy',
-  'eighty',
-  'ninety',
+    '',
+    'ten',
+    'twenty',
+    'thirty',
+    'forty',
+    'fifty',
+    'sixty',
+    'seventy',
+    'eighty',
+    'ninety',
 ]
 
 _digit_groups = [
-  '',
-  'thousand',
-  'million',
-  'billion',
-  'trillion',
-  'quadrillion',
+    '',
+    'thousand',
+    'million',
+    'billion',
+    'trillion',
+    'quadrillion',
 ]
 
 _ordinal_suffixes = [
-  ('one', 'first'),
-  ('two', 'second'),
-  ('three', 'third'),
-  ('five', 'fifth'),
-  ('eight', 'eighth'),
-  ('nine', 'ninth'),
-  ('twelve', 'twelfth'),
-  ('ty', 'tieth'),
+    ('one', 'first'),
+    ('two', 'second'),
+    ('three', 'third'),
+    ('five', 'fifth'),
+    ('eight', 'eighth'),
+    ('nine', 'ninth'),
+    ('twelve', 'twelfth'),
+    ('ty', 'tieth'),
 ]
 
+
 def _remove_commas(m):
     return m.group(1).replace(',', '')
 
@@ -114,7 +98,7 @@ def _standard_number_to_words(n, digit_group):
 def _number_to_words(n):
     # Handle special cases first, then go to the standard case:
     if n >= 1000000000000000000:
-        return str(n)   # Too large, just return the digits
+        return str(n)  # Too large, just return the digits
     elif n == 0:
         return 'zero'
     elif n % 100 == 0 and n % 1000 != 0 and n < 3000:

utils/text/symbols.py

@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
-
-
 '''
 Defines the set of symbols used in text input to the model.
 
@@ -19,6 +17,5 @@ _arpabet = ['@' + s for s in cmudict.valid_symbols]
 # Export all symbols:
 symbols = [_pad, _eos] + list(_characters) + _arpabet
 
-
 if __name__ == '__main__':
     print(symbols)

utils/visual.py

@@ -6,8 +6,8 @@ import matplotlib.pyplot as plt
 
 def plot_alignment(alignment, info=None):
     fig, ax = plt.subplots(figsize=(16, 10))
-    im = ax.imshow(alignment.T, aspect='auto', origin='lower',
-                   interpolation='none')
+    im = ax.imshow(
+        alignment.T, aspect='auto', origin='lower', interpolation='none')
     fig.colorbar(im, ax=ax)
     xlabel = 'Decoder timestep'
     if info is not None:
@@ -15,11 +15,7 @@ def plot_alignment(alignment, info=None):
     plt.xlabel(xlabel)
     plt.ylabel('Encoder timestep')
     plt.tight_layout()
-    fig.canvas.draw()
-    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
-    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
-    plt.close()
-    return data
+    return fig
 
 
 def plot_spectrogram(linear_output, audio):
@@ -28,8 +24,4 @@ def plot_spectrogram(linear_output, audio):
     plt.imshow(spectrogram.T, aspect="auto", origin="lower")
     plt.colorbar()
     plt.tight_layout()
-    fig.canvas.draw()
-    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
-    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
-    plt.close()
-    return data
+    return fig