Merge branch 'dev-tacotron2'

tests/__init__.py

@@ -0,0 +1,16 @@
+import os
+
+
+def get_tests_path():
+    """Returns the path to the test directory."""
+    return os.path.dirname(os.path.realpath(__file__))
+
+
+def get_tests_input_path():
+    """Returns the path to the test data directory."""
+    return os.path.join(get_tests_path(), "inputs")
+
+
+def get_tests_output_path():
+    """Returns the path to the directory for test outputs."""
+    return os.path.join(get_tests_path(), "outputs")

tests/inputs/common_voice.tsv

@@ -0,0 +1,10 @@
+client_id	path	sentence	up_votes	down_votes	age	gender	accent
+aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3	21fce545b24d9a5af0403b949e95e8dd3c10c4ff3e371f14e4d5b4ebf588670b7c9e618285fc872d94a89ed7f0217d9019fe5de33f1577b49dcd518eacf63c4b	Man sollte den Länderfinanzausgleich durch einen Bundesliga-Soli ersetzen.	2	0	fourties	male	germany
+aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3	42758baa4e91ef6b82b78b11a04bc5117a035a8d3bc42c33c0bb3084909af17043a194cfd8cd9839f0d6ef1ea5413acda5de5d1936abcc8ca073e2da7f9488ea	Folgende Lektüre kann ich Ihnen zum Thema Kognitionspsychologie empfehlen.	2	0	fourties	male	germany
+aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3	478f172c2dbda6675247e9674ade79a5b49efeefb7c9e99040dcc69a847a01d69398cf180570859b0cdb6fc887717e04cd8b149c723d48d00b5d18f41314667c	Touristen winkten den Leuten am Ufer zu.	2	0	fourties	male	germany
+aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3	4854368d6d21cb44103e432b5332f31e8d14030582a40850501bcf9377d699314a5ff27a8206fa89254ddde7f3f1c65d33836f3dfcfa16bcabec08537f2b5f08	Valentin hat das Handtuch geworfen.	2	0	fourties	male	germany
+aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3	a841a9f3e032495dd47560e65fba99eeacb3618c07de8b1351c20188e5b71e33cc52f73315f721a3a24b65763c65bb52fbf3ae052eb5774e834dcb57f296db5c	Ohne Gehörschutz bei der Arbeit wäre Klaus wohl nach zwei Wochen taub.	2	0	fourties	male	germany
+aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3	03ab970a5bf5410bc3260b073cce1c7f49c688ace83dc8836b1c0f79a09fea45a27725c769f4a9d2e6181defd016d22642789d7ac51da252b42958a9192bd4c7	Gerrit erinnerte sich daran, dass er einst einen Eid geschworen hatte.	2	0	fourties	male	germany
+aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3	c4a94df443ad5f2c7241413ef7145d5f0de41ae929759073917fe96166da3c7d3a612c920ed7b0f3d5950a38d6205e9dba24af5bfb27e390a220d004e6e26744	Auf das, was jetzt kommt, habe ich nämlich absolut keinen Bock.	2	0	fourties	male	germany
+aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3	104695983b1112229b4a48696405d044dad9ddef713aa6eb1a6240cc16b7b7a2a96354ae9da99783850dde08a982091e48d3037288a3a58269cac9fe70a6bd7a	Von Salzburg ist es doch nicht weit bis zum Chiemsee.	2	0	fourties	male	germany
+d5b5da343bb0f65e3580bc2e1902a4f5d004241488d751503f2020bc1c93f89715e355e35f6e25def2b90cb3eea99fda403eb92ae3afbb84d039a54a4ed2d875	ad2f69e053b0e20e01c82b9821fe5787f1cc8e4b0b97f0e4cab1e9a652c577169c8244fb222281a60ee3081854014113e04c4ca43643100b7c01dab0fac11974	Warum werden da keine strafrechtlichen Konsequenzen gezogen?	2	0	thirties	male	germany

tests/test_audio.py

@@ -2,21 +2,23 @@ import os
 import unittest
 import numpy as np
 import torch as T
+
+from tests import get_tests_path, get_tests_input_path, get_tests_output_path
 from utils.audio import AudioProcessor
 from utils.generic_utils import load_config
 
-file_path = os.path.dirname(os.path.realpath(__file__))
-INPUTPATH = os.path.join(file_path, 'inputs')
-OUTPATH = os.path.join(file_path, "outputs/audio_tests")
-os.makedirs(OUTPATH, exist_ok=True)
+TESTS_PATH = get_tests_path()
+OUT_PATH = os.path.join(get_tests_output_path(), "audio_tests")
+WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav")
 
-c = load_config(os.path.join(file_path, 'test_config.json'))
+os.makedirs(OUT_PATH, exist_ok=True)
+conf = load_config(os.path.join(TESTS_PATH, 'test_config.json'))
 
 
 class TestAudio(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(TestAudio, self).__init__(*args, **kwargs)
-        self.ap = AudioProcessor(**c.audio)
+        self.ap = AudioProcessor(**conf.audio)
 
     def test_audio_synthesis(self):
         """ 1. load wav
@@ -31,13 +33,13 @@ class TestAudio(unittest.TestCase):
             self.ap.signal_norm = signal_norm
             self.ap.symmetric_norm = symmetric_norm
             self.ap.clip_norm = clip_norm
-            wav = self.ap.load_wav(INPUTPATH + "/example_1.wav")
+            wav = self.ap.load_wav(WAV_FILE)
             mel = self.ap.melspectrogram(wav)
             wav_ = self.ap.inv_mel_spectrogram(mel)
             file_name = "/audio_test-melspec_max_norm_{}-signal_norm_{}-symmetric_{}-clip_norm_{}.wav"\
                 .format(max_norm, signal_norm, symmetric_norm, clip_norm)
             print(" | > Creating wav file at : ", file_name)
-            self.ap.save_wav(wav_, OUTPATH + file_name)
+            self.ap.save_wav(wav_, OUT_PATH + file_name)
 
         # maxnorm = 1.0
         _test(1., False, False, False)
@@ -55,7 +57,7 @@ class TestAudio(unittest.TestCase):
     def test_normalize(self):
         """Check normalization and denormalization for range values and consistency """
         print(" > Testing normalization and denormalization.")
-        wav = self.ap.load_wav(INPUTPATH + "/example_1.wav")
+        wav = self.ap.load_wav(WAV_FILE)
         self.ap.signal_norm = False
         x = self.ap.melspectrogram(wav)
         x_old = x

tests/test_config.json

@@ -36,7 +36,6 @@
 
     "save_step": 200,
     "data_path": "/home/erogol/Data/LJSpeech-1.1/",
-    "data_path_cache": "/media/erogol/data_ssd/Data/Nancy/tts_cache/",
     "output_path": "result",
     "min_seq_len": 0,
     "max_seq_len": 300,

tests/test_layers.py

@@ -38,18 +38,16 @@ class CBHGTests(unittest.TestCase):
 
 class DecoderTests(unittest.TestCase):
     def test_in_out(self):
-        layer = Decoder(in_features=256, memory_dim=80, r=2, memory_size=4, attn_windowing=False)
+        layer = Decoder(in_features=256, memory_dim=80, r=2, memory_size=4, attn_windowing=False, attn_norm="sigmoid")
         dummy_input = T.rand(4, 8, 256)
         dummy_memory = T.rand(4, 2, 80)
 
-        output, alignment, stop_tokens = layer(dummy_input, dummy_memory)
+        output, alignment, stop_tokens = layer(dummy_input, dummy_memory, mask=None)
 
         assert output.shape[0] == 4
         assert output.shape[1] == 1, "size not {}".format(output.shape[1])
         assert output.shape[2] == 80 * 2, "size not {}".format(output.shape[2])
         assert stop_tokens.shape[0] == 4
-        assert stop_tokens.max() <= 1.0
-        assert stop_tokens.min() >= 0
 
 
 class EncoderTests(unittest.TestCase):

tests/test_loader.py

@@ -7,7 +7,7 @@ from torch.utils.data import DataLoader
 from utils.generic_utils import load_config
 from utils.audio import AudioProcessor
 from datasets import TTSDataset
-from datasets.preprocess import ljspeech, tts_cache
+from datasets.preprocess import ljspeech
 
 file_path = os.path.dirname(os.path.realpath(__file__))
 OUTPATH = os.path.join(file_path, "outputs/loader_tests/")
@@ -16,15 +16,11 @@ c = load_config(os.path.join(file_path, 'test_config.json'))
 ok_ljspeech = os.path.exists(c.data_path)
 
 DATA_EXIST = True
-CACHE_EXIST = True
-if not os.path.exists(c.data_path_cache):
-    CACHE_EXIST = False
-
 if not os.path.exists(c.data_path):
     DATA_EXIST = False
 
 print(" > Dynamic data loader test: {}".format(DATA_EXIST))
-print(" > Cache data loader test: {}".format(CACHE_EXIST))
+
 
 class TestTTSDataset(unittest.TestCase):
     def __init__(self, *args, **kwargs):
@@ -126,8 +122,9 @@ class TestTTSDataset(unittest.TestCase):
                 wav = self.ap.load_wav(item_idx[0])
                 mel = self.ap.melspectrogram(wav)
                 mel_dl = mel_input[0].cpu().numpy()
-                assert (
-                    abs(mel.T).astype("float32") - abs(mel_dl[:-1])).sum() == 0
+                assert (abs(mel.T).astype("float32")
+                        - abs(mel_dl[:-1])
+                        ).sum() == 0
 
                 # check mel-spec correctness
                 mel_spec = mel_input[0].cpu().numpy()
@@ -139,7 +136,8 @@ class TestTTSDataset(unittest.TestCase):
                 linear_spec = linear_input[0].cpu().numpy()
                 wav = self.ap.inv_spectrogram(linear_spec.T)
                 self.ap.save_wav(wav, OUTPATH + '/linear_inv_dataloader.wav')
-                shutil.copy(item_idx[0], OUTPATH + '/linear_target_dataloader.wav')
+                shutil.copy(item_idx[0],
+                            OUTPATH + '/linear_target_dataloader.wav')
 
                 # check the last time step to be zero padded
                 assert linear_input[0, -1].sum() == 0
@@ -192,4 +190,4 @@ class TestTTSDataset(unittest.TestCase):
 
                 # check batch conditions
                 assert (linear_input * stop_target.unsqueeze(2)).sum() == 0
-                assert (mel_input * stop_target.unsqueeze(2)).sum() == 0
+                assert (mel_input * stop_target.unsqueeze(2)).sum() == 0

tests/test_preprocessors.py

@@ -0,0 +1,28 @@
+import unittest
+import os
+from tests import get_tests_input_path
+
+from datasets.preprocess import common_voice
+
+
+class TestPreprocessors(unittest.TestCase):
+
+    def test_common_voice_preprocessor(self):
+        root_path = get_tests_input_path()
+        meta_file = "common_voice.tsv"
+        items = common_voice(root_path, meta_file)
+        assert items[0][0] == "Man sollte den Länderfinanzausgleich durch " \
+                              "einen Bundesliga-Soli ersetzen."
+        assert items[0][1] == os.path.join(get_tests_input_path(), "clips",
+                                           "21fce545b24d9a5af0403b949e95e8dd3"
+                                           "c10c4ff3e371f14e4d5b4ebf588670b7c"
+                                           "9e618285fc872d94a89ed7f0217d9019f"
+                                           "e5de33f1577b49dcd518eacf63c4b.wav")
+
+        assert items[-1][0] == "Warum werden da keine strafrechtlichen " \
+                               "Konsequenzen gezogen?"
+        assert items[-1][1] == os.path.join(get_tests_input_path(), "clips",
+                                            "ad2f69e053b0e20e01c82b9821fe5787f1"
+                                            "cc8e4b0b97f0e4cab1e9a652c577169c82"
+                                            "44fb222281a60ee3081854014113e04c4c"
+                                            "a43643100b7c01dab0fac11974.wav")

tests/test_tacotron2_model.py

@@ -0,0 +1,69 @@
+import os
+import copy
+import torch
+import unittest
+import numpy as np
+
+from torch import optim
+from torch import nn
+from utils.generic_utils import load_config
+from layers.losses import MSELossMasked
+from models.tacotron2 import Tacotron2
+
+torch.manual_seed(1)
+use_cuda = torch.cuda.is_available()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+file_path = os.path.dirname(os.path.realpath(__file__))
+c = load_config(os.path.join(file_path, 'test_config.json'))
+
+
+class TacotronTrainTest(unittest.TestCase):
+    def test_train_step(self):
+        input = torch.randint(0, 24, (8, 128)).long().to(device)
+        input_lengths = torch.randint(100, 128, (8, )).long().to(device)
+        input_lengths = torch.sort(input_lengths, descending=True)[0]
+        mel_spec = torch.rand(8, 30, c.audio['num_mels']).to(device)
+        mel_postnet_spec = torch.rand(8, 30, c.audio['num_mels']).to(device)
+        mel_lengths = torch.randint(20, 30, (8, )).long().to(device)
+        stop_targets = torch.zeros(8, 30, 1).float().to(device)
+
+        for idx in mel_lengths:
+            stop_targets[:, int(idx.item()):, 0] = 1.0
+
+        stop_targets = stop_targets.view(input.shape[0],
+                                         stop_targets.size(1) // c.r, -1)
+        stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float().squeeze()
+
+        criterion = MSELossMasked().to(device)
+        criterion_st = nn.BCEWithLogitsLoss().to(device)
+        model = Tacotron2(24, c.r).to(device)
+        model.train()
+        model_ref = copy.deepcopy(model)
+        count = 0
+        for param, param_ref in zip(model.parameters(),
+                                    model_ref.parameters()):
+            assert (param - param_ref).sum() == 0, param
+            count += 1
+        optimizer = optim.Adam(model.parameters(), lr=c.lr)
+        for i in range(5):
+            mel_out, mel_postnet_out, align, stop_tokens = model.forward(
+                input, input_lengths, mel_spec)
+            assert torch.sigmoid(stop_tokens).data.max() <= 1.0
+            assert torch.sigmoid(stop_tokens).data.min() >= 0.0
+            optimizer.zero_grad()
+            loss = criterion(mel_out, mel_spec, mel_lengths)
+            stop_loss = criterion_st(stop_tokens, stop_targets)
+            loss = loss + criterion(mel_postnet_out, mel_postnet_spec, mel_lengths) + stop_loss
+            loss.backward()
+            optimizer.step()
+        # check parameter changes
+        count = 0
+        for param, param_ref in zip(model.parameters(),
+                                    model_ref.parameters()):
+            # ignore pre-higway layer since it works conditional
+            # if count not in [145, 59]:
+            assert (param != param_ref).any(
+            ), "param {} with shape {} not updated!! \n{}\n{}".format(
+                count, param.shape, param, param_ref)
+            count += 1

tests/test_tacotron_model.py

@@ -18,9 +18,16 @@ file_path = os.path.dirname(os.path.realpath(__file__))
 c = load_config(os.path.join(file_path, 'test_config.json'))
 
 
+def count_parameters(model):
+    r"""Count number of trainable parameters in a network"""
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+
 class TacotronTrainTest(unittest.TestCase):
     def test_train_step(self):
         input = torch.randint(0, 24, (8, 128)).long().to(device)
+        input_lengths = torch.randint(100, 129, (8, )).long().to(device)
+        input_lengths[-1] = 128
         mel_spec = torch.rand(8, 30, c.audio['num_mels']).to(device)
         linear_spec = torch.rand(8, 30, c.audio['num_freq']).to(device)
         mel_lengths = torch.randint(20, 30, (8, )).long().to(device)
@@ -31,13 +38,19 @@ class TacotronTrainTest(unittest.TestCase):
 
         stop_targets = stop_targets.view(input.shape[0],
                                          stop_targets.size(1) // c.r, -1)
-        stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float()
+        stop_targets = (stop_targets.sum(2) >
+                        0.0).unsqueeze(2).float().squeeze()
 
         criterion = L1LossMasked().to(device)
-        criterion_st = nn.BCELoss().to(device)
-        model = Tacotron(32, c.embedding_size, c.audio['num_freq'], c.audio['num_mels'],
-                         c.r, c.memory_size).to(device)
+        criterion_st = nn.BCEWithLogitsLoss().to(device)
+        model = Tacotron(
+            32,
+            linear_dim=c.audio['num_freq'],
+            mel_dim=c.audio['num_mels'],
+            r=c.r,
+            memory_size=c.memory_size).to(device)
         model.train()
+        print(" > Num parameters for Tacotron model:%s"%(count_parameters(model)))
         model_ref = copy.deepcopy(model)
         count = 0
         for param, param_ref in zip(model.parameters(),
@@ -47,9 +60,7 @@ class TacotronTrainTest(unittest.TestCase):
         optimizer = optim.Adam(model.parameters(), lr=c.lr)
         for i in range(5):
             mel_out, linear_out, align, stop_tokens = model.forward(
-                input, mel_spec)
-            assert stop_tokens.data.max() <= 1.0
-            assert stop_tokens.data.min() >= 0.0
+                input, input_lengths, mel_spec)
             optimizer.zero_grad()
             loss = criterion(mel_out, mel_spec, mel_lengths)
             stop_loss = criterion_st(stop_tokens, stop_targets)

tests/test_text_processing.py

@@ -0,0 +1,76 @@
+import unittest
+import torch as T
+
+from utils.text import *
+
+def test_phoneme_to_sequence():
+    text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!"
+    text_cleaner = ["phoneme_cleaners"]
+    lang = "en-us"
+    sequence = phoneme_to_sequence(text, text_cleaner, lang)
+    text_hat = sequence_to_phoneme(sequence)
+    gt = "ɹiːsənt ɹɪsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪnkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹɪspɑːnsəbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjuːleɪʃən ænd lɜːnɪŋ!"
+    assert text_hat == gt
+
+    # multiple punctuations
+    text = "Be a voice, not an! echo?"
+    sequence = phoneme_to_sequence(text, text_cleaner, lang)
+    text_hat = sequence_to_phoneme(sequence)
+    gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ?"
+    print(text_hat)
+    print(len(sequence))
+    assert text_hat == gt
+
+    # not ending with punctuation
+    text = "Be a voice, not an! echo"
+    sequence = phoneme_to_sequence(text, text_cleaner, lang)
+    text_hat = sequence_to_phoneme(sequence)
+    gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ"
+    print(text_hat)
+    print(len(sequence))
+    assert text_hat == gt
+
+    # original
+    text = "Be a voice, not an echo!"
+    sequence = phoneme_to_sequence(text, text_cleaner, lang)
+    text_hat = sequence_to_phoneme(sequence)
+    gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!"
+    print(text_hat)
+    print(len(sequence))
+    assert text_hat == gt
+
+    # extra space after the sentence
+    text = "Be a voice, not an! echo.  "
+    sequence = phoneme_to_sequence(text, text_cleaner, lang)
+    text_hat = sequence_to_phoneme(sequence)
+    gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ."
+    print(text_hat)
+    print(len(sequence))
+    assert text_hat == gt
+
+    # extra space after the sentence
+    text = "Be a voice, not an! echo.  "
+    sequence = phoneme_to_sequence(text, text_cleaner, lang, True)
+    text_hat = sequence_to_phoneme(sequence)
+    gt = "^biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ.~"
+    print(text_hat)
+    print(len(sequence))
+    assert text_hat == gt
+
+    # padding char
+    text = "_Be a _voice, not an! echo_"
+    sequence = phoneme_to_sequence(text, text_cleaner, lang)
+    text_hat = sequence_to_phoneme(sequence)
+    gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ"
+    print(text_hat)
+    print(len(sequence))
+    assert text_hat == gt
+
+
+def test_text2phone():
+    text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!"
+    text_cleaner = ["phoneme_cleaners"]
+    gt = "ɹ|iː|s|ə|n|t| |ɹ|ɪ|s|ɜː|tʃ| |æ|t| |h|ɑːɹ|v|ɚ|d| |h|ɐ|z| |ʃ|oʊ|n| |m|ɛ|d|ᵻ|t|eɪ|ɾ|ɪ|ŋ| |f|ɔː|ɹ| |æ|z| |l|ɪ|ɾ|əl| |æ|z| |eɪ|t| |w|iː|k|s| |k|æ|n| |æ|k|tʃ|uː|əl|i|| |ɪ|n|k|ɹ|iː|s|,| |ð|ə| |ɡ|ɹ|eɪ| |m|æ|ɾ|ɚ|ɹ| |ɪ|n|ð|ə| |p|ɑːɹ|t|s| |ʌ|v|ð|ə| |b|ɹ|eɪ|n| |ɹ|ɪ|s|p|ɑː|n|s|ə|b|əl| |f|ɔː|ɹ| |ɪ|m|oʊ|ʃ|ə|n|əl| |ɹ|ɛ|ɡ|j|uː|l|eɪ|ʃ|ə|n||| |æ|n|d| |l|ɜː|n|ɪ|ŋ|!"
+    lang = "en-us"
+    phonemes = text2phone(text, lang)
+    assert gt == phonemes