mirror of
https://github.com/wassname/TTS.git
synced 2026-09-11 12:00:24 +08:00
Capacitron (#977)
* new CI config * initial Capacitron implementation * delete old unused file * fix empty formatting changes * update losses and training script * fix previous commit * fix commit * Add Capacitron test and first round of test fixes * revert formatter change * add changes to the synthesizer * add stepwise gradual lr scheduler and changes to the recipe * add inference script for dev use * feat: add posterior inference arguments to synth methods - added reference wav and text args for posterior inference - some formatting * fix: add espeak flag to base_tts and dataset APIs - use_espeak_phonemes flag was not implemented in those APIs - espeak is now able to be utilised for phoneme generation - necessary phonemizer for the Capacitron model * chore: update training script and style - training script includes the espeak flag and other hyperparams - made style * chore: fix linting * feat: add Tacotron 2 support * leftover from dev * chore:rename parser args * feat: extract optimizers - created a separate optimizer class to merge the two optimizers * chore: revert arbitrary trainer changes * fmt: revert formatting bug * formatting again * formatting fixed * fix: log func * fix: update optimizer - Implemented load_state_dict for continuing training * fix: clean optimizer init for standard models * improvement: purge espeak flags and add training scripts * Delete capacitronT2.py delete old training script, new one is pushed * feat: capacitron trainer methods - extracted capacitron specific training operations from the trainer into custom methods in taco1 and taco2 models * chore: renaming and merging capacitron and gst style args * fix: bug fixes from the previous commit * fix: implement state_dict method on CapacitronOptimizer * fix: call method * fix: inference naming * Delete train_capacitron.py * fix: synthesize * feat: update tests * chore: fix style * Delete capacitron_inference.py * fix: fix train tts t2 capacitron tests * fix: double forward in T2 train step * fix: double forward in T1 train step * fix: run make style * fix: remove unused import * fix: test for T1 capacitron * fix: make lint * feat: add blizzard2013 recipes * make style * fix: update recipes * chore: make style * Plot test sentences in Tacotron * chore: make style and fix import * fix: call forward first before problematic floordiv op * fix: update recipes * feat: add min_audio_len to recipes * aux_input["style_mel"] * chore: make style * Make capacitron T2 recipe more stable * Remove T1 capacitron Ljspeech * feat: implement new grad clipping routine and update configs * make style * Add pretrained checkpoints * Add default vocoder * Change trainer package * Fix grad clip issue for tacotron * Fix scheduler issue with tacotron Co-authored-by: Eren Gölge <egolge@coqui.ai> Co-authored-by: WeberJulian <julian.weber@hotmail.fr> Co-authored-by: Eren Gölge <erogol@hotmail.com>
This commit is contained in:
a-froghyar
co-authored by
Eren Gölge
WeberJulian
Eren Gölge
Eren Gölge
WeberJulian
Eren Gölge
parent
ee99a6c1e2
commit
8be21ec387
@@ -6,7 +6,7 @@ import torch
|
||||
from torch import nn, optim
|
||||
|
||||
from tests import get_tests_input_path
|
||||
from TTS.tts.configs.shared_configs import GSTConfig
|
||||
from TTS.tts.configs.shared_configs import CapacitronVAEConfig, GSTConfig
|
||||
from TTS.tts.configs.tacotron2_config import Tacotron2Config
|
||||
from TTS.tts.layers.losses import MSELossMasked
|
||||
from TTS.tts.models.tacotron2 import Tacotron2
|
||||
@@ -260,6 +260,73 @@ class TacotronGSTTrainTest(unittest.TestCase):
|
||||
count += 1
|
||||
|
||||
|
||||
class TacotronCapacitronTrainTest(unittest.TestCase):
|
||||
@staticmethod
|
||||
def test_train_step():
|
||||
config = Tacotron2Config(
|
||||
num_chars=32,
|
||||
num_speakers=10,
|
||||
use_speaker_embedding=True,
|
||||
out_channels=80,
|
||||
decoder_output_dim=80,
|
||||
use_capacitron_vae=True,
|
||||
capacitron_vae=CapacitronVAEConfig(),
|
||||
optimizer="CapacitronOptimizer",
|
||||
optimizer_params={
|
||||
"RAdam": {"betas": [0.9, 0.998], "weight_decay": 1e-6},
|
||||
"SGD": {"lr": 1e-5, "momentum": 0.9},
|
||||
},
|
||||
)
|
||||
|
||||
batch = dict({})
|
||||
batch["text_input"] = torch.randint(0, 24, (8, 128)).long().to(device)
|
||||
batch["text_lengths"] = torch.randint(100, 129, (8,)).long().to(device)
|
||||
batch["text_lengths"] = torch.sort(batch["text_lengths"], descending=True)[0]
|
||||
batch["text_lengths"][0] = 128
|
||||
batch["mel_input"] = torch.rand(8, 120, config.audio["num_mels"]).to(device)
|
||||
batch["mel_lengths"] = torch.randint(20, 120, (8,)).long().to(device)
|
||||
batch["mel_lengths"] = torch.sort(batch["mel_lengths"], descending=True)[0]
|
||||
batch["mel_lengths"][0] = 120
|
||||
batch["stop_targets"] = torch.zeros(8, 120, 1).float().to(device)
|
||||
batch["stop_target_lengths"] = torch.randint(0, 120, (8,)).to(device)
|
||||
batch["speaker_ids"] = torch.randint(0, 5, (8,)).long().to(device)
|
||||
batch["d_vectors"] = None
|
||||
|
||||
for idx in batch["mel_lengths"]:
|
||||
batch["stop_targets"][:, int(idx.item()) :, 0] = 1.0
|
||||
|
||||
batch["stop_targets"] = batch["stop_targets"].view(
|
||||
batch["text_input"].shape[0], batch["stop_targets"].size(1) // config.r, -1
|
||||
)
|
||||
batch["stop_targets"] = (batch["stop_targets"].sum(2) > 0.0).unsqueeze(2).float().squeeze()
|
||||
|
||||
model = Tacotron2(config).to(device)
|
||||
criterion = model.get_criterion()
|
||||
optimizer = model.get_optimizer()
|
||||
|
||||
model.train()
|
||||
model_ref = copy.deepcopy(model)
|
||||
count = 0
|
||||
for param, param_ref in zip(model.parameters(), model_ref.parameters()):
|
||||
assert (param - param_ref).sum() == 0, param
|
||||
count += 1
|
||||
for _ in range(10):
|
||||
_, loss_dict = model.train_step(batch, criterion)
|
||||
optimizer.zero_grad()
|
||||
loss_dict["capacitron_vae_beta_loss"].backward()
|
||||
optimizer.first_step()
|
||||
loss_dict["loss"].backward()
|
||||
optimizer.step()
|
||||
# check parameter changes
|
||||
count = 0
|
||||
for param, param_ref in zip(model.parameters(), model_ref.parameters()):
|
||||
# ignore pre-higway layer since it works conditional
|
||||
assert (param != param_ref).any(), "param {} with shape {} not updated!! \n{}\n{}".format(
|
||||
count, param.shape, param, param_ref
|
||||
)
|
||||
count += 1
|
||||
|
||||
|
||||
class SCGSTMultiSpeakeTacotronTrainTest(unittest.TestCase):
|
||||
"""Test multi-speaker Tacotron2 with Global Style Tokens and d-vector inputs."""
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import torch
|
||||
from torch import nn, optim
|
||||
|
||||
from tests import get_tests_input_path
|
||||
from TTS.tts.configs.shared_configs import GSTConfig
|
||||
from TTS.tts.configs.shared_configs import CapacitronVAEConfig, GSTConfig
|
||||
from TTS.tts.configs.tacotron_config import TacotronConfig
|
||||
from TTS.tts.layers.losses import L1LossMasked
|
||||
from TTS.tts.models.tacotron import Tacotron
|
||||
@@ -248,6 +248,74 @@ class TacotronGSTTrainTest(unittest.TestCase):
|
||||
count += 1
|
||||
|
||||
|
||||
class TacotronCapacitronTrainTest(unittest.TestCase):
|
||||
@staticmethod
|
||||
def test_train_step():
|
||||
config = TacotronConfig(
|
||||
num_chars=32,
|
||||
num_speakers=10,
|
||||
use_speaker_embedding=True,
|
||||
out_channels=513,
|
||||
decoder_output_dim=80,
|
||||
use_capacitron_vae=True,
|
||||
capacitron_vae=CapacitronVAEConfig(),
|
||||
optimizer="CapacitronOptimizer",
|
||||
optimizer_params={
|
||||
"RAdam": {"betas": [0.9, 0.998], "weight_decay": 1e-6},
|
||||
"SGD": {"lr": 1e-5, "momentum": 0.9},
|
||||
},
|
||||
)
|
||||
|
||||
batch = dict({})
|
||||
batch["text_input"] = torch.randint(0, 24, (8, 128)).long().to(device)
|
||||
batch["text_lengths"] = torch.randint(100, 129, (8,)).long().to(device)
|
||||
batch["text_lengths"] = torch.sort(batch["text_lengths"], descending=True)[0]
|
||||
batch["text_lengths"][0] = 128
|
||||
batch["linear_input"] = torch.rand(8, 120, config.audio["fft_size"] // 2 + 1).to(device)
|
||||
batch["mel_input"] = torch.rand(8, 120, config.audio["num_mels"]).to(device)
|
||||
batch["mel_lengths"] = torch.randint(20, 120, (8,)).long().to(device)
|
||||
batch["mel_lengths"] = torch.sort(batch["mel_lengths"], descending=True)[0]
|
||||
batch["mel_lengths"][0] = 120
|
||||
batch["stop_targets"] = torch.zeros(8, 120, 1).float().to(device)
|
||||
batch["stop_target_lengths"] = torch.randint(0, 120, (8,)).to(device)
|
||||
batch["speaker_ids"] = torch.randint(0, 5, (8,)).long().to(device)
|
||||
batch["d_vectors"] = None
|
||||
|
||||
for idx in batch["mel_lengths"]:
|
||||
batch["stop_targets"][:, int(idx.item()) :, 0] = 1.0
|
||||
|
||||
batch["stop_targets"] = batch["stop_targets"].view(
|
||||
batch["text_input"].shape[0], batch["stop_targets"].size(1) // config.r, -1
|
||||
)
|
||||
batch["stop_targets"] = (batch["stop_targets"].sum(2) > 0.0).unsqueeze(2).float().squeeze()
|
||||
|
||||
model = Tacotron(config).to(device)
|
||||
criterion = model.get_criterion()
|
||||
optimizer = model.get_optimizer()
|
||||
model.train()
|
||||
print(" > Num parameters for Tacotron with Capacitron VAE model:%s" % (count_parameters(model)))
|
||||
model_ref = copy.deepcopy(model)
|
||||
count = 0
|
||||
for param, param_ref in zip(model.parameters(), model_ref.parameters()):
|
||||
assert (param - param_ref).sum() == 0, param
|
||||
count += 1
|
||||
for _ in range(10):
|
||||
_, loss_dict = model.train_step(batch, criterion)
|
||||
optimizer.zero_grad()
|
||||
loss_dict["capacitron_vae_beta_loss"].backward()
|
||||
optimizer.first_step()
|
||||
loss_dict["loss"].backward()
|
||||
optimizer.step()
|
||||
# check parameter changes
|
||||
count = 0
|
||||
for param, param_ref in zip(model.parameters(), model_ref.parameters()):
|
||||
# ignore pre-higway layer since it works conditional
|
||||
assert (param != param_ref).any(), "param {} with shape {} not updated!! \n{}\n{}".format(
|
||||
count, param.shape, param, param_ref
|
||||
)
|
||||
count += 1
|
||||
|
||||
|
||||
class SCGSTMultiSpeakeTacotronTrainTest(unittest.TestCase):
|
||||
@staticmethod
|
||||
def test_train_step():
|
||||
|
||||
Reference in New Issue
Block a user