From d2b6326b8bc9a4f7f9aabd0137e9f50b76b7d388 Mon Sep 17 00:00:00 2001 From: Edresson Date: Fri, 23 Apr 2021 07:54:39 -0300 Subject: [PATCH 01/87] change optimizer initialization for compatibility with Hifi-GAN official implementation --- TTS/bin/train_vocoder_gan.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/TTS/bin/train_vocoder_gan.py b/TTS/bin/train_vocoder_gan.py index 730506c1..59409ad0 100644 --- a/TTS/bin/train_vocoder_gan.py +++ b/TTS/bin/train_vocoder_gan.py @@ -5,6 +5,7 @@ import os import sys import time +import itertools import traceback from inspect import signature @@ -495,7 +496,11 @@ def main(args): # pylint: disable=redefined-outer-name optimizer_gen = getattr(torch.optim, c.optimizer) optimizer_gen = optimizer_gen(model_gen.parameters(), lr=c.lr_gen, **c.optimizer_params) optimizer_disc = getattr(torch.optim, c.optimizer) - optimizer_disc = optimizer_disc(model_disc.parameters(), lr=c.lr_disc, **c.optimizer_params) + + if c.discriminator_model == 'hifigan_discriminator': + optimizer_disc = optimizer_disc(itertools.chain(model_disc.msd.parameters(), model_disc.mpd.parameters()), lr=c.lr_disc, **c.optimizer_params) + else: + optimizer_disc = optimizer_disc(model_disc.parameters(), lr=c.lr_disc, **c.optimizer_params) # schedulers scheduler_gen = None From 8228091f928017575fc552a6d869c32fc1871ebf Mon Sep 17 00:00:00 2001 From: Edresson Date: Fri, 23 Apr 2021 14:17:46 -0300 Subject: [PATCH 02/87] add script for extraction of tts spectrograms --- TTS/bin/extract_tts_spectrograms.py | 281 ++++++++++++++++++++++++++++ TTS/bin/train_tacotron.py | 2 +- TTS/bin/train_vocoder_gan.py | 2 +- TTS/tts/models/glow_tts.py | 75 ++++++++ TTS/tts/utils/speakers.py | 7 +- 5 files changed, 362 insertions(+), 5 deletions(-) create mode 100755 TTS/bin/extract_tts_spectrograms.py mode change 100644 => 100755 TTS/bin/train_tacotron.py mode change 100644 => 100755 TTS/bin/train_vocoder_gan.py mode change 100644 => 100755 TTS/tts/models/glow_tts.py mode change 100644 => 100755 TTS/tts/utils/speakers.py diff --git a/TTS/bin/extract_tts_spectrograms.py b/TTS/bin/extract_tts_spectrograms.py new file mode 100755 index 00000000..1ba5b839 --- /dev/null +++ b/TTS/bin/extract_tts_spectrograms.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Extract Mel spectrograms with teacher forcing.""" + +import os +import argparse +import numpy as np +from tqdm import tqdm +import torch + +from torch.utils.data import DataLoader + +from TTS.tts.datasets.preprocess import load_meta_data +from TTS.tts.datasets.TTSDataset import MyDataset +from TTS.tts.utils.generic_utils import setup_model +from TTS.tts.utils.speakers import parse_speakers +from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols +from TTS.utils.io import load_config +from TTS.utils.audio import AudioProcessor +from TTS.utils.generic_utils import count_parameters + +use_cuda = torch.cuda.is_available() + +def setup_loader(ap, r, verbose=False): + dataset = MyDataset( + r, + c.text_cleaner, + compute_linear_spec=False, + meta_data=meta_data, + ap=ap, + tp=c.characters if "characters" in c.keys() else None, + add_blank=c["add_blank"] if "add_blank" in c.keys() else False, + batch_group_size=0, + min_seq_len=c.min_seq_len, + max_seq_len=c.max_seq_len, + phoneme_cache_path=c.phoneme_cache_path, + use_phonemes=c.use_phonemes, + phoneme_language=c.phoneme_language, + enable_eos_bos=c.enable_eos_bos_chars, + use_noise_augment=False, + verbose=verbose, + speaker_mapping=speaker_mapping + if c.use_speaker_embedding and c.use_external_speaker_embedding_file + else None, + ) + + if c.use_phonemes and c.compute_input_seq_cache: + # precompute phonemes to have a better estimate of sequence lengths. + dataset.compute_input_seq(c.num_loader_workers) + dataset.sort_items() + + loader = DataLoader( + dataset, + batch_size=c.batch_size, + shuffle=False, + collate_fn=dataset.collate_fn, + drop_last=False, + sampler=None, + num_workers=c.num_loader_workers, + pin_memory=False, + ) + return loader + +def set_filename(wav_path, out_path): + wav_file = os.path.basename(wav_path) + file_name = wav_file.split('.')[0] + os.makedirs(os.path.join(out_path, "quant"), exist_ok=True) + os.makedirs(os.path.join(out_path, "mel"), exist_ok=True) + os.makedirs(os.path.join(out_path, "wav_gl"), exist_ok=True) + wavq_path = os.path.join(out_path, "quant", file_name) + mel_path = os.path.join(out_path, "mel", file_name) + wav_path = os.path.join(out_path, "wav_gl", file_name+'.wav') + return file_name, wavq_path, mel_path, wav_path + +def format_data(data): + # setup input data + text_input = data[0] + text_lengths = data[1] + speaker_names = data[2] + mel_input = data[4] + mel_lengths = data[5] + item_idx = data[7] + attn_mask = data[9] + avg_text_length = torch.mean(text_lengths.float()) + avg_spec_length = torch.mean(mel_lengths.float()) + + if c.use_speaker_embedding: + if c.use_external_speaker_embedding_file: + speaker_embeddings = data[8] + speaker_ids = None + else: + speaker_ids = [speaker_mapping[speaker_name] for speaker_name in speaker_names] + speaker_ids = torch.LongTensor(speaker_ids) + speaker_embeddings = None + else: + speaker_embeddings = None + speaker_ids = None + + # dispatch data to GPU + if use_cuda: + text_input = text_input.cuda(non_blocking=True) + text_lengths = text_lengths.cuda(non_blocking=True) + mel_input = mel_input.cuda(non_blocking=True) + mel_lengths = mel_lengths.cuda(non_blocking=True) + if speaker_ids is not None: + speaker_ids = speaker_ids.cuda(non_blocking=True) + if speaker_embeddings is not None: + speaker_embeddings = speaker_embeddings.cuda(non_blocking=True) + + if attn_mask is not None: + attn_mask = attn_mask.cuda(non_blocking=True) + return ( + text_input, + text_lengths, + mel_input, + mel_lengths, + speaker_ids, + speaker_embeddings, + avg_text_length, + avg_spec_length, + attn_mask, + item_idx, + ) + +@torch.no_grad() +def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=False, debug=False, metada_name="metada.txt"): + model.eval() + export_metadata = [] + for _, data in tqdm(enumerate(data_loader), total=len(data_loader)): + + # format data + ( + text_input, + text_lengths, + mel_input, + mel_lengths, + speaker_ids, + speaker_embeddings, + _, + _, + attn_mask, + item_idx, + ) = format_data(data) + + if c.model.lower() == "glow_tts": + mel_input = mel_input.permute(0, 2, 1) # B x D x T + speaker_c = None + if speaker_ids is not None: + speaker_c = speaker_ids + elif speaker_embeddings is not None: + speaker_c = speaker_embeddings + + model_output, _, _, _, _, _, _ = model.inference_with_MAS( + text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c + ) + model_output = model_output.transpose(1, 2).detach().cpu().numpy() + + elif "tacotron" in c.model.lower(): + if c.bidirectional_decoder or c.double_decoder_consistency: + ( + _, + postnet_outputs, + _, + _, + _, + _, + ) = model( + text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings + ) + else: + _, postnet_outputs, _, _ = model( + text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings + ) + # normalize tacotron output + if c.model.lower() == "tacotron": + mel_specs = [] + postnet_outputs = postnet_outputs.data.cpu().numpy() + for b in range(postnet_outputs.shape[0]): + postnet_output = postnet_outputs[b] + mel_specs.append(torch.FloatTensor(ap.out_linear_to_mel(postnet_output.T).T).cuda()) + model_output = torch.stack(mel_specs) + + elif c.model.lower() == "tacotron2": + model_output = postnet_outputs.detach().cpu().numpy() + + for idx in range(text_input.shape[0]): + wav_file_path = item_idx[idx] + wav = ap.load_wav(wav_file_path) + _, wavq_path, mel_path, wav_path = set_filename(wav_file_path, output_path) + + # quantize and save wav + if quantized_wav: + wavq = ap.quantize(wav) + np.save(wavq_path, wavq) + + # save TTS mel + mel = model_output[idx] + mel_length = mel_lengths[idx] + mel = mel[:mel_length, :].T + np.save(mel_path, mel) + + export_metadata.append([wav_file_path, mel_path]) + + if debug: + print("Audio for debug saved at:", wav_path) + wav = ap.inv_melspectrogram(mel) + ap.save_wav(wav, wav_path) + + with open(os.path.join(output_path, metada_name), "w") as f: + for data in export_metadata: + f.write(f"{data[0]}|{data[1]+'.npy'}\n") + +def main(args): # pylint: disable=redefined-outer-name + # pylint: disable=global-variable-undefined + global meta_data, symbols, phonemes, model_characters, speaker_mapping + + # Audio processor + ap = AudioProcessor(**c.audio) + if "characters" in c.keys(): + symbols, phonemes = make_symbols(**c.characters) + + # set model characters + model_characters = phonemes if c.use_phonemes else symbols + num_chars = len(model_characters) + + # load data instances + meta_data_train, meta_data_eval = load_meta_data(c.datasets) + + # use eval and training partitions + meta_data = meta_data_train + meta_data_eval + + # parse speakers + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, None) + + # setup model + model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim) + + # restore model + checkpoint = torch.load(args.checkpoint_path, map_location="cpu") + model.load_state_dict(checkpoint["model"]) + + if use_cuda: + model.cuda() + + num_params = count_parameters(model) + print("\n > Model has {} parameters".format(num_params), flush=True) + # set r + r = 1 if c.model.lower() == "glow_tts" else model.decoder.r + own_loader = setup_loader(ap, r, verbose=True) + + extract_spectrograms(own_loader, model, ap, args.output_path, quantized_wav=args.quantized, debug=args.debug, metada_name="metada.txt") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + '--config_path', + type=str, + help='Path to config file for training.', + required=True) + parser.add_argument( + '--checkpoint_path', + type=str, + help='Model file to be restored.', + required=True) + parser.add_argument( + '--output_path', + type=str, + help='Path to save mel specs', + required=True) + parser.add_argument('--debug', + default=False, + action='store_true', + help='Save audio files for debug') + parser.add_argument('--quantized', + action='store_true', + help='Save quantized audio files') + args = parser.parse_args() + + c = load_config(args.config_path) + + main(args) diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py old mode 100644 new mode 100755 index c8346c3a..9bfa4296 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -117,7 +117,7 @@ def format_data(data): text_lengths = text_lengths.cuda(non_blocking=True) mel_input = mel_input.cuda(non_blocking=True) mel_lengths = mel_lengths.cuda(non_blocking=True) - linear_input = linear_input.cuda(non_blocking=True) if c.model in ["Tacotron"] else None + linear_input = linear_input.cuda(non_blocking=True) if c.model.lower() in ["tacotron"] else None stop_targets = stop_targets.cuda(non_blocking=True) if speaker_ids is not None: speaker_ids = speaker_ids.cuda(non_blocking=True) diff --git a/TTS/bin/train_vocoder_gan.py b/TTS/bin/train_vocoder_gan.py old mode 100644 new mode 100755 index 59409ad0..f33df3e8 --- a/TTS/bin/train_vocoder_gan.py +++ b/TTS/bin/train_vocoder_gan.py @@ -497,7 +497,7 @@ def main(args): # pylint: disable=redefined-outer-name optimizer_gen = optimizer_gen(model_gen.parameters(), lr=c.lr_gen, **c.optimizer_params) optimizer_disc = getattr(torch.optim, c.optimizer) - if c.discriminator_model == 'hifigan_discriminator': + if c.discriminator_model == 'hifigan_discriminator': optimizer_disc = optimizer_disc(itertools.chain(model_disc.msd.parameters(), model_disc.mpd.parameters()), lr=c.lr_disc, **c.optimizer_params) else: optimizer_disc = optimizer_disc(model_disc.parameters(), lr=c.lr_disc, **c.optimizer_params) diff --git a/TTS/tts/models/glow_tts.py b/TTS/tts/models/glow_tts.py old mode 100644 new mode 100755 index 0717e2a8..fddd94cf --- a/TTS/tts/models/glow_tts.py +++ b/TTS/tts/models/glow_tts.py @@ -176,6 +176,81 @@ class GlowTTS(nn.Module): attn = attn.squeeze(1).permute(0, 2, 1) return z, logdet, y_mean, y_log_scale, attn, o_dur_log, o_attn_dur + @torch.no_grad() + def inference_with_MAS(self, x, x_lengths, y=None, y_lengths=None, attn=None, g=None): + """ + It's similar to the teacher forcing in Tacotron. + It was proposed in: https://arxiv.org/abs/2104.05557 + Shapes: + x: [B, T] + x_lenghts: B + y: [B, C, T] + y_lengths: B + g: [B, C] or B + """ + y_max_length = y.size(2) + # norm speaker embeddings + if g is not None: + if self.external_speaker_embedding_dim: + g = F.normalize(g).unsqueeze(-1) + else: + g = F.normalize(self.emb_g(g)).unsqueeze(-1) # [b, h, 1] + + # embedding pass + o_mean, o_log_scale, o_dur_log, x_mask = self.encoder(x, x_lengths, g=g) + # drop redisual frames wrt num_squeeze and set y_lengths. + y, y_lengths, y_max_length, attn = self.preprocess(y, y_lengths, y_max_length, None) + # create masks + y_mask = torch.unsqueeze(sequence_mask(y_lengths, y_max_length), 1).to(x_mask.dtype) + attn_mask = torch.unsqueeze(x_mask, -1) * torch.unsqueeze(y_mask, 2) + # decoder pass + z, logdet = self.decoder(y, y_mask, g=g, reverse=False) + # find the alignment path between z and encoder output + o_scale = torch.exp(-2 * o_log_scale) + logp1 = torch.sum(-0.5 * math.log(2 * math.pi) - o_log_scale, [1]).unsqueeze(-1) # [b, t, 1] + logp2 = torch.matmul(o_scale.transpose(1, 2), -0.5 * (z ** 2)) # [b, t, d] x [b, d, t'] = [b, t, t'] + logp3 = torch.matmul((o_mean * o_scale).transpose(1, 2), z) # [b, t, d] x [b, d, t'] = [b, t, t'] + logp4 = torch.sum(-0.5 * (o_mean ** 2) * o_scale, [1]).unsqueeze(-1) # [b, t, 1] + logp = logp1 + logp2 + logp3 + logp4 # [b, t, t'] + attn = maximum_path(logp, attn_mask.squeeze(1)).unsqueeze(1).detach() + + y_mean, y_log_scale, o_attn_dur = self.compute_outputs(attn, o_mean, o_log_scale, x_mask) + attn = attn.squeeze(1).permute(0, 2, 1) + + # get predited aligned distribution + z = y_mean * y_mask + + # reverse the decoder and predict using the aligned distribution + y, logdet = self.decoder(z, y_mask, g=g, reverse=True) + + return y, logdet, y_mean, y_log_scale, attn, o_dur_log, o_attn_dur + + @torch.no_grad() + def decoder_inference(self, y, y_lengths=None, g=None): + """ + Shapes: + y: [B, C, T] + y_lengths: B + g: [B, C] or B + """ + y_max_length = y.size(2) + # norm speaker embeddings + if g is not None: + if self.external_speaker_embedding_dim: + g = F.normalize(g).unsqueeze(-1) + else: + g = F.normalize(self.emb_g(g)).unsqueeze(-1) # [b, h, 1] + + y_mask = torch.unsqueeze(sequence_mask(y_lengths, y_max_length), 1).to(y.dtype) + + # decoder pass + z, logdet = self.decoder(y, y_mask, g=g, reverse=False) + + # reverse decoder and predict + y, logdet = self.decoder(z, y_mask, g=g, reverse=True) + + return y, logdet + @torch.no_grad() def inference(self, x, x_lengths, g=None): if g is not None: diff --git a/TTS/tts/utils/speakers.py b/TTS/tts/utils/speakers.py old mode 100644 new mode 100755 index cb2827fd..07061b81 --- a/TTS/tts/utils/speakers.py +++ b/TTS/tts/utils/speakers.py @@ -22,9 +22,10 @@ def load_speaker_mapping(out_path): def save_speaker_mapping(out_path, speaker_mapping): """Saves speaker mapping if not yet present.""" - speakers_json_path = make_speakers_json_path(out_path) - with open(speakers_json_path, "w") as f: - json.dump(speaker_mapping, f, indent=4) + if out_path is not None: + speakers_json_path = make_speakers_json_path(out_path) + with open(speakers_json_path, "w") as f: + json.dump(speaker_mapping, f, indent=4) def get_speakers(items): From 20e42a3381e6bf09d92a0acd56feb1dde79ada21 Mon Sep 17 00:00:00 2001 From: Edresson Date: Fri, 23 Apr 2021 15:00:00 -0300 Subject: [PATCH 03/87] add save audio option --- TTS/bin/extract_tts_spectrograms.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/TTS/bin/extract_tts_spectrograms.py b/TTS/bin/extract_tts_spectrograms.py index 1ba5b839..517a281e 100755 --- a/TTS/bin/extract_tts_spectrograms.py +++ b/TTS/bin/extract_tts_spectrograms.py @@ -66,10 +66,12 @@ def set_filename(wav_path, out_path): os.makedirs(os.path.join(out_path, "quant"), exist_ok=True) os.makedirs(os.path.join(out_path, "mel"), exist_ok=True) os.makedirs(os.path.join(out_path, "wav_gl"), exist_ok=True) + os.makedirs(os.path.join(out_path, "wav"), exist_ok=True) wavq_path = os.path.join(out_path, "quant", file_name) mel_path = os.path.join(out_path, "mel", file_name) - wav_path = os.path.join(out_path, "wav_gl", file_name+'.wav') - return file_name, wavq_path, mel_path, wav_path + wav_gl_path = os.path.join(out_path, "wav_gl", file_name+'.wav') + wav_path = os.path.join(out_path, "wav", file_name+'.wav') + return file_name, wavq_path, mel_path, wav_gl_path, wav_path def format_data(data): # setup input data @@ -122,7 +124,7 @@ def format_data(data): ) @torch.no_grad() -def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=False, debug=False, metada_name="metada.txt"): +def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=False, save_audio=False, debug=False, metada_name="metada.txt"): model.eval() export_metadata = [] for _, data in tqdm(enumerate(data_loader), total=len(data_loader)): @@ -185,7 +187,7 @@ def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=Fals for idx in range(text_input.shape[0]): wav_file_path = item_idx[idx] wav = ap.load_wav(wav_file_path) - _, wavq_path, mel_path, wav_path = set_filename(wav_file_path, output_path) + _, wavq_path, mel_path, wav_gl_path, wav_path = set_filename(wav_file_path, output_path) # quantize and save wav if quantized_wav: @@ -199,11 +201,13 @@ def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=Fals np.save(mel_path, mel) export_metadata.append([wav_file_path, mel_path]) + if save_audio: + ap.save_wav(wav, wav_path) if debug: - print("Audio for debug saved at:", wav_path) + print("Audio for debug saved at:", wav_gl_path) wav = ap.inv_melspectrogram(mel) - ap.save_wav(wav, wav_path) + ap.save_wav(wav, wav_gl_path) with open(os.path.join(output_path, metada_name), "w") as f: for data in export_metadata: @@ -247,7 +251,7 @@ def main(args): # pylint: disable=redefined-outer-name r = 1 if c.model.lower() == "glow_tts" else model.decoder.r own_loader = setup_loader(ap, r, verbose=True) - extract_spectrograms(own_loader, model, ap, args.output_path, quantized_wav=args.quantized, debug=args.debug, metada_name="metada.txt") + extract_spectrograms(own_loader, model, ap, args.output_path, quantized_wav=args.quantized, save_audio=args.save_audio, debug=args.debug, metada_name="metada.txt") if __name__ == "__main__": @@ -271,6 +275,10 @@ if __name__ == "__main__": default=False, action='store_true', help='Save audio files for debug') + parser.add_argument('--save_audio', + default=False, + action='store_true', + help='Save audio files') parser.add_argument('--quantized', action='store_true', help='Save quantized audio files') From 446b1da93622db3efd5c43327a177584c29d7450 Mon Sep 17 00:00:00 2001 From: Edresson Date: Thu, 29 Apr 2021 18:18:37 -0300 Subject: [PATCH 04/87] create inference function --- TTS/bin/extract_tts_spectrograms.py | 84 +++++++++++++++-------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/TTS/bin/extract_tts_spectrograms.py b/TTS/bin/extract_tts_spectrograms.py index 517a281e..1a77e45a 100755 --- a/TTS/bin/extract_tts_spectrograms.py +++ b/TTS/bin/extract_tts_spectrograms.py @@ -124,6 +124,49 @@ def format_data(data): ) @torch.no_grad() +def inference(model_name, model, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask=None, speaker_ids=None, speaker_embeddings=None): + if model_name == "glow_tts": + mel_input = mel_input.permute(0, 2, 1) # B x D x T + speaker_c = None + if speaker_ids is not None: + speaker_c = speaker_ids + elif speaker_embeddings is not None: + speaker_c = speaker_embeddings + + model_output, _, _, _, _, _, _ = model.inference_with_MAS( + text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c + ) + model_output = model_output.transpose(1, 2).detach().cpu().numpy() + + elif "tacotron" in model_name: + if c.bidirectional_decoder or c.double_decoder_consistency: + ( + _, + postnet_outputs, + _, + _, + _, + _, + ) = model( + text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings + ) + else: + _, postnet_outputs, _, _ = model( + text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings + ) + # normalize tacotron output + if model_name == "tacotron": + mel_specs = [] + postnet_outputs = postnet_outputs.data.cpu().numpy() + for b in range(postnet_outputs.shape[0]): + postnet_output = postnet_outputs[b] + mel_specs.append(torch.FloatTensor(ap.out_linear_to_mel(postnet_output.T).T).cuda()) + model_output = torch.stack(mel_specs) + + elif model_name == "tacotron2": + model_output = postnet_outputs.detach().cpu().numpy() + return model_output + def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=False, save_audio=False, debug=False, metada_name="metada.txt"): model.eval() export_metadata = [] @@ -143,46 +186,7 @@ def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=Fals item_idx, ) = format_data(data) - if c.model.lower() == "glow_tts": - mel_input = mel_input.permute(0, 2, 1) # B x D x T - speaker_c = None - if speaker_ids is not None: - speaker_c = speaker_ids - elif speaker_embeddings is not None: - speaker_c = speaker_embeddings - - model_output, _, _, _, _, _, _ = model.inference_with_MAS( - text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c - ) - model_output = model_output.transpose(1, 2).detach().cpu().numpy() - - elif "tacotron" in c.model.lower(): - if c.bidirectional_decoder or c.double_decoder_consistency: - ( - _, - postnet_outputs, - _, - _, - _, - _, - ) = model( - text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings - ) - else: - _, postnet_outputs, _, _ = model( - text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings - ) - # normalize tacotron output - if c.model.lower() == "tacotron": - mel_specs = [] - postnet_outputs = postnet_outputs.data.cpu().numpy() - for b in range(postnet_outputs.shape[0]): - postnet_output = postnet_outputs[b] - mel_specs.append(torch.FloatTensor(ap.out_linear_to_mel(postnet_output.T).T).cuda()) - model_output = torch.stack(mel_specs) - - elif c.model.lower() == "tacotron2": - model_output = postnet_outputs.detach().cpu().numpy() + model_output = inference(c.model.lower(), model, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask, speaker_ids, speaker_embeddings) for idx in range(text_input.shape[0]): wav_file_path = item_idx[idx] From bb82f4ae8b3fc7c7aeae80ac1ecdcf73153bab67 Mon Sep 17 00:00:00 2001 From: Edresson Date: Thu, 29 Apr 2021 19:39:09 -0300 Subject: [PATCH 05/87] add unit test for GlowTTS inference with MAS --- tests/test_glow_tts.py | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_glow_tts.py b/tests/test_glow_tts.py index 8e699faf..e71c167a 100644 --- a/tests/test_glow_tts.py +++ b/tests/test_glow_tts.py @@ -129,3 +129,58 @@ class GlowTTSTrainTest(unittest.TestCase): count, param.shape, param, param_ref ) count += 1 + +class GlowTTSInferenceTest(unittest.TestCase): + @staticmethod + def test_inference(): + input_dummy = torch.randint(0, 24, (8, 128)).long().to(device) + input_lengths = torch.randint(100, 129, (8,)).long().to(device) + input_lengths[-1] = 128 + mel_spec = torch.rand(8, c.audio["num_mels"], 30).to(device) + mel_lengths = torch.randint(20, 30, (8,)).long().to(device) + speaker_ids = torch.randint(0, 5, (8,)).long().to(device) + + # create model + model = GlowTTS( + num_chars=32, + hidden_channels_enc=48, + hidden_channels_dec=48, + hidden_channels_dp=32, + out_channels=80, + encoder_type="rel_pos_transformer", + encoder_params={ + "kernel_size": 3, + "dropout_p": 0.1, + "num_layers": 6, + "num_heads": 2, + "hidden_channels_ffn": 16, # 4 times the hidden_channels + "input_length": None, + }, + use_encoder_prenet=True, + num_flow_blocks_dec=12, + kernel_size_dec=5, + dilation_rate=1, + num_block_layers=4, + dropout_p_dec=0.0, + num_speakers=0, + c_in_channels=0, + num_splits=4, + num_squeeze=1, + sigmoid_scale=False, + mean_only=False, + ).to(device) + + model.eval() + print(" > Num parameters for GlowTTS model:%s" % (count_parameters(model))) + + # inference encoder and decoder with MAS + y, _, _, _, _, _, _ = model.inference_with_MAS( + input_dummy, input_lengths, mel_spec, mel_lengths, None + ) + + y_dec, _ = model.decoder_inference(mel_spec, mel_lengths + ) + + assert (y_dec.shape == y.shape), "Difference between the shapes of the glowTTS inference with MAS ({}) and the inference using only the decoder ({}) !!".format( + y.shape, y_dec.shape + ) From 3ecd556bbecb44feed757550ddb67b47903965bf Mon Sep 17 00:00:00 2001 From: Edresson Date: Sat, 1 May 2021 13:41:56 -0300 Subject: [PATCH 06/87] add unit test for extract tts spectrograms script --- TTS/bin/extract_tts_spectrograms.py | 6 +- tests/test_extract_tts_spectrograms.py | 85 ++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 tests/test_extract_tts_spectrograms.py diff --git a/TTS/bin/extract_tts_spectrograms.py b/TTS/bin/extract_tts_spectrograms.py index 1a77e45a..5e230bbd 100755 --- a/TTS/bin/extract_tts_spectrograms.py +++ b/TTS/bin/extract_tts_spectrograms.py @@ -124,7 +124,7 @@ def format_data(data): ) @torch.no_grad() -def inference(model_name, model, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask=None, speaker_ids=None, speaker_embeddings=None): +def inference(model_name, model, config, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask=None, speaker_ids=None, speaker_embeddings=None): if model_name == "glow_tts": mel_input = mel_input.permute(0, 2, 1) # B x D x T speaker_c = None @@ -139,7 +139,7 @@ def inference(model_name, model, ap, text_input, text_lengths, mel_input, mel_le model_output = model_output.transpose(1, 2).detach().cpu().numpy() elif "tacotron" in model_name: - if c.bidirectional_decoder or c.double_decoder_consistency: + if config.bidirectional_decoder or config.double_decoder_consistency: ( _, postnet_outputs, @@ -186,7 +186,7 @@ def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=Fals item_idx, ) = format_data(data) - model_output = inference(c.model.lower(), model, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask, speaker_ids, speaker_embeddings) + model_output = inference(c.model.lower(), model, c, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask, speaker_ids, speaker_embeddings) for idx in range(text_input.shape[0]): wav_file_path = item_idx[idx] diff --git a/tests/test_extract_tts_spectrograms.py b/tests/test_extract_tts_spectrograms.py new file mode 100644 index 00000000..41e52229 --- /dev/null +++ b/tests/test_extract_tts_spectrograms.py @@ -0,0 +1,85 @@ +import os +import unittest + +import torch + +from tests import get_tests_input_path + +from TTS.tts.models.tacotron2 import Tacotron2 +from TTS.tts.models.glow_tts import GlowTTS + +from TTS.utils.audio import AudioProcessor +from TTS.utils.io import load_config + +from TTS.bin.extract_tts_spectrograms import inference + +torch.manual_seed(1) +use_cuda = torch.cuda.is_available() +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +# set params from tacotron inference +c.bidirectional_decoder = False +c.double_decoder_consistency = False +ap = AudioProcessor(**c.audio) + + +# pylint: disable=protected-access +class TestExtractTTSSpectrograms(unittest.TestCase): + @staticmethod + def test_GlowTTS(): + input_dummy = torch.randint(0, 24, (8, 128)).long().to(device) + input_lengths = torch.randint(100, 129, (8,)).long().to(device) + input_lengths[-1] = 128 + mel_spec = torch.rand(8, c.audio["num_mels"], 30).to(device) + mel_lengths = torch.randint(20, 30, (8,)).long().to(device) + + # create model + model = GlowTTS( + num_chars=32, + hidden_channels_enc=48, + hidden_channels_dec=48, + hidden_channels_dp=32, + out_channels=c.audio["num_mels"], + encoder_type="rel_pos_transformer", + encoder_params={ + "kernel_size": 3, + "dropout_p": 0.1, + "num_layers": 6, + "num_heads": 2, + "hidden_channels_ffn": 16, # 4 times the hidden_channels + "input_length": None, + }, + use_encoder_prenet=True, + num_flow_blocks_dec=12, + kernel_size_dec=5, + dilation_rate=1, + num_block_layers=4, + dropout_p_dec=0.0, + num_speakers=0, + c_in_channels=0, + num_splits=4, + num_squeeze=1, + sigmoid_scale=False, + mean_only=False, + ).to(device) + + model.eval() + _ = inference('glow_tts', model, c, ap, input_dummy, input_lengths, mel_spec.permute(0, 2, 1), mel_lengths) + print("GlowTTS extract tts spectrograms ok !") + + @staticmethod + def test_Tacotron(): + input_dummy = torch.randint(0, 24, (8, 128)).long().to(device) + input_lengths = torch.randint(100, 128, (8,)).long().to(device) + input_lengths = torch.sort(input_lengths, descending=True)[0] + mel_spec = torch.rand(8, 30, c.audio["num_mels"]).to(device) + mel_lengths = torch.randint(20, 30, (8,)).long().to(device) + mel_lengths[0] = 30 + + # create model + model = Tacotron2(num_chars=24, decoder_output_dim=c.audio["num_mels"], r=c.r, num_speakers=1).to(device) + model.eval() + + _ = inference('tacotron2', model, c, ap, input_dummy, input_lengths, mel_spec, mel_lengths) + print("Tacotron extract tts spectrograms ok !") From 501c8e03020d87c753cd80778984e21a758cf9b4 Mon Sep 17 00:00:00 2001 From: Edresson Date: Tue, 4 May 2021 19:04:13 -0300 Subject: [PATCH 07/87] remove unused vars on extract tts spectrograms script --- TTS/bin/extract_tts_spectrograms.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/TTS/bin/extract_tts_spectrograms.py b/TTS/bin/extract_tts_spectrograms.py index 5e230bbd..d5c23ccd 100755 --- a/TTS/bin/extract_tts_spectrograms.py +++ b/TTS/bin/extract_tts_spectrograms.py @@ -124,7 +124,7 @@ def format_data(data): ) @torch.no_grad() -def inference(model_name, model, config, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask=None, speaker_ids=None, speaker_embeddings=None): +def inference(model_name, model, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask=None, speaker_ids=None, speaker_embeddings=None): if model_name == "glow_tts": mel_input = mel_input.permute(0, 2, 1) # B x D x T speaker_c = None @@ -133,35 +133,22 @@ def inference(model_name, model, config, ap, text_input, text_lengths, mel_input elif speaker_embeddings is not None: speaker_c = speaker_embeddings - model_output, _, _, _, _, _, _ = model.inference_with_MAS( + model_output, *_ = model.inference_with_MAS( text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c ) model_output = model_output.transpose(1, 2).detach().cpu().numpy() elif "tacotron" in model_name: - if config.bidirectional_decoder or config.double_decoder_consistency: - ( - _, - postnet_outputs, - _, - _, - _, - _, - ) = model( - text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings - ) - else: - _, postnet_outputs, _, _ = model( - text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings - ) + _, postnet_outputs, *_ = model( + text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings) # normalize tacotron output if model_name == "tacotron": mel_specs = [] postnet_outputs = postnet_outputs.data.cpu().numpy() for b in range(postnet_outputs.shape[0]): postnet_output = postnet_outputs[b] - mel_specs.append(torch.FloatTensor(ap.out_linear_to_mel(postnet_output.T).T).cuda()) - model_output = torch.stack(mel_specs) + mel_specs.append(torch.FloatTensor(ap.out_linear_to_mel(postnet_output.T).T)) + model_output = torch.stack(mel_specs).cpu().numpy() elif model_name == "tacotron2": model_output = postnet_outputs.detach().cpu().numpy() @@ -186,7 +173,7 @@ def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=Fals item_idx, ) = format_data(data) - model_output = inference(c.model.lower(), model, c, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask, speaker_ids, speaker_embeddings) + model_output = inference(c.model.lower(), model, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask, speaker_ids, speaker_embeddings) for idx in range(text_input.shape[0]): wav_file_path = item_idx[idx] From e3f56b613b114e74fd35c47c0912e8139790747f Mon Sep 17 00:00:00 2001 From: Edresson Date: Tue, 4 May 2021 20:45:07 -0300 Subject: [PATCH 08/87] update unit test for extract tts spectrograms script --- tests/test_extract_tts_spectrograms.py | 111 ++++++++++--------------- 1 file changed, 46 insertions(+), 65 deletions(-) diff --git a/tests/test_extract_tts_spectrograms.py b/tests/test_extract_tts_spectrograms.py index 41e52229..618d7b64 100644 --- a/tests/test_extract_tts_spectrograms.py +++ b/tests/test_extract_tts_spectrograms.py @@ -5,81 +5,62 @@ import torch from tests import get_tests_input_path -from TTS.tts.models.tacotron2 import Tacotron2 -from TTS.tts.models.glow_tts import GlowTTS +from tests import get_tests_output_path, run_cli + +from TTS.tts.utils.generic_utils import setup_model -from TTS.utils.audio import AudioProcessor from TTS.utils.io import load_config - -from TTS.bin.extract_tts_spectrograms import inference +from TTS.tts.utils.text.symbols import phonemes, symbols torch.manual_seed(1) -use_cuda = torch.cuda.is_available() -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) -# set params from tacotron inference -c.bidirectional_decoder = False -c.double_decoder_consistency = False -ap = AudioProcessor(**c.audio) - # pylint: disable=protected-access class TestExtractTTSSpectrograms(unittest.TestCase): @staticmethod def test_GlowTTS(): - input_dummy = torch.randint(0, 24, (8, 128)).long().to(device) - input_lengths = torch.randint(100, 129, (8,)).long().to(device) - input_lengths[-1] = 128 - mel_spec = torch.rand(8, c.audio["num_mels"], 30).to(device) - mel_lengths = torch.randint(20, 30, (8,)).long().to(device) - + # set paths + config_path = os.path.join(get_tests_input_path(), "test_glow_tts.json") + checkpoint_path = os.path.join(get_tests_output_path(), 'checkpoint_test.pth.tar') + output_path = os.path.join(get_tests_output_path(), 'output_extract_tts_spectrograms/') + # load config + c = load_config(config_path) # create model - model = GlowTTS( - num_chars=32, - hidden_channels_enc=48, - hidden_channels_dec=48, - hidden_channels_dp=32, - out_channels=c.audio["num_mels"], - encoder_type="rel_pos_transformer", - encoder_params={ - "kernel_size": 3, - "dropout_p": 0.1, - "num_layers": 6, - "num_heads": 2, - "hidden_channels_ffn": 16, # 4 times the hidden_channels - "input_length": None, - }, - use_encoder_prenet=True, - num_flow_blocks_dec=12, - kernel_size_dec=5, - dilation_rate=1, - num_block_layers=4, - dropout_p_dec=0.0, - num_speakers=0, - c_in_channels=0, - num_splits=4, - num_squeeze=1, - sigmoid_scale=False, - mean_only=False, - ).to(device) - - model.eval() - _ = inference('glow_tts', model, c, ap, input_dummy, input_lengths, mel_spec.permute(0, 2, 1), mel_lengths) - print("GlowTTS extract tts spectrograms ok !") - + num_chars = len(phonemes if c.use_phonemes else symbols) + model = setup_model(num_chars, 1, c, speaker_embedding_dim=None) + # save model + torch.save({"model": model.state_dict()}, checkpoint_path) + # run test + run_cli(f'CUDA_VISIBLE_DEVICES="" python3 TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') + @staticmethod + def test_Tacotron2(): + # set paths + config_path = os.path.join(get_tests_input_path(), "test_tacotron2_config.json") + checkpoint_path = os.path.join(get_tests_output_path(), 'checkpoint_test.pth.tar') + output_path = os.path.join(get_tests_output_path(), 'output_extract_tts_spectrograms/') + # load config + c = load_config(config_path) + # create model + num_chars = len(phonemes if c.use_phonemes else symbols) + model = setup_model(num_chars, 1, c, speaker_embedding_dim=None) + # save model + torch.save({"model": model.state_dict()}, checkpoint_path) + # run test + run_cli(f'CUDA_VISIBLE_DEVICES="" python3 TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') @staticmethod def test_Tacotron(): - input_dummy = torch.randint(0, 24, (8, 128)).long().to(device) - input_lengths = torch.randint(100, 128, (8,)).long().to(device) - input_lengths = torch.sort(input_lengths, descending=True)[0] - mel_spec = torch.rand(8, 30, c.audio["num_mels"]).to(device) - mel_lengths = torch.randint(20, 30, (8,)).long().to(device) - mel_lengths[0] = 30 - + # set paths + config_path = os.path.join(get_tests_input_path(), "test_tacotron_config.json") + checkpoint_path = os.path.join(get_tests_output_path(), 'checkpoint_test.pth.tar') + output_path = os.path.join(get_tests_output_path(), 'output_extract_tts_spectrograms/') + # load config + c = load_config(config_path) # create model - model = Tacotron2(num_chars=24, decoder_output_dim=c.audio["num_mels"], r=c.r, num_speakers=1).to(device) - model.eval() - - _ = inference('tacotron2', model, c, ap, input_dummy, input_lengths, mel_spec, mel_lengths) - print("Tacotron extract tts spectrograms ok !") + num_chars = len(phonemes if c.use_phonemes else symbols) + model = setup_model(num_chars, 1, c, speaker_embedding_dim=None) + # save model + torch.save({"model": model.state_dict()}, checkpoint_path) + # run test + run_cli(f'CUDA_VISIBLE_DEVICES="" python3 TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') From d78f27ea41ec6406e9369c59c935bd05449fe515 Mon Sep 17 00:00:00 2001 From: Edresson Date: Wed, 5 May 2021 06:38:01 -0300 Subject: [PATCH 09/87] bugfix on tacotron unit test --- tests/test_tacotron_model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_tacotron_model.py b/tests/test_tacotron_model.py index e3ed8ae2..72b47d23 100644 --- a/tests/test_tacotron_model.py +++ b/tests/test_tacotron_model.py @@ -37,6 +37,7 @@ class TacotronTrainTest(unittest.TestCase): mel_spec = torch.rand(8, 30, c.audio["num_mels"]).to(device) linear_spec = torch.rand(8, 30, c.audio["fft_size"]).to(device) mel_lengths = torch.randint(20, 30, (8,)).long().to(device) + mel_lengths[-1] = mel_spec.size(1) stop_targets = torch.zeros(8, 30, 1).float().to(device) speaker_ids = torch.randint(0, 5, (8,)).long().to(device) @@ -96,6 +97,7 @@ class MultiSpeakeTacotronTrainTest(unittest.TestCase): mel_spec = torch.rand(8, 30, c.audio["num_mels"]).to(device) linear_spec = torch.rand(8, 30, c.audio["fft_size"]).to(device) mel_lengths = torch.randint(20, 30, (8,)).long().to(device) + mel_lengths[-1] = mel_spec.size(1) stop_targets = torch.zeros(8, 30, 1).float().to(device) speaker_embeddings = torch.rand(8, 55).to(device) From 65860a954a9e9642a8c8f704cf3af31a04c8bb44 Mon Sep 17 00:00:00 2001 From: Edresson Date: Wed, 5 May 2021 07:15:36 -0300 Subject: [PATCH 10/87] remove unused vars on test glow tts --- tests/test_glow_tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_glow_tts.py b/tests/test_glow_tts.py index e71c167a..7e17ed45 100644 --- a/tests/test_glow_tts.py +++ b/tests/test_glow_tts.py @@ -174,7 +174,7 @@ class GlowTTSInferenceTest(unittest.TestCase): print(" > Num parameters for GlowTTS model:%s" % (count_parameters(model))) # inference encoder and decoder with MAS - y, _, _, _, _, _, _ = model.inference_with_MAS( + y, *_ = model.inference_with_MAS( input_dummy, input_lengths, mel_spec, mel_lengths, None ) From 070227d2ab551fd3a839afac4d97c454446bd913 Mon Sep 17 00:00:00 2001 From: Edresson Date: Thu, 6 May 2021 11:32:42 -0300 Subject: [PATCH 11/87] change python3 to python in the extract tts espectrograms script to avoid incompatibility --- tests/test_extract_tts_spectrograms.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_extract_tts_spectrograms.py b/tests/test_extract_tts_spectrograms.py index 618d7b64..65db9c0e 100644 --- a/tests/test_extract_tts_spectrograms.py +++ b/tests/test_extract_tts_spectrograms.py @@ -30,7 +30,7 @@ class TestExtractTTSSpectrograms(unittest.TestCase): # save model torch.save({"model": model.state_dict()}, checkpoint_path) # run test - run_cli(f'CUDA_VISIBLE_DEVICES="" python3 TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli(f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') @staticmethod def test_Tacotron2(): @@ -46,7 +46,7 @@ class TestExtractTTSSpectrograms(unittest.TestCase): # save model torch.save({"model": model.state_dict()}, checkpoint_path) # run test - run_cli(f'CUDA_VISIBLE_DEVICES="" python3 TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli(f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') @staticmethod def test_Tacotron(): @@ -62,5 +62,5 @@ class TestExtractTTSSpectrograms(unittest.TestCase): # save model torch.save({"model": model.state_dict()}, checkpoint_path) # run test - run_cli(f'CUDA_VISIBLE_DEVICES="" python3 TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli(f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') From 7ddc885f378030f239a7e3a912139e8e7b1e9fef Mon Sep 17 00:00:00 2001 From: Adam Froghyar Date: Mon, 10 May 2021 15:42:59 +0200 Subject: [PATCH 12/87] deleted a line the broke GravesAttention --- TTS/tts/layers/tacotron/tacotron.py | 1 - TTS/tts/layers/tacotron/tacotron2.py | 1 - 2 files changed, 2 deletions(-) diff --git a/TTS/tts/layers/tacotron/tacotron.py b/TTS/tts/layers/tacotron/tacotron.py index dcb5fdc5..dc38173f 100644 --- a/TTS/tts/layers/tacotron/tacotron.py +++ b/TTS/tts/layers/tacotron/tacotron.py @@ -463,7 +463,6 @@ class Decoder(nn.Module): stop_tokens = [] t = 0 self._init_states(inputs) - self.attention.init_win_idx() self.attention.init_states(inputs) while True: if t > 0: diff --git a/TTS/tts/layers/tacotron/tacotron2.py b/TTS/tts/layers/tacotron/tacotron2.py index df14aead..aeca8953 100644 --- a/TTS/tts/layers/tacotron/tacotron2.py +++ b/TTS/tts/layers/tacotron/tacotron2.py @@ -375,7 +375,6 @@ class Decoder(nn.Module): else: self._init_states(inputs, mask=None, keep_states=True) - self.attention.init_win_idx() self.attention.init_states(inputs) outputs, stop_tokens, alignments, t = [], [], [], 0 while True: From 607d5cf3773080eb4511b9547f48ea9a95df3a85 Mon Sep 17 00:00:00 2001 From: chmodsss Date: Mon, 10 May 2021 19:46:34 +0200 Subject: [PATCH 13/87] [#480] Adding version variable --- TTS/__init__.py | 1 + TTS/_version.py | 1 + setup.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 TTS/_version.py diff --git a/TTS/__init__.py b/TTS/__init__.py index e69de29b..8dee4bf8 100644 --- a/TTS/__init__.py +++ b/TTS/__init__.py @@ -0,0 +1 @@ +from ._version import __version__ diff --git a/TTS/_version.py b/TTS/_version.py new file mode 100644 index 00000000..d7d14b11 --- /dev/null +++ b/TTS/_version.py @@ -0,0 +1 @@ +__version__ = '0.0.13.2' diff --git a/setup.py b/setup.py index abed43a2..a68b09e0 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import os import subprocess import sys from distutils.version import LooseVersion +from TTS._version import __version__ import numpy import setuptools.command.build_py @@ -18,7 +19,7 @@ if LooseVersion(sys.version) < LooseVersion("3.6") or LooseVersion(sys.version) ) -version = '0.0.13.2' +version = __version__ cwd = os.path.dirname(os.path.abspath(__file__)) class build_py(setuptools.command.build_py.build_py): # pylint: disable=too-many-ancestors From 06f80a48064d26884b3efe9088281e32e0ab5b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 19 Mar 2021 02:51:00 +0100 Subject: [PATCH 14/87] update check argument --- TTS/utils/generic_utils.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index 22287d14..140cf811 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -139,11 +139,11 @@ class KeepAverage: self.update_value(key, value) -def check_argument( - name, c, enum_list=None, max_val=None, min_val=None, restricted=False, val_type=None, alternative=None -): +def check_argument(name, c, enum_list=None, max_val=None, min_val=None, restricted=False, alternative=None, allow_none=False): if alternative in c.keys() and c[alternative] is not None: return + if allow_none and c[name] is None: + return if restricted: assert name in c.keys(), f" [!] {name} not defined in config.json" if name in c.keys(): @@ -152,14 +152,4 @@ def check_argument( if min_val: assert c[name] >= min_val, f" [!] {name} is smaller than min value {min_val}" if enum_list: - assert c[name].lower() in enum_list, f" [!] {name} is not a valid value" - if isinstance(val_type, list): - is_valid = False - for typ in val_type: - if isinstance(c[name], typ): - is_valid = True - assert is_valid or c[name] is None, f" [!] {name} has wrong type - {type(c[name])} vs {val_type}" - elif val_type: - assert ( - isinstance(c[name], val_type) or c[name] is None - ), f" [!] {name} has wrong type - {type(c[name])} vs {val_type}" + assert c[name].lower() in enum_list, f' [!] {name} is not a valid value' From e092ae40dc3b0a62d150eb764567f982c35bdfe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Sat, 20 Mar 2021 00:50:15 +0100 Subject: [PATCH 15/87] config update WIP --- TTS/tts/utils/generic_utils.py | 69 +++++++++++++++++----------------- TTS/utils/generic_utils.py | 17 ++++++++- 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index c58f37c9..d2725eee 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -242,32 +242,32 @@ def check_config_tts(c): check_argument("trim_db", c["audio"], restricted=True, val_type=int) # training parameters - check_argument("batch_size", c, restricted=True, val_type=int, min_val=1) - check_argument("eval_batch_size", c, restricted=True, val_type=int, min_val=1) - check_argument("r", c, restricted=True, val_type=int, min_val=1) - check_argument("gradual_training", c, restricted=False, val_type=list) - check_argument("mixed_precision", c, restricted=False, val_type=bool) + # check_argument('batch_size', c, restricted=True, val_type=int, min_val=1) + # check_argument('eval_batch_size', c, restricted=True, val_type=int, min_val=1) + check_argument('r', c, restricted=True, val_type=int, min_val=1) + check_argument('gradual_training', c, restricted=False, val_type=list) + # check_argument('mixed_precision', c, restricted=False, val_type=bool) # check_argument('grad_accum', c, restricted=True, val_type=int, min_val=1, max_val=100) # loss parameters - check_argument("loss_masking", c, restricted=True, val_type=bool) - if c["model"].lower() in ["tacotron", "tacotron2"]: - check_argument("decoder_loss_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("postnet_loss_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("postnet_diff_spec_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("decoder_diff_spec_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("decoder_ssim_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("postnet_ssim_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("ga_alpha", c, restricted=True, val_type=float, min_val=0) - if c["model"].lower in ["speedy_speech", "align_tts"]: - check_argument("ssim_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("l1_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("huber_alpha", c, restricted=True, val_type=float, min_val=0) + # check_argument('loss_masking', c, restricted=True, val_type=bool) + if c['model'].lower() in ['tacotron', 'tacotron2']: + check_argument('decoder_loss_alpha', c, restricted=True, val_type=float, min_val=0) + check_argument('postnet_loss_alpha', c, restricted=True, val_type=float, min_val=0) + check_argument('postnet_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) + check_argument('decoder_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) + check_argument('decoder_ssim_alpha', c, restricted=True, val_type=float, min_val=0) + check_argument('postnet_ssim_alpha', c, restricted=True, val_type=float, min_val=0) + check_argument('ga_alpha', c, restricted=True, val_type=float, min_val=0) + if c['model'].lower in ["speedy_speech", "align_tts"]: + check_argument('ssim_alpha', c, restricted=True, val_type=float, min_val=0) + check_argument('l1_alpha', c, restricted=True, val_type=float, min_val=0) + check_argument('huber_alpha', c, restricted=True, val_type=float, min_val=0) # validation parameters - check_argument("run_eval", c, restricted=True, val_type=bool) - check_argument("test_delay_epochs", c, restricted=True, val_type=int, min_val=0) - check_argument("test_sentences_file", c, restricted=False, val_type=str) + # check_argument('run_eval', c, restricted=True, val_type=bool) + # check_argument('test_delay_epochs', c, restricted=True, val_type=int, min_val=0) + # check_argument('test_sentences_file', c, restricted=False, val_type=str) # optimizer check_argument("noam_schedule", c, restricted=False, val_type=bool) @@ -319,24 +319,23 @@ def check_config_tts(c): check_argument("encoder_type", c, restricted=not is_tacotron(c), val_type=str) # tensorboard - check_argument("print_step", c, restricted=True, val_type=int, min_val=1) - check_argument("tb_plot_step", c, restricted=True, val_type=int, min_val=1) - check_argument("save_step", c, restricted=True, val_type=int, min_val=1) - check_argument("checkpoint", c, restricted=True, val_type=bool) - check_argument("tb_model_param_stats", c, restricted=True, val_type=bool) + # check_argument('print_step', c, restricted=True, val_type=int, min_val=1) + # check_argument('tb_plot_step', c, restricted=True, val_type=int, min_val=1) + # check_argument('save_step', c, restricted=True, val_type=int, min_val=1) + # check_argument('checkpoint', c, restricted=True, val_type=bool) + # check_argument('tb_model_param_stats', c, restricted=True, val_type=bool) # dataloading # pylint: disable=import-outside-toplevel from TTS.tts.utils.text import cleaners - - check_argument("text_cleaner", c, restricted=True, val_type=str, enum_list=dir(cleaners)) - check_argument("enable_eos_bos_chars", c, restricted=True, val_type=bool) - check_argument("num_loader_workers", c, restricted=True, val_type=int, min_val=0) - check_argument("num_val_loader_workers", c, restricted=True, val_type=int, min_val=0) - check_argument("batch_group_size", c, restricted=True, val_type=int, min_val=0) - check_argument("min_seq_len", c, restricted=True, val_type=int, min_val=0) - check_argument("max_seq_len", c, restricted=True, val_type=int, min_val=10) - check_argument("compute_input_seq_cache", c, restricted=True, val_type=bool) + # check_argument('text_cleaner', c, restricted=True, val_type=str, enum_list=dir(cleaners)) + # check_argument('enable_eos_bos_chars', c, restricted=True, val_type=bool) + # check_argument('num_loader_workers', c, restricted=True, val_type=int, min_val=0) + # check_argument('num_val_loader_workers', c, restricted=True, val_type=int, min_val=0) + # check_argument('batch_group_size', c, restricted=True, val_type=int, min_val=0) + # check_argument('min_seq_len', c, restricted=True, val_type=int, min_val=0) + # check_argument('max_seq_len', c, restricted=True, val_type=int, min_val=10) + # check_argument('compute_input_seq_cache', c, restricted=True, val_type=bool) # paths check_argument("output_path", c, restricted=True, val_type=str) diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index 140cf811..a3a604df 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -5,6 +5,7 @@ import shutil import subprocess import sys from pathlib import Path +from typing import List def get_git_branch(): @@ -139,7 +140,21 @@ class KeepAverage: self.update_value(key, value) -def check_argument(name, c, enum_list=None, max_val=None, min_val=None, restricted=False, alternative=None, allow_none=False): +def check_argument(name, + c, + prerequest=None, + enum_list=None, + max_val=None, + min_val=None, + restricted=False, + alternative=None, + allow_none=False): + if isinstance(prerequest, List()): + if any([f not in c.keys() for f in prerequest]): + return + else: + if prerequest not in c.keys(): + return if alternative in c.keys() and c[alternative] is not None: return if allow_none and c[name] is None: From a21c0b55855acc115d7bb173cd1aad83469feb13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 25 Mar 2021 03:34:31 +0100 Subject: [PATCH 16/87] config update 2 WIP --- TTS/bin/train_tacotron.py | 4 +- TTS/tts/configs/config.json | 4 +- .../ljspeech_tacotron2_dynamic_conv_attn.json | 4 +- TTS/tts/models/tacotron.py | 2 +- TTS/tts/models/tacotron2.py | 2 +- TTS/tts/models/tacotron_abstract.py | 2 +- TTS/tts/utils/generic_utils.py | 118 ++++++++---------- tests/inputs/test_config.json | 4 +- tests/inputs/test_tacotron2_config.json | 4 +- 9 files changed, 66 insertions(+), 78 deletions(-) diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index 9bfa4296..e5e956b5 100755 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -547,9 +547,9 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # inicialize GST with zero dict. style_wav = {} print("WARNING: You don't provided a gst style wav, for this reason we use a zero tensor!") - for i in range(c.gst["gst_style_tokens"]): + for i in range(c.gst['gst_num_style_tokens']): style_wav[str(i)] = 0 - style_wav = c.get("gst_style_input") + style_wav = c.get("gst_style_input", style_wav) for idx, test_sentence in enumerate(test_sentences): try: wav, alignment, decoder_output, postnet_output, stop_tokens, _ = synthesis( diff --git a/TTS/tts/configs/config.json b/TTS/tts/configs/config.json index 4092a1b0..91b2134e 100644 --- a/TTS/tts/configs/config.json +++ b/TTS/tts/configs/config.json @@ -153,10 +153,10 @@ "gst_style_input": null, // Condition the style input either on a // -> wave file [path to wave] or // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_style_tokens). + // with the dictionary being len(dict) <= len(gst_num_style_tokens). "gst_embedding_dim": 512, "gst_num_heads": 4, - "gst_style_tokens": 10, + "gst_num_style_tokens": 10, "gst_use_speaker_embedding": false }, diff --git a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json b/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json index 11e42259..947462aa 100644 --- a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json +++ b/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json @@ -152,10 +152,10 @@ "gst_style_input": null, // Condition the style input either on a // -> wave file [path to wave] or // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_style_tokens). + // with the dictionary being len(dict) <= len(gst_num_style_tokens). "gst_embedding_dim": 512, "gst_num_heads": 4, - "gst_style_tokens": 10, + "gst_num_style_tokens": 10, "gst_use_speaker_embedding": false }, diff --git a/TTS/tts/models/tacotron.py b/TTS/tts/models/tacotron.py index 297c8e3e..85d90116 100644 --- a/TTS/tts/models/tacotron.py +++ b/TTS/tts/models/tacotron.py @@ -44,7 +44,7 @@ class Tacotron(TacotronAbstract): gst (bool, optional): enable/disable global style token learning. Defaults to False. gst_embedding_dim (int, optional): size of channels for GST vectors. Defaults to 512. gst_num_heads (int, optional): number of attention heads for GST. Defaults to 4. - gst_style_tokens (int, optional): number of GST tokens. Defaults to 10. + gst_num_style_tokens (int, optional): number of GST tokens. Defaults to 10. gst_use_speaker_embedding (bool, optional): enable/disable inputing speaker embedding to GST. Defaults to False. memory_size (int, optional): size of the history queue fed to the prenet. Model feeds the last ```memory_size``` output frames to the prenet. diff --git a/TTS/tts/models/tacotron2.py b/TTS/tts/models/tacotron2.py index c015a195..44c81735 100644 --- a/TTS/tts/models/tacotron2.py +++ b/TTS/tts/models/tacotron2.py @@ -44,7 +44,7 @@ class Tacotron2(TacotronAbstract): gst (bool, optional): enable/disable global style token learning. Defaults to False. gst_embedding_dim (int, optional): size of channels for GST vectors. Defaults to 512. gst_num_heads (int, optional): number of attention heads for GST. Defaults to 4. - gst_style_tokens (int, optional): number of GST tokens. Defaults to 10. + gst_num_style_tokens (int, optional): number of GST tokens. Defaults to 10. gst_use_speaker_embedding (bool, optional): enable/disable inputing speaker embedding to GST. Defaults to False. """ diff --git a/TTS/tts/models/tacotron_abstract.py b/TTS/tts/models/tacotron_abstract.py index 1dc49e1b..c6bdb19e 100644 --- a/TTS/tts/models/tacotron_abstract.py +++ b/TTS/tts/models/tacotron_abstract.py @@ -48,7 +48,7 @@ class TacotronAbstract(ABC, nn.Module): self.gst = gst self.gst_embedding_dim = gst_embedding_dim self.gst_num_heads = gst_num_heads - self.gst_style_tokens = gst_style_tokens + self.gst_num_style_tokens = gst_num_style_tokens self.gst_use_speaker_embedding = gst_use_speaker_embedding self.num_speakers = num_speakers self.bidirectional_decoder = bidirectional_decoder diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index d2725eee..1f889b8a 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -176,29 +176,19 @@ def check_config_tts(c): check_argument("run_description", c, val_type=str) # AUDIO - check_argument("audio", c, restricted=True, val_type=dict) + # check_argument('audio', c, restricted=True, val_type=dict) # audio processing parameters - check_argument("num_mels", c["audio"], restricted=True, val_type=int, min_val=10, max_val=2056) - check_argument("fft_size", c["audio"], restricted=True, val_type=int, min_val=128, max_val=4058) - check_argument("sample_rate", c["audio"], restricted=True, val_type=int, min_val=512, max_val=100000) - check_argument( - "frame_length_ms", - c["audio"], - restricted=True, - val_type=float, - min_val=10, - max_val=1000, - alternative="win_length", - ) - check_argument( - "frame_shift_ms", c["audio"], restricted=True, val_type=float, min_val=1, max_val=1000, alternative="hop_length" - ) - check_argument("preemphasis", c["audio"], restricted=True, val_type=float, min_val=0, max_val=1) - check_argument("min_level_db", c["audio"], restricted=True, val_type=int, min_val=-1000, max_val=10) - check_argument("ref_level_db", c["audio"], restricted=True, val_type=int, min_val=0, max_val=1000) - check_argument("power", c["audio"], restricted=True, val_type=float, min_val=1, max_val=5) - check_argument("griffin_lim_iters", c["audio"], restricted=True, val_type=int, min_val=10, max_val=1000) + # check_argument('num_mels', c['audio'], restricted=True, val_type=int, min_val=10, max_val=2056) + # check_argument('fft_size', c['audio'], restricted=True, val_type=int, min_val=128, max_val=4058) + # check_argument('sample_rate', c['audio'], restricted=True, val_type=int, min_val=512, max_val=100000) + # check_argument('frame_length_ms', c['audio'], restricted=True, val_type=float, min_val=10, max_val=1000, alternative='win_length') + # check_argument('frame_shift_ms', c['audio'], restricted=True, val_type=float, min_val=1, max_val=1000, alternative='hop_length') + # check_argument('preemphasis', c['audio'], restricted=True, val_type=float, min_val=0, max_val=1) + # check_argument('min_level_db', c['audio'], restricted=True, val_type=int, min_val=-1000, max_val=10) + # check_argument('ref_level_db', c['audio'], restricted=True, val_type=int, min_val=0, max_val=1000) + # check_argument('power', c['audio'], restricted=True, val_type=float, min_val=1, max_val=5) + # check_argument('griffin_lim_iters', c['audio'], restricted=True, val_type=int, min_val=10, max_val=1000) # vocabulary parameters check_argument("characters", c, restricted=False, val_type=dict) @@ -231,34 +221,34 @@ def check_config_tts(c): ) # normalization parameters - check_argument("signal_norm", c["audio"], restricted=True, val_type=bool) - check_argument("symmetric_norm", c["audio"], restricted=True, val_type=bool) - check_argument("max_norm", c["audio"], restricted=True, val_type=float, min_val=0.1, max_val=1000) - check_argument("clip_norm", c["audio"], restricted=True, val_type=bool) - check_argument("mel_fmin", c["audio"], restricted=True, val_type=float, min_val=0.0, max_val=1000) - check_argument("mel_fmax", c["audio"], restricted=True, val_type=float, min_val=500.0) - check_argument("spec_gain", c["audio"], restricted=True, val_type=[int, float], min_val=1, max_val=100) - check_argument("do_trim_silence", c["audio"], restricted=True, val_type=bool) - check_argument("trim_db", c["audio"], restricted=True, val_type=int) + # check_argument('signal_norm', c['audio'], restricted=True, val_type=bool) + # check_argument('symmetric_norm', c['audio'], restricted=True, val_type=bool) + # check_argument('max_norm', c['audio'], restricted=True, val_type=float, min_val=0.1, max_val=1000) + # check_argument('clip_norm', c['audio'], restricted=True, val_type=bool) + # check_argument('mel_fmin', c['audio'], restricted=True, val_type=float, min_val=0.0, max_val=1000) + # check_argument('mel_fmax', c['audio'], restricted=True, val_type=float, min_val=500.0) + # check_argument('spec_gain', c['audio'], restricted=True, val_type=[int, float], min_val=1, max_val=100) + # check_argument('do_trim_silence', c['audio'], restricted=True, val_type=bool) + # check_argument('trim_db', c['audio'], restricted=True, val_type=int) # training parameters # check_argument('batch_size', c, restricted=True, val_type=int, min_val=1) # check_argument('eval_batch_size', c, restricted=True, val_type=int, min_val=1) - check_argument('r', c, restricted=True, val_type=int, min_val=1) - check_argument('gradual_training', c, restricted=False, val_type=list) + # check_argument('r', c, restricted=True, val_type=int, min_val=1) + # check_argument('gradual_training', c, restricted=False, val_type=list) # check_argument('mixed_precision', c, restricted=False, val_type=bool) # check_argument('grad_accum', c, restricted=True, val_type=int, min_val=1, max_val=100) # loss parameters # check_argument('loss_masking', c, restricted=True, val_type=bool) - if c['model'].lower() in ['tacotron', 'tacotron2']: - check_argument('decoder_loss_alpha', c, restricted=True, val_type=float, min_val=0) - check_argument('postnet_loss_alpha', c, restricted=True, val_type=float, min_val=0) - check_argument('postnet_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) - check_argument('decoder_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) - check_argument('decoder_ssim_alpha', c, restricted=True, val_type=float, min_val=0) - check_argument('postnet_ssim_alpha', c, restricted=True, val_type=float, min_val=0) - check_argument('ga_alpha', c, restricted=True, val_type=float, min_val=0) + # if c['model'].lower() in ['tacotron', 'tacotron2']: + # check_argument('decoder_loss_alpha', c, restricted=True, val_type=float, min_val=0) + # check_argument('postnet_loss_alpha', c, restricted=True, val_type=float, min_val=0) + # check_argument('postnet_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) + # check_argument('decoder_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) + # check_argument('decoder_ssim_alpha', c, restricted=True, val_type=float, min_val=0) + # check_argument('postnet_ssim_alpha', c, restricted=True, val_type=float, min_val=0) + # check_argument('ga_alpha', c, restricted=True, val_type=float, min_val=0) if c['model'].lower in ["speedy_speech", "align_tts"]: check_argument('ssim_alpha', c, restricted=True, val_type=float, min_val=0) check_argument('l1_alpha', c, restricted=True, val_type=float, min_val=0) @@ -279,9 +269,9 @@ def check_config_tts(c): check_argument("seq_len_norm", c, restricted=is_tacotron(c), val_type=bool) # tacotron prenet - check_argument("memory_size", c, restricted=is_tacotron(c), val_type=int, min_val=-1) - check_argument("prenet_type", c, restricted=is_tacotron(c), val_type=str, enum_list=["original", "bn"]) - check_argument("prenet_dropout", c, restricted=is_tacotron(c), val_type=bool) + # check_argument('memory_size', c, restricted=is_tacotron(c), val_type=int, min_val=-1) + # check_argument('prenet_type', c, restricted=is_tacotron(c), val_type=str, enum_list=['original', 'bn']) + # check_argument('prenet_dropout', c, restricted=is_tacotron(c), val_type=bool) # attention check_argument( @@ -305,8 +295,8 @@ def check_config_tts(c): if c["model"].lower() in ["tacotron", "tacotron2"]: # stopnet - check_argument("stopnet", c, restricted=is_tacotron(c), val_type=bool) - check_argument("separate_stopnet", c, restricted=is_tacotron(c), val_type=bool) + # check_argument('stopnet', c, restricted=is_tacotron(c), val_type=bool) + # check_argument('separate_stopnet', c, restricted=is_tacotron(c), val_type=bool) # Model Parameters for non-tacotron models if c["model"].lower in ["speedy_speech", "align_tts"]: @@ -338,27 +328,25 @@ def check_config_tts(c): # check_argument('compute_input_seq_cache', c, restricted=True, val_type=bool) # paths - check_argument("output_path", c, restricted=True, val_type=str) + # check_argument('output_path', c, restricted=True, val_type=str) # multi-speaker and gst - check_argument("use_speaker_embedding", c, restricted=True, val_type=bool) - check_argument("use_external_speaker_embedding_file", c, restricted=c["use_speaker_embedding"], val_type=bool) - check_argument( - "external_speaker_embedding_file", c, restricted=c["use_external_speaker_embedding_file"], val_type=str - ) - if c["model"].lower() in ["tacotron", "tacotron2"] and c["use_gst"]: - check_argument("use_gst", c, restricted=is_tacotron(c), val_type=bool) - check_argument("gst", c, restricted=is_tacotron(c), val_type=dict) - check_argument("gst_style_input", c["gst"], restricted=is_tacotron(c), val_type=[str, dict]) - check_argument("gst_embedding_dim", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=0, max_val=1000) - check_argument("gst_use_speaker_embedding", c["gst"], restricted=is_tacotron(c), val_type=bool) - check_argument("gst_num_heads", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=2, max_val=10) - check_argument("gst_style_tokens", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=1, max_val=1000) + # check_argument('use_speaker_embedding', c, restricted=True, val_type=bool) + # check_argument('use_external_speaker_embedding_file', c, restricted=c['use_speaker_embedding'], val_type=bool) + # check_argument('external_speaker_embedding_file', c, restricted=c['use_external_speaker_embedding_file'], val_type=str) + if c['model'].lower() in ['tacotron', 'tacotron2'] and c['use_gst']: + # check_argument('use_gst', c, restricted=is_tacotron(c), val_type=bool) + # check_argument('gst', c, restricted=is_tacotron(c), val_type=dict) + # check_argument('gst_style_input', c['gst'], restricted=is_tacotron(c), val_type=[str, dict]) + # check_argument('gst_embedding_dim', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=0, max_val=1000) + # check_argument('gst_use_speaker_embedding', c['gst'], restricted=is_tacotron(c), val_type=bool) + # check_argument('gst_num_heads', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=2, max_val=10) + # check_argument('gst_num_style_tokens', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=1, max_val=1000) # datasets - checking only the first entry - check_argument("datasets", c, restricted=True, val_type=list) - for dataset_entry in c["datasets"]: - check_argument("name", dataset_entry, restricted=True, val_type=str) - check_argument("path", dataset_entry, restricted=True, val_type=str) - check_argument("meta_file_train", dataset_entry, restricted=True, val_type=[str, list]) - check_argument("meta_file_val", dataset_entry, restricted=True, val_type=str) + # check_argument('datasets', c, restricted=True, val_type=list) + # for dataset_entry in c['datasets']: + # check_argument('name', dataset_entry, restricted=True, val_type=str) + # check_argument('path', dataset_entry, restricted=True, val_type=str) + # check_argument('meta_file_train', dataset_entry, restricted=True, val_type=[str, list]) + # check_argument('meta_file_val', dataset_entry, restricted=True, val_type=str) diff --git a/tests/inputs/test_config.json b/tests/inputs/test_config.json index b28bec64..2fb52bb6 100644 --- a/tests/inputs/test_config.json +++ b/tests/inputs/test_config.json @@ -60,10 +60,10 @@ "gst_style_input": null, // Condition the style input either on a // -> wave file [path to wave] or // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_style_tokens). + // with the dictionary being len(dict) <= len(gst_num_style_tokens). "gst_use_speaker_embedding": true, // if true pass speaker embedding in attention input GST. "gst_embedding_dim": 512, "gst_num_heads": 4, - "gst_style_tokens": 10 + "gst_num_style_tokens": 10 } } diff --git a/tests/inputs/test_tacotron2_config.json b/tests/inputs/test_tacotron2_config.json index 14449867..779f925d 100644 --- a/tests/inputs/test_tacotron2_config.json +++ b/tests/inputs/test_tacotron2_config.json @@ -153,11 +153,11 @@ "gst_style_input": null, // Condition the style input either on a // -> wave file [path to wave] or // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) == len(gst_style_tokens). + // with the dictionary being len(dict) == len(gst_num_style_tokens). "gst_use_speaker_embedding": true, // if true pass speaker embedding in attention input GST. "gst_embedding_dim": 512, "gst_num_heads": 4, - "gst_style_tokens": 10 + "gst_num_style_tokens": 10 }, // DATASETS From 97bd5f9734e16abea5474180a0f7527532b38708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 30 Mar 2021 14:18:35 +0200 Subject: [PATCH 17/87] [ci skip] config update #3 WIP --- TTS/bin/train_tacotron.py | 1 + TTS/tts/configs/config.json | 287 +++++++++++++++--------------------- TTS/utils/arguments.py | 43 ++---- TTS/utils/io.py | 47 +++--- 4 files changed, 158 insertions(+), 220 deletions(-) diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index e5e956b5..b864d303 100755 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -14,6 +14,7 @@ from torch.utils.data import DataLoader from TTS.tts.datasets.preprocess import load_meta_data from TTS.tts.datasets.TTSDataset import MyDataset from TTS.tts.layers.losses import TacotronLoss +from TTS.tts.configs.tacotron_config import TacotronConfig from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.io import save_best_model, save_checkpoint from TTS.tts.utils.measures import alignment_diagonal_score diff --git a/TTS/tts/configs/config.json b/TTS/tts/configs/config.json index 91b2134e..95e787e0 100644 --- a/TTS/tts/configs/config.json +++ b/TTS/tts/configs/config.json @@ -1,173 +1,126 @@ { - "model": "Tacotron2", - "run_name": "ljspeech-ddc", - "run_description": "tacotron2 with DDC and differential spectral loss.", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! + "attention_heads": 4, + "attention_norm": "sigmoid", + "attention_type": "original", + "audio_config": { + "clip_norm": true, + "do_trim_silence": true, + "fft_size": 1024, + "frame_length_ms": null, + "frame_shift_ms": null, + "griffin_lim_iters": 60, + "hop_length": 256, + "max_norm": 4, + "mel_fmax": 7600, + "mel_fmin": 50, + "min_level_db": -100, + "num_mels": 80, + "power": 1.5, + "preemphasis": 0, + "ref_level_db": 20, + "sample_rate": 22050, + "signal_norm": true, "spec_gain": 1, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy", + "symmetric_norm": true, + "trim_db": 60, + "win_length": 1024 }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. - "mixed_precision": true, // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate. - - // LOSS SETTINGS - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled - "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled - "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled - "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled - "ga_alpha": 5.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. - - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TACOTRON PRENET - "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. - "prenet_type": "original", // "original" or "bn". - "prenet_dropout": true, // enable/disable dropout at prenet. - - // TACOTRON ATTENTION - "attention_type": "original", // 'original' , 'graves', 'dynamic_convolution' - "attention_heads": 4, // number of attention heads (only for 'graves') - "attention_norm": "sigmoid", // softmax or sigmoid. - "windowing": false, // Enables attention windowing. Used only in eval mode. - "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. - "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. - "transition_agent": false, // enable/disable transition agent of forward attention. - "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. - "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. - "double_decoder_consistency": true, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ - "ddc_r": 7, // reduction rate for coarse decoder. - - // STOPNET - "stopnet": true, // Train stopnet predicting the end of synthesis. - "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 4, //Number of batches to shuffle after bucketing. - "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 153, // DATASET-RELATED: maximum text length - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - "use_noise_augment": true, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": false, // use global style tokens - "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_num_style_tokens). - "gst_embedding_dim": 512, - "gst_num_heads": 4, - "gst_num_style_tokens": 10, - "gst_use_speaker_embedding": false - }, - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. + "bidirectional_decoder": false, + "compute_input_seq_cache": false, + "ddc_r": 7, + "decoder_diff_spec_alpha": 0.25, + "decoder_loss_alpha": 0.5, + "decoder_ssim_alpha": 0.5, + "double_decoder_consistency": true, + "enable_eos_bos_chars": false, + "forward_attn_mask": false, + "ga_alpha": 5, + "grad_clip": 1, + "gradual_training": [ [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers - "meta_file_val": null - } + 0, + 7, + 64 + ], + [ + 1, + 5, + 64 + ], + [ + 50000, + 3, + 32 + ], + [ + 130000, + 2, + 32 + ], + [ + 290000, + 1, + 32 ] + ], + "location_attn": true, + "lr": 0.0001, + "memory_size": -1, + "noam_schedule": false, + "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", + "phoneme_language": "en-us", + "postnet_diff_spec_alpha": 0.25, + "postnet_loss_alpha": 0.25, + "postnet_ssim_alpha": 0.25, + "prenet_dropout": false, + "prenet_type": "original", + "r": 7, + "separate_stopnet": true, + "seq_len_norm": false, + "stopnet": true, + "stopnet_pos_weight": 15, + "test_sentences_file": null, + "text_cleaner": "phoneme_cleaners", + "training_config": { + "batch_group_size": 4, + "batch_size": 32, + "checkpoint": true, + "datasets": [ + { + "meta_file_train": "metadata.csv", + "meta_file_val": null, + "name": "ljspeech", + "path": "/home/erogol/Data/LJSpeech-1.1/" + } + ], + "epochs": 1000, + "eval_batch_size": 16, + "keep_after": 10000, + "keep_all_best": false, + "loss_masking": true, + "max_seq_len": 153, + "min_seq_len": 6, + "mixed_precision": true, + "model": "Tacotron2", + "num_loader_workers": 4, + "num_val_loader_workers": 4, + "output_path": "/home/erogol/Models/LJSpeech/", + "print_eval": false, + "print_step": 25, + "run_description": "tacotron2 with DDC and differential spectral loss.", + "run_eval": true, + "run_name": "ljspeech-ddc", + "save_step": 10000, + "tb_model_param_stats": false, + "tb_plot_step": 100, + "test_delay_epochs": 10, + "use_noise_augment": true + }, + "transition_agent": false, + "use_forward_attn": false, + "use_phonemes": true, + "warmup_steps": 4000, + "wd": 0.000001, + "windowing": false } diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index af0a1598..4f6e2317 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -117,16 +117,11 @@ def get_last_checkpoint(path): return last_models["checkpoint"], last_models["best_model"] -def process_args(args, model_class): - """Process parsed comand line arguments based on model class (tts or vocoder). +def process_args(args, config, tb_prefix): + """Process parsed comand line arguments. Args: args (argparse.Namespace or dict like): Parsed input arguments. - model_type (str): Model type used to check config parameters and setup - the TensorBoard logger. One of ['tts', 'vocoder']. - - Raises: - ValueError: If `model_type` is not one of implemented choices. Returns: c (TTS.utils.io.AttrDict): Config paramaters. @@ -138,28 +133,21 @@ def process_args(args, model_class): the TensorBoard loggind. """ if args.continue_path: + # continue a previous training from its output folder args.output_path = args.continue_path args.config_path = os.path.join(args.continue_path, "config.json") args.restore_path, best_model = get_last_checkpoint(args.continue_path) if not args.best_path: args.best_path = best_model - # setup output paths and read configs - c = load_config(args.config_path) - _ = os.path.dirname(os.path.realpath(__file__)) - - if "mixed_precision" in c and c.mixed_precision: + c = config.load_json(args.config_path) + if c.mixed_precision: print(" > Mixed precision mode is ON") - - out_path = args.continue_path - if not out_path: - out_path = create_experiment_folder(c.output_path, c.run_name, args.debug) - + if not os.path.exists(c.output_path): + out_path = create_experiment_folder(c.output_path, c.run_name, + args.debug) audio_path = os.path.join(out_path, "test_audios") - - c_logger = ConsoleLogger() - tb_logger = None - + # setup rank 0 process in distributed training if args.rank == 0: os.makedirs(audio_path, exist_ok=True) new_fields = {} @@ -169,18 +157,15 @@ def process_args(args, model_class): # if model characters are not set in the config file # save the default set to the config file for future # compatibility. - if model_class == "tts" and "characters" not in c: + if c.has('characters_config'): used_characters = parse_symbols() new_fields["characters"] = used_characters copy_model_files(c, args.config_path, out_path, new_fields) os.chmod(audio_path, 0o775) os.chmod(out_path, 0o775) - log_path = out_path - - tb_logger = TensorboardLogger(log_path, model_name=model_class.upper()) - - # write model config to tensorboard - tb_logger.tb_add_text("model-config", f"
{json.dumps(c, indent=4)}
", 0) - + tb_logger = TensorboardLogger(log_path, model_name=tb_prefix) + # write model desc to tensorboard + tb_logger.tb_add_text("model-description", c["run_description"], 0) + c_logger = ConsoleLogger() return c, out_path, audio_path, c_logger, tb_logger diff --git a/TTS/utils/io.py b/TTS/utils/io.py index 84493e07..12745459 100644 --- a/TTS/utils/io.py +++ b/TTS/utils/io.py @@ -23,33 +23,32 @@ class AttrDict(dict): self.__dict__ = self -def read_json_with_comments(json_path): - # fallback to json - with open(json_path, "r", encoding="utf-8") as f: - input_str = f.read() - # handle comments - input_str = re.sub(r"\\\n", "", input_str) - input_str = re.sub(r"//.*\n", "\n", input_str) - data = json.loads(input_str) - return data +# def read_json_with_comments(json_path): +# # fallback to json +# with open(json_path, "r", encoding="utf-8") as f: +# input_str = f.read() +# # handle comments +# input_str = re.sub(r'\\\n', '', input_str) +# input_str = re.sub(r'//.*\n', '\n', input_str) +# data = json.loads(input_str) +# return data +# def load_config(config_path: str) -> AttrDict: +# """Load config files and discard comments -def load_config(config_path: str) -> AttrDict: - """Load config files and discard comments +# Args: +# config_path (str): path to config file. +# """ +# config = AttrDict() - Args: - config_path (str): path to config file. - """ - config = AttrDict() - - ext = os.path.splitext(config_path)[1] - if ext in (".yml", ".yaml"): - with open(config_path, "r", encoding="utf-8") as f: - data = yaml.safe_load(f) - else: - data = read_json_with_comments(config_path) - config.update(data) - return config +# ext = os.path.splitext(config_path)[1] +# # if ext in (".yml", ".yaml"): +# # with open(config_path, "r", encoding="utf-8") as f: +# # data = yaml.safe_load(f) +# # else: +# data = read_json_with_comments(config_path) +# config.update(data) +# return config def copy_model_files(c, config_file, out_path, new_fields): From dc50f5f0b0ef823fa27d3396a7ec41df07fa6e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 1 Apr 2021 18:22:24 +0200 Subject: [PATCH 18/87] config refactor #4 WIP --- TTS/bin/compute_statistics.py | 47 +++-- TTS/bin/train_tacotron.py | 14 +- TTS/tts/utils/generic_utils.py | 318 ++++++++++++++------------------- TTS/utils/arguments.py | 16 +- TTS/utils/generic_utils.py | 47 ++--- TTS/utils/io.py | 54 +++--- 6 files changed, 229 insertions(+), 267 deletions(-) diff --git a/TTS/bin/compute_statistics.py b/TTS/bin/compute_statistics.py index 9e2b7415..a8c4240e 100755 --- a/TTS/bin/compute_statistics.py +++ b/TTS/bin/compute_statistics.py @@ -8,6 +8,7 @@ import os import numpy as np from tqdm import tqdm +from TTS.utils.config_manager import ConfigManager from TTS.tts.datasets.preprocess import load_meta_data from TTS.utils.audio import AudioProcessor from TTS.utils.io import load_config @@ -15,26 +16,33 @@ from TTS.utils.io import load_config def main(): """Run preprocessing process.""" - parser = argparse.ArgumentParser(description="Compute mean and variance of spectrogtram features.") - parser.add_argument( - "--config_path", type=str, required=True, help="TTS config file path to define audio processin parameters." - ) - parser.add_argument("--out_path", type=str, required=True, help="save path (directory and filename).") + CONFIG = ConfigManager() + + parser = argparse.ArgumentParser( + description="Compute mean and variance of spectrogtram features.") + parser.add_argument("config_path", type=str, + help="TTS config file path to define audio processin parameters.") + parser.add_argument("out_path", type=str, + help="save path (directory and filename).") + parser.add_argument("--data_path", type=str, required=False, + help="folder including the target set of wavs overriding dataset config.") + parser = CONFIG.init_argparse(parser) args = parser.parse_args() + CONFIG.parse_argparse(args) # load config - CONFIG = load_config(args.config_path) - CONFIG.audio["signal_norm"] = False # do not apply earlier normalization - CONFIG.audio["stats_path"] = None # discard pre-defined stats + CONFIG.load_config(args.config_path) + CONFIG.audio_config.signal_norm = False # do not apply earlier normalization + CONFIG.audio_config.stats_path = None # discard pre-defined stats # load audio processor - ap = AudioProcessor(**CONFIG.audio) + ap = AudioProcessor(**CONFIG.audio_config.to_dict()) # load the meta data of target dataset - if "data_path" in CONFIG.keys(): - dataset_items = glob.glob(os.path.join(CONFIG.data_path, "**", "*.wav"), recursive=True) + if args.data_path: + dataset_items = glob.glob(os.path.join(args.data_path, '**', '*.wav'), recursive=True) else: - dataset_items = load_meta_data(CONFIG.datasets)[0] # take only train data + dataset_items = load_meta_data(CONFIG.dataset_config)[0] # take only train data print(f" > There are {len(dataset_items)} files.") mel_sum = 0 @@ -73,14 +81,15 @@ def main(): print(f" > Avg lienar spec scale: {linear_scale.mean()}") # set default config values for mean-var scaling - CONFIG.audio["stats_path"] = output_file_path - CONFIG.audio["signal_norm"] = True + CONFIG.audio_config.stats_path = output_file_path + CONFIG.audio_config.signal_norm = True # remove redundant values - del CONFIG.audio["max_norm"] - del CONFIG.audio["min_level_db"] - del CONFIG.audio["symmetric_norm"] - del CONFIG.audio["clip_norm"] - stats["audio_config"] = CONFIG.audio + del CONFIG.audio_config.max_norm + del CONFIG.audio_config.min_level_db + del CONFIG.audio_config.symmetric_norm + del CONFIG.audio_config.clip_norm + breakpoint() + stats['audio_config'] = CONFIG.audio_config.to_dict() np.save(output_file_path, stats, allow_pickle=True) print(f" > stats saved to {output_file_path}") diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index b864d303..65bb4da6 100755 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -10,11 +10,9 @@ from random import randrange import numpy as np import torch from torch.utils.data import DataLoader - from TTS.tts.datasets.preprocess import load_meta_data from TTS.tts.datasets.TTSDataset import MyDataset from TTS.tts.layers.losses import TacotronLoss -from TTS.tts.configs.tacotron_config import TacotronConfig from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.io import save_best_model, save_checkpoint from TTS.tts.utils.measures import alignment_diagonal_score @@ -24,8 +22,11 @@ from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram from TTS.utils.arguments import parse_arguments, process_args from TTS.utils.audio import AudioProcessor -from TTS.utils.distribute import DistributedSampler, apply_gradient_allreduce, init_distributed, reduce_tensor -from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict +from TTS.utils.config_manager import ConfigManager +from TTS.utils.distribute import (DistributedSampler, apply_gradient_allreduce, + init_distributed, reduce_tensor) +from TTS.utils.generic_utils import (KeepAverage, count_parameters, + remove_experiment_folder, set_init_dict) from TTS.utils.radam import RAdam from TTS.utils.training import ( NoamLR, @@ -739,7 +740,10 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts") + c = TacotronConfig() + args = c.init_argparse(args) + c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args( + args, c, model_type='tacotron') try: main(args) diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index 1f889b8a..0e80111b 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -2,10 +2,7 @@ import importlib import re from collections import Counter -import numpy as np -import torch - -from TTS.utils.generic_utils import check_argument +from TTS.utils.generic_utils import find_module def split_dataset(items): @@ -39,17 +36,9 @@ def sequence_mask(sequence_length, max_len=None): return seq_range.unsqueeze(0) < sequence_length.unsqueeze(1) -def to_camel(text): - text = text.capitalize() - text = re.sub(r"(?!^)_([a-zA-Z])", lambda m: m.group(1).upper(), text) - text = text.replace("Tts", "TTS") - return text - - def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): print(" > Using model: {}".format(c.model)) - MyModel = importlib.import_module("TTS.tts.models." + c.model.lower()) - MyModel = getattr(MyModel, to_camel(c.model)) + find_module("TTS.tts.models", c.model.lower()) if c.model.lower() in "tacotron": model = MyModel( num_chars=num_chars + getattr(c, "add_blank", False), @@ -164,189 +153,156 @@ def is_tacotron(c): return "tacotron" in c["model"].lower() -def check_config_tts(c): - check_argument( - "model", - c, - enum_list=["tacotron", "tacotron2", "glow_tts", "speedy_speech", "align_tts"], - restricted=True, - val_type=str, - ) - check_argument("run_name", c, restricted=True, val_type=str) - check_argument("run_description", c, val_type=str) +# def check_config_tts(c): +# check_argument('model', c, enum_list=['tacotron', 'tacotron2', 'glow_tts', 'speedy_speech', 'align_tts'], restricted=True, val_type=str) +# check_argument('run_name', c, restricted=True, val_type=str) +# check_argument('run_description', c, val_type=str) - # AUDIO - # check_argument('audio', c, restricted=True, val_type=dict) +# # AUDIO +# # check_argument('audio', c, restricted=True, val_type=dict) - # audio processing parameters - # check_argument('num_mels', c['audio'], restricted=True, val_type=int, min_val=10, max_val=2056) - # check_argument('fft_size', c['audio'], restricted=True, val_type=int, min_val=128, max_val=4058) - # check_argument('sample_rate', c['audio'], restricted=True, val_type=int, min_val=512, max_val=100000) - # check_argument('frame_length_ms', c['audio'], restricted=True, val_type=float, min_val=10, max_val=1000, alternative='win_length') - # check_argument('frame_shift_ms', c['audio'], restricted=True, val_type=float, min_val=1, max_val=1000, alternative='hop_length') - # check_argument('preemphasis', c['audio'], restricted=True, val_type=float, min_val=0, max_val=1) - # check_argument('min_level_db', c['audio'], restricted=True, val_type=int, min_val=-1000, max_val=10) - # check_argument('ref_level_db', c['audio'], restricted=True, val_type=int, min_val=0, max_val=1000) - # check_argument('power', c['audio'], restricted=True, val_type=float, min_val=1, max_val=5) - # check_argument('griffin_lim_iters', c['audio'], restricted=True, val_type=int, min_val=10, max_val=1000) +# # audio processing parameters +# # check_argument('num_mels', c['audio'], restricted=True, val_type=int, min_val=10, max_val=2056) +# # check_argument('fft_size', c['audio'], restricted=True, val_type=int, min_val=128, max_val=4058) +# # check_argument('sample_rate', c['audio'], restricted=True, val_type=int, min_val=512, max_val=100000) +# # check_argument('frame_length_ms', c['audio'], restricted=True, val_type=float, min_val=10, max_val=1000, alternative='win_length') +# # check_argument('frame_shift_ms', c['audio'], restricted=True, val_type=float, min_val=1, max_val=1000, alternative='hop_length') +# # check_argument('preemphasis', c['audio'], restricted=True, val_type=float, min_val=0, max_val=1) +# # check_argument('min_level_db', c['audio'], restricted=True, val_type=int, min_val=-1000, max_val=10) +# # check_argument('ref_level_db', c['audio'], restricted=True, val_type=int, min_val=0, max_val=1000) +# # check_argument('power', c['audio'], restricted=True, val_type=float, min_val=1, max_val=5) +# # check_argument('griffin_lim_iters', c['audio'], restricted=True, val_type=int, min_val=10, max_val=1000) - # vocabulary parameters - check_argument("characters", c, restricted=False, val_type=dict) - check_argument( - "pad", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str - ) - check_argument( - "eos", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str - ) - check_argument( - "bos", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str - ) - check_argument( - "characters", - c["characters"] if "characters" in c.keys() else {}, - restricted="characters" in c.keys(), - val_type=str, - ) - check_argument( - "phonemes", - c["characters"] if "characters" in c.keys() else {}, - restricted="characters" in c.keys() and c["use_phonemes"], - val_type=str, - ) - check_argument( - "punctuations", - c["characters"] if "characters" in c.keys() else {}, - restricted="characters" in c.keys(), - val_type=str, - ) +# # vocabulary parameters +# check_argument('characters', c, restricted=False, val_type=dict) +# check_argument('pad', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) +# check_argument('eos', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) +# check_argument('bos', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) +# check_argument('characters', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) +# check_argument('phonemes', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys() and c['use_phonemes'], val_type=str) +# check_argument('punctuations', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) - # normalization parameters - # check_argument('signal_norm', c['audio'], restricted=True, val_type=bool) - # check_argument('symmetric_norm', c['audio'], restricted=True, val_type=bool) - # check_argument('max_norm', c['audio'], restricted=True, val_type=float, min_val=0.1, max_val=1000) - # check_argument('clip_norm', c['audio'], restricted=True, val_type=bool) - # check_argument('mel_fmin', c['audio'], restricted=True, val_type=float, min_val=0.0, max_val=1000) - # check_argument('mel_fmax', c['audio'], restricted=True, val_type=float, min_val=500.0) - # check_argument('spec_gain', c['audio'], restricted=True, val_type=[int, float], min_val=1, max_val=100) - # check_argument('do_trim_silence', c['audio'], restricted=True, val_type=bool) - # check_argument('trim_db', c['audio'], restricted=True, val_type=int) +# # normalization parameters +# # check_argument('signal_norm', c['audio'], restricted=True, val_type=bool) +# # check_argument('symmetric_norm', c['audio'], restricted=True, val_type=bool) +# # check_argument('max_norm', c['audio'], restricted=True, val_type=float, min_val=0.1, max_val=1000) +# # check_argument('clip_norm', c['audio'], restricted=True, val_type=bool) +# # check_argument('mel_fmin', c['audio'], restricted=True, val_type=float, min_val=0.0, max_val=1000) +# # check_argument('mel_fmax', c['audio'], restricted=True, val_type=float, min_val=500.0) +# # check_argument('spec_gain', c['audio'], restricted=True, val_type=[int, float], min_val=1, max_val=100) +# # check_argument('do_trim_silence', c['audio'], restricted=True, val_type=bool) +# # check_argument('trim_db', c['audio'], restricted=True, val_type=int) - # training parameters - # check_argument('batch_size', c, restricted=True, val_type=int, min_val=1) - # check_argument('eval_batch_size', c, restricted=True, val_type=int, min_val=1) - # check_argument('r', c, restricted=True, val_type=int, min_val=1) - # check_argument('gradual_training', c, restricted=False, val_type=list) - # check_argument('mixed_precision', c, restricted=False, val_type=bool) - # check_argument('grad_accum', c, restricted=True, val_type=int, min_val=1, max_val=100) +# # training parameters +# # check_argument('batch_size', c, restricted=True, val_type=int, min_val=1) +# # check_argument('eval_batch_size', c, restricted=True, val_type=int, min_val=1) +# # check_argument('r', c, restricted=True, val_type=int, min_val=1) +# # check_argument('gradual_training', c, restricted=False, val_type=list) +# # check_argument('mixed_precision', c, restricted=False, val_type=bool) +# # check_argument('grad_accum', c, restricted=True, val_type=int, min_val=1, max_val=100) - # loss parameters - # check_argument('loss_masking', c, restricted=True, val_type=bool) - # if c['model'].lower() in ['tacotron', 'tacotron2']: - # check_argument('decoder_loss_alpha', c, restricted=True, val_type=float, min_val=0) - # check_argument('postnet_loss_alpha', c, restricted=True, val_type=float, min_val=0) - # check_argument('postnet_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) - # check_argument('decoder_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) - # check_argument('decoder_ssim_alpha', c, restricted=True, val_type=float, min_val=0) - # check_argument('postnet_ssim_alpha', c, restricted=True, val_type=float, min_val=0) - # check_argument('ga_alpha', c, restricted=True, val_type=float, min_val=0) - if c['model'].lower in ["speedy_speech", "align_tts"]: - check_argument('ssim_alpha', c, restricted=True, val_type=float, min_val=0) - check_argument('l1_alpha', c, restricted=True, val_type=float, min_val=0) - check_argument('huber_alpha', c, restricted=True, val_type=float, min_val=0) +# # loss parameters +# # check_argument('loss_masking', c, restricted=True, val_type=bool) +# # if c['model'].lower() in ['tacotron', 'tacotron2']: +# # check_argument('decoder_loss_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('postnet_loss_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('postnet_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('decoder_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('decoder_ssim_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('postnet_ssim_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('ga_alpha', c, restricted=True, val_type=float, min_val=0) +# if c['model'].lower in ["speedy_speech", "align_tts"]: +# check_argument('ssim_alpha', c, restricted=True, val_type=float, min_val=0) +# check_argument('l1_alpha', c, restricted=True, val_type=float, min_val=0) +# check_argument('huber_alpha', c, restricted=True, val_type=float, min_val=0) - # validation parameters - # check_argument('run_eval', c, restricted=True, val_type=bool) - # check_argument('test_delay_epochs', c, restricted=True, val_type=int, min_val=0) - # check_argument('test_sentences_file', c, restricted=False, val_type=str) +# # validation parameters +# # check_argument('run_eval', c, restricted=True, val_type=bool) +# # check_argument('test_delay_epochs', c, restricted=True, val_type=int, min_val=0) +# # check_argument('test_sentences_file', c, restricted=False, val_type=str) - # optimizer - check_argument("noam_schedule", c, restricted=False, val_type=bool) - check_argument("grad_clip", c, restricted=True, val_type=float, min_val=0.0) - check_argument("epochs", c, restricted=True, val_type=int, min_val=1) - check_argument("lr", c, restricted=True, val_type=float, min_val=0) - check_argument("wd", c, restricted=is_tacotron(c), val_type=float, min_val=0) - check_argument("warmup_steps", c, restricted=True, val_type=int, min_val=0) - check_argument("seq_len_norm", c, restricted=is_tacotron(c), val_type=bool) +# # optimizer +# check_argument('noam_schedule', c, restricted=False, val_type=bool) +# check_argument('grad_clip', c, restricted=True, val_type=float, min_val=0.0) +# check_argument('epochs', c, restricted=True, val_type=int, min_val=1) +# check_argument('lr', c, restricted=True, val_type=float, min_val=0) +# check_argument('wd', c, restricted=is_tacotron(c), val_type=float, min_val=0) +# check_argument('warmup_steps', c, restricted=True, val_type=int, min_val=0) +# check_argument('seq_len_norm', c, restricted=is_tacotron(c), val_type=bool) - # tacotron prenet - # check_argument('memory_size', c, restricted=is_tacotron(c), val_type=int, min_val=-1) - # check_argument('prenet_type', c, restricted=is_tacotron(c), val_type=str, enum_list=['original', 'bn']) - # check_argument('prenet_dropout', c, restricted=is_tacotron(c), val_type=bool) +# # tacotron prenet +# # check_argument('memory_size', c, restricted=is_tacotron(c), val_type=int, min_val=-1) +# # check_argument('prenet_type', c, restricted=is_tacotron(c), val_type=str, enum_list=['original', 'bn']) +# # check_argument('prenet_dropout', c, restricted=is_tacotron(c), val_type=bool) - # attention - check_argument( - "attention_type", - c, - restricted=is_tacotron(c), - val_type=str, - enum_list=["graves", "original", "dynamic_convolution"], - ) - check_argument("attention_heads", c, restricted=is_tacotron(c), val_type=int) - check_argument("attention_norm", c, restricted=is_tacotron(c), val_type=str, enum_list=["sigmoid", "softmax"]) - check_argument("windowing", c, restricted=is_tacotron(c), val_type=bool) - check_argument("use_forward_attn", c, restricted=is_tacotron(c), val_type=bool) - check_argument("forward_attn_mask", c, restricted=is_tacotron(c), val_type=bool) - check_argument("transition_agent", c, restricted=is_tacotron(c), val_type=bool) - check_argument("transition_agent", c, restricted=is_tacotron(c), val_type=bool) - check_argument("location_attn", c, restricted=is_tacotron(c), val_type=bool) - check_argument("bidirectional_decoder", c, restricted=is_tacotron(c), val_type=bool) - check_argument("double_decoder_consistency", c, restricted=is_tacotron(c), val_type=bool) - check_argument("ddc_r", c, restricted="double_decoder_consistency" in c.keys(), min_val=1, max_val=7, val_type=int) +# # attention +# check_argument('attention_type', c, restricted=is_tacotron(c), val_type=str, enum_list=['graves', 'original', 'dynamic_convolution']) +# check_argument('attention_heads', c, restricted=is_tacotron(c), val_type=int) +# check_argument('attention_norm', c, restricted=is_tacotron(c), val_type=str, enum_list=['sigmoid', 'softmax']) +# check_argument('windowing', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('use_forward_attn', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('forward_attn_mask', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('transition_agent', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('transition_agent', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('location_attn', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('bidirectional_decoder', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('double_decoder_consistency', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('ddc_r', c, restricted='double_decoder_consistency' in c.keys(), min_val=1, max_val=7, val_type=int) - if c["model"].lower() in ["tacotron", "tacotron2"]: - # stopnet - # check_argument('stopnet', c, restricted=is_tacotron(c), val_type=bool) - # check_argument('separate_stopnet', c, restricted=is_tacotron(c), val_type=bool) +# if c['model'].lower() in ['tacotron', 'tacotron2']: +# # stopnet +# # check_argument('stopnet', c, restricted=is_tacotron(c), val_type=bool) +# # check_argument('separate_stopnet', c, restricted=is_tacotron(c), val_type=bool) - # Model Parameters for non-tacotron models - if c["model"].lower in ["speedy_speech", "align_tts"]: - check_argument("positional_encoding", c, restricted=True, val_type=type) - check_argument("encoder_type", c, restricted=True, val_type=str) - check_argument("encoder_params", c, restricted=True, val_type=dict) - check_argument("decoder_residual_conv_bn_params", c, restricted=True, val_type=dict) +# # Model Parameters for non-tacotron models +# if c['model'].lower in ["speedy_speech", "align_tts"]: +# check_argument('positional_encoding', c, restricted=True, val_type=type) +# check_argument('encoder_type', c, restricted=True, val_type=str) +# check_argument('encoder_params', c, restricted=True, val_type=dict) +# check_argument('decoder_residual_conv_bn_params', c, restricted=True, val_type=dict) - # GlowTTS parameters - check_argument("encoder_type", c, restricted=not is_tacotron(c), val_type=str) +# # GlowTTS parameters +# check_argument('encoder_type', c, restricted=not is_tacotron(c), val_type=str) - # tensorboard - # check_argument('print_step', c, restricted=True, val_type=int, min_val=1) - # check_argument('tb_plot_step', c, restricted=True, val_type=int, min_val=1) - # check_argument('save_step', c, restricted=True, val_type=int, min_val=1) - # check_argument('checkpoint', c, restricted=True, val_type=bool) - # check_argument('tb_model_param_stats', c, restricted=True, val_type=bool) +# # tensorboard +# # check_argument('print_step', c, restricted=True, val_type=int, min_val=1) +# # check_argument('tb_plot_step', c, restricted=True, val_type=int, min_val=1) +# # check_argument('save_step', c, restricted=True, val_type=int, min_val=1) +# # check_argument('checkpoint', c, restricted=True, val_type=bool) +# # check_argument('tb_model_param_stats', c, restricted=True, val_type=bool) - # dataloading - # pylint: disable=import-outside-toplevel - from TTS.tts.utils.text import cleaners - # check_argument('text_cleaner', c, restricted=True, val_type=str, enum_list=dir(cleaners)) - # check_argument('enable_eos_bos_chars', c, restricted=True, val_type=bool) - # check_argument('num_loader_workers', c, restricted=True, val_type=int, min_val=0) - # check_argument('num_val_loader_workers', c, restricted=True, val_type=int, min_val=0) - # check_argument('batch_group_size', c, restricted=True, val_type=int, min_val=0) - # check_argument('min_seq_len', c, restricted=True, val_type=int, min_val=0) - # check_argument('max_seq_len', c, restricted=True, val_type=int, min_val=10) - # check_argument('compute_input_seq_cache', c, restricted=True, val_type=bool) +# # dataloading +# # pylint: disable=import-outside-toplevel +# from TTS.tts.utils.text import cleaners +# # check_argument('text_cleaner', c, restricted=True, val_type=str, enum_list=dir(cleaners)) +# # check_argument('enable_eos_bos_chars', c, restricted=True, val_type=bool) +# # check_argument('num_loader_workers', c, restricted=True, val_type=int, min_val=0) +# # check_argument('num_val_loader_workers', c, restricted=True, val_type=int, min_val=0) +# # check_argument('batch_group_size', c, restricted=True, val_type=int, min_val=0) +# # check_argument('min_seq_len', c, restricted=True, val_type=int, min_val=0) +# # check_argument('max_seq_len', c, restricted=True, val_type=int, min_val=10) +# # check_argument('compute_input_seq_cache', c, restricted=True, val_type=bool) - # paths - # check_argument('output_path', c, restricted=True, val_type=str) +# # paths +# # check_argument('output_path', c, restricted=True, val_type=str) - # multi-speaker and gst - # check_argument('use_speaker_embedding', c, restricted=True, val_type=bool) - # check_argument('use_external_speaker_embedding_file', c, restricted=c['use_speaker_embedding'], val_type=bool) - # check_argument('external_speaker_embedding_file', c, restricted=c['use_external_speaker_embedding_file'], val_type=str) - if c['model'].lower() in ['tacotron', 'tacotron2'] and c['use_gst']: - # check_argument('use_gst', c, restricted=is_tacotron(c), val_type=bool) - # check_argument('gst', c, restricted=is_tacotron(c), val_type=dict) - # check_argument('gst_style_input', c['gst'], restricted=is_tacotron(c), val_type=[str, dict]) - # check_argument('gst_embedding_dim', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=0, max_val=1000) - # check_argument('gst_use_speaker_embedding', c['gst'], restricted=is_tacotron(c), val_type=bool) - # check_argument('gst_num_heads', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=2, max_val=10) - # check_argument('gst_num_style_tokens', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=1, max_val=1000) +# # multi-speaker and gst +# # check_argument('use_speaker_embedding', c, restricted=True, val_type=bool) +# # check_argument('use_external_speaker_embedding_file', c, restricted=c['use_speaker_embedding'], val_type=bool) +# # check_argument('external_speaker_embedding_file', c, restricted=c['use_external_speaker_embedding_file'], val_type=str) +# if c['model'].lower() in ['tacotron', 'tacotron2'] and c['use_gst']: +# # check_argument('use_gst', c, restricted=is_tacotron(c), val_type=bool) +# # check_argument('gst', c, restricted=is_tacotron(c), val_type=dict) +# # check_argument('gst_style_input', c['gst'], restricted=is_tacotron(c), val_type=[str, dict]) +# # check_argument('gst_embedding_dim', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=0, max_val=1000) +# # check_argument('gst_use_speaker_embedding', c['gst'], restricted=is_tacotron(c), val_type=bool) +# # check_argument('gst_num_heads', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=2, max_val=10) +# # check_argument('gst_num_style_tokens', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=1, max_val=1000) - # datasets - checking only the first entry - # check_argument('datasets', c, restricted=True, val_type=list) - # for dataset_entry in c['datasets']: - # check_argument('name', dataset_entry, restricted=True, val_type=str) - # check_argument('path', dataset_entry, restricted=True, val_type=str) - # check_argument('meta_file_train', dataset_entry, restricted=True, val_type=[str, list]) - # check_argument('meta_file_val', dataset_entry, restricted=True, val_type=str) +# # datasets - checking only the first entry +# # check_argument('datasets', c, restricted=True, val_type=list) +# # for dataset_entry in c['datasets']: +# # check_argument('name', dataset_entry, restricted=True, val_type=str) +# # check_argument('path', dataset_entry, restricted=True, val_type=str) +# # check_argument('meta_file_train', dataset_entry, restricted=True, val_type=[str, list]) +# # check_argument('meta_file_val', dataset_entry, restricted=True, val_type=str) diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 4f6e2317..364baaf9 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -8,12 +8,10 @@ import json import os import re -import torch - from TTS.tts.utils.text.symbols import parse_symbols from TTS.utils.console_logger import ConsoleLogger from TTS.utils.generic_utils import create_experiment_folder, get_git_branch -from TTS.utils.io import copy_model_files, load_config +from TTS.utils.io import copy_model_files from TTS.utils.tensorboard_logger import TensorboardLogger @@ -140,11 +138,11 @@ def process_args(args, config, tb_prefix): if not args.best_path: args.best_path = best_model # setup output paths and read configs - c = config.load_json(args.config_path) - if c.mixed_precision: + config.load_json(args.config_path) + if config.mixed_precision: print(" > Mixed precision mode is ON") - if not os.path.exists(c.output_path): - out_path = create_experiment_folder(c.output_path, c.run_name, + if not os.path.exists(config.output_path): + out_path = create_experiment_folder(config.output_path, config.run_name, args.debug) audio_path = os.path.join(out_path, "test_audios") # setup rank 0 process in distributed training @@ -157,7 +155,7 @@ def process_args(args, config, tb_prefix): # if model characters are not set in the config file # save the default set to the config file for future # compatibility. - if c.has('characters_config'): + if config.has('characters_config'): used_characters = parse_symbols() new_fields["characters"] = used_characters copy_model_files(c, args.config_path, out_path, new_fields) @@ -166,6 +164,6 @@ def process_args(args, config, tb_prefix): log_path = out_path tb_logger = TensorboardLogger(log_path, model_name=tb_prefix) # write model desc to tensorboard - tb_logger.tb_add_text("model-description", c["run_description"], 0) + tb_logger.tb_add_text("model-description", config["run_description"], 0) c_logger = ConsoleLogger() return c, out_path, audio_path, c_logger, tb_logger diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index a3a604df..87307032 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -1,6 +1,10 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- import datetime import glob +import importlib import os +import re import shutil import subprocess import sys @@ -67,6 +71,20 @@ def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad) +def to_camel(text): + text = text.capitalize() + text = re.sub(r'(?!^)_([a-zA-Z])', lambda m: m.group(1).upper(), text) + text = text.replace('Tts', 'TTS') + return text + + +def find_module(module_path: str, module_name: str) -> object: + module_name = module_name.lower() + module = importlib.import_module(module_path+'.'+module_name) + class_name = to_camel(module_name) + return getattr(module, class_name) + + def get_user_data_dir(appname): if sys.platform == "win32": import winreg # pylint: disable=import-outside-toplevel @@ -139,32 +157,3 @@ class KeepAverage: for key, value in value_dict.items(): self.update_value(key, value) - -def check_argument(name, - c, - prerequest=None, - enum_list=None, - max_val=None, - min_val=None, - restricted=False, - alternative=None, - allow_none=False): - if isinstance(prerequest, List()): - if any([f not in c.keys() for f in prerequest]): - return - else: - if prerequest not in c.keys(): - return - if alternative in c.keys() and c[alternative] is not None: - return - if allow_none and c[name] is None: - return - if restricted: - assert name in c.keys(), f" [!] {name} not defined in config.json" - if name in c.keys(): - if max_val: - assert c[name] <= max_val, f" [!] {name} is larger than max value {max_val}" - if min_val: - assert c[name] >= min_val, f" [!] {name} is smaller than min value {min_val}" - if enum_list: - assert c[name].lower() in enum_list, f' [!] {name} is not a valid value' diff --git a/TTS/utils/io.py b/TTS/utils/io.py index 12745459..2d5662eb 100644 --- a/TTS/utils/io.py +++ b/TTS/utils/io.py @@ -3,6 +3,7 @@ import os import pickle as pickle_tts import re from shutil import copyfile +from TTS.utils.generic_utils import find_module import yaml @@ -23,32 +24,37 @@ class AttrDict(dict): self.__dict__ = self -# def read_json_with_comments(json_path): -# # fallback to json -# with open(json_path, "r", encoding="utf-8") as f: -# input_str = f.read() -# # handle comments -# input_str = re.sub(r'\\\n', '', input_str) -# input_str = re.sub(r'//.*\n', '\n', input_str) -# data = json.loads(input_str) -# return data +def read_json_with_comments(json_path): + """DEPRECATED""" + # fallback to json + with open(json_path, "r", encoding="utf-8") as f: + input_str = f.read() + # handle comments + input_str = re.sub(r'\\\n', '', input_str) + input_str = re.sub(r'//.*\n', '\n', input_str) + data = json.loads(input_str) + return data -# def load_config(config_path: str) -> AttrDict: -# """Load config files and discard comments +def load_config(config_path: str) -> AttrDict: + """DEPRECATED: Load config files and discard comments -# Args: -# config_path (str): path to config file. -# """ -# config = AttrDict() - -# ext = os.path.splitext(config_path)[1] -# # if ext in (".yml", ".yaml"): -# # with open(config_path, "r", encoding="utf-8") as f: -# # data = yaml.safe_load(f) -# # else: -# data = read_json_with_comments(config_path) -# config.update(data) -# return config + Args: + config_path (str): path to config file. + """ + config_dict = AttrDict() + ext = os.path.splitext(config_path)[1] + if ext in (".yml", ".yaml"): + with open(config_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + else: + with open(config_path, "r", encoding="utf-8") as f: + input_str = f.read() + data = json.loads(input_str) + config_dict.update(data) + config_class = find_module('TTS.tts.configs', config_dict.model.lower()+'_config') + config = config_class() + config.from_dict(config_dict) + return def copy_model_files(c, config_file, out_path, new_fields): From 79d721514233f1770352369e5abcf6d388bfd4ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 2 Apr 2021 14:24:12 +0200 Subject: [PATCH 19/87] config refactor #5 WIP --- TTS/bin/compute_statistics.py | 32 ++-- TTS/bin/train_tacotron.py | 339 +++++++++++++++------------------ TTS/tts/datasets/preprocess.py | 4 +- TTS/tts/utils/generic_utils.py | 12 +- TTS/utils/arguments.py | 72 ++++--- TTS/utils/io.py | 21 +- 6 files changed, 236 insertions(+), 244 deletions(-) diff --git a/TTS/bin/compute_statistics.py b/TTS/bin/compute_statistics.py index a8c4240e..f333e55b 100755 --- a/TTS/bin/compute_statistics.py +++ b/TTS/bin/compute_statistics.py @@ -8,7 +8,6 @@ import os import numpy as np from tqdm import tqdm -from TTS.utils.config_manager import ConfigManager from TTS.tts.datasets.preprocess import load_meta_data from TTS.utils.audio import AudioProcessor from TTS.utils.io import load_config @@ -16,8 +15,6 @@ from TTS.utils.io import load_config def main(): """Run preprocessing process.""" - CONFIG = ConfigManager() - parser = argparse.ArgumentParser( description="Compute mean and variance of spectrogtram features.") parser.add_argument("config_path", type=str, @@ -26,17 +23,17 @@ def main(): help="save path (directory and filename).") parser.add_argument("--data_path", type=str, required=False, help="folder including the target set of wavs overriding dataset config.") - parser = CONFIG.init_argparse(parser) - args = parser.parse_args() - CONFIG.parse_argparse(args) + args, overrides = parser.parse_known_args() + + CONFIG = load_config(args.config_path) + CONFIG.parse_args(overrides) # load config - CONFIG.load_config(args.config_path) - CONFIG.audio_config.signal_norm = False # do not apply earlier normalization - CONFIG.audio_config.stats_path = None # discard pre-defined stats + CONFIG.audio.signal_norm = False # do not apply earlier normalization + CONFIG.audio.stats_path = None # discard pre-defined stats # load audio processor - ap = AudioProcessor(**CONFIG.audio_config.to_dict()) + ap = AudioProcessor(**CONFIG.audio.to_dict()) # load the meta data of target dataset if args.data_path: @@ -81,15 +78,14 @@ def main(): print(f" > Avg lienar spec scale: {linear_scale.mean()}") # set default config values for mean-var scaling - CONFIG.audio_config.stats_path = output_file_path - CONFIG.audio_config.signal_norm = True + CONFIG.audio.stats_path = output_file_path + CONFIG.audio.signal_norm = True # remove redundant values - del CONFIG.audio_config.max_norm - del CONFIG.audio_config.min_level_db - del CONFIG.audio_config.symmetric_norm - del CONFIG.audio_config.clip_norm - breakpoint() - stats['audio_config'] = CONFIG.audio_config.to_dict() + del CONFIG.audio.max_norm + del CONFIG.audio.min_level_db + del CONFIG.audio.symmetric_norm + del CONFIG.audio.clip_norm + stats['audio_config'] = CONFIG.audio.to_dict() np.save(output_file_path, stats, allow_pickle=True) print(f" > stats saved to {output_file_path}") diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index 65bb4da6..a69008b8 100755 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -20,9 +20,8 @@ from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.synthesis import synthesis from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor -from TTS.utils.config_manager import ConfigManager from TTS.utils.distribute import (DistributedSampler, apply_gradient_allreduce, init_distributed, reduce_tensor) from TTS.utils.generic_utils import (KeepAverage, count_parameters, @@ -41,47 +40,49 @@ use_cuda, num_gpus = setup_torch_training_env(True, False) def setup_loader(ap, r, is_val=False, verbose=False, dataset=None): - if is_val and not c.run_eval: + if is_val and not config.run_eval: loader = None else: if dataset is None: dataset = MyDataset( r, - c.text_cleaner, - compute_linear_spec=c.model.lower() == "tacotron", + config.text_cleaner, + compute_linear_spec=config.model.lower() == 'tacotron', meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, - tp=c.characters if "characters" in c.keys() else None, - add_blank=c["add_blank"] if "add_blank" in c.keys() else False, - batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, - min_seq_len=c.min_seq_len, - max_seq_len=c.max_seq_len, - phoneme_cache_path=c.phoneme_cache_path, - use_phonemes=c.use_phonemes, - phoneme_language=c.phoneme_language, - enable_eos_bos=c.enable_eos_bos_chars, + tp=config.characters, + add_blank=config['add_blank'], + batch_group_size=0 if is_val else config.batch_group_size * + config.batch_size, + min_seq_len=config.min_seq_len, + max_seq_len=config.max_seq_len, + phoneme_cache_path=config.phoneme_cache_path, + use_phonemes=config.use_phonemes, + phoneme_language=config.phoneme_language, + enable_eos_bos=config.enable_eos_bos_chars, verbose=verbose, - speaker_mapping=( - speaker_mapping if (c.use_speaker_embedding and c.use_external_speaker_embedding_file) else None - ), - ) + speaker_mapping=(speaker_mapping if ( + config.use_speaker_embedding + and config.use_external_speaker_embedding_file + ) else None) + ) - if c.use_phonemes and c.compute_input_seq_cache: + if config.use_phonemes and config.compute_input_seq_cache: # precompute phonemes to have a better estimate of sequence lengths. - dataset.compute_input_seq(c.num_loader_workers) + dataset.compute_input_seq(config.num_loader_workers) dataset.sort_items() sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader( dataset, - batch_size=c.eval_batch_size if is_val else c.batch_size, + batch_size=config.eval_batch_size if is_val else config.batch_size, shuffle=False, collate_fn=dataset.collate_fn, drop_last=False, sampler=sampler, - num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers, - pin_memory=False, - ) + num_workers=config.num_val_loader_workers + if is_val else config.num_loader_workers, + pin_memory=False) return loader @@ -90,15 +91,15 @@ def format_data(data): text_input = data[0] text_lengths = data[1] speaker_names = data[2] - linear_input = data[3] if c.model.lower() in ["tacotron"] else None + linear_input = data[3] if config.model in ["Tacotron"] else None mel_input = data[4] mel_lengths = data[5] stop_targets = data[6] max_text_length = torch.max(text_lengths.float()) max_spec_length = torch.max(mel_lengths.float()) - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: speaker_embeddings = data[8] speaker_ids = None else: @@ -110,8 +111,10 @@ def format_data(data): speaker_ids = None # set stop targets view, we predict a single stop token per iteration. - stop_targets = stop_targets.view(text_input.shape[0], stop_targets.size(1) // c.r, -1) - stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float().squeeze(2) + stop_targets = stop_targets.view(text_input.shape[0], + stop_targets.size(1) // config.r, -1) + stop_targets = (stop_targets.sum(2) > + 0.0).unsqueeze(2).float().squeeze(2) # dispatch data to GPU if use_cuda: @@ -119,7 +122,7 @@ def format_data(data): text_lengths = text_lengths.cuda(non_blocking=True) mel_input = mel_input.cuda(non_blocking=True) mel_lengths = mel_lengths.cuda(non_blocking=True) - linear_input = linear_input.cuda(non_blocking=True) if c.model.lower() in ["tacotron"] else None + linear_input = linear_input.cuda(non_blocking=True) if config.model.lower() in ["tacotron"] else None stop_targets = stop_targets.cuda(non_blocking=True) if speaker_ids is not None: speaker_ids = speaker_ids.cuda(non_blocking=True) @@ -145,9 +148,10 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, epoch_time = 0 keep_avg = KeepAverage() if use_cuda: - batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus)) + batch_n_iter = int( + len(data_loader.dataset) / (config.batch_size * num_gpus)) else: - batch_n_iter = int(len(data_loader.dataset) / c.batch_size) + batch_n_iter = int(len(data_loader.dataset) / config.batch_size) end_time = time.time() c_logger.print_train_start() for num_iter, data in enumerate(data_loader): @@ -171,31 +175,18 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, global_step += 1 # setup lr - if c.noam_schedule: + if config.noam_schedule: scheduler.step() optimizer.zero_grad() if optimizer_st: optimizer_st.zero_grad() - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): # forward pass model - if c.bidirectional_decoder or c.double_decoder_consistency: - ( - decoder_output, - postnet_output, - alignments, - stop_tokens, - decoder_backward_output, - alignments_backward, - ) = model( - text_input, - text_lengths, - mel_input, - mel_lengths, - speaker_ids=speaker_ids, - speaker_embeddings=speaker_embeddings, - ) + if config.bidirectional_decoder or config.double_decoder_consistency: + decoder_output, postnet_output, alignments, stop_tokens, decoder_backward_output, alignments_backward = model( + text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings) else: decoder_output, postnet_output, alignments, stop_tokens = model( text_input, @@ -237,18 +228,18 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, raise RuntimeError(f"Detected NaN loss at step {global_step}.") # optimizer step - if c.mixed_precision: + if config.mixed_precision: # model optimizer step in mixed precision mode scaler.scale(loss_dict["loss"]).backward() scaler.unscale_(optimizer) optimizer, current_lr = adam_weight_decay(optimizer) - grad_norm, _ = check_update(model, c.grad_clip, ignore_stopnet=True) + grad_norm, _ = check_update(model, config.grad_clip, ignore_stopnet=True) scaler.step(optimizer) scaler.update() # stopnet optimizer step - if c.separate_stopnet: - scaler_st.scale(loss_dict["stopnet_loss"]).backward() + if config.separate_stopnet: + scaler_st.scale(loss_dict['stopnet_loss']).backward() scaler.unscale_(optimizer_st) optimizer_st, _ = adam_weight_decay(optimizer_st) grad_norm_st, _ = check_update(model.decoder.stopnet, 1.0) @@ -260,12 +251,12 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, # main model optimizer step loss_dict["loss"].backward() optimizer, current_lr = adam_weight_decay(optimizer) - grad_norm, _ = check_update(model, c.grad_clip, ignore_stopnet=True) + grad_norm, _ = check_update(model, config.grad_clip, ignore_stopnet=True) optimizer.step() # stopnet optimizer step - if c.separate_stopnet: - loss_dict["stopnet_loss"].backward() + if config.separate_stopnet: + loss_dict['stopnet_loss'].backward() optimizer_st, _ = adam_weight_decay(optimizer_st) grad_norm_st, _ = check_update(model.decoder.stopnet, 1.0) optimizer_st.step() @@ -281,12 +272,10 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, # aggregate losses from processes if num_gpus > 1: - loss_dict["postnet_loss"] = reduce_tensor(loss_dict["postnet_loss"].data, num_gpus) - loss_dict["decoder_loss"] = reduce_tensor(loss_dict["decoder_loss"].data, num_gpus) - loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) - loss_dict["stopnet_loss"] = ( - reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) if c.stopnet else loss_dict["stopnet_loss"] - ) + loss_dict['postnet_loss'] = reduce_tensor(loss_dict['postnet_loss'].data, num_gpus) + loss_dict['decoder_loss'] = reduce_tensor(loss_dict['decoder_loss'].data, num_gpus) + loss_dict['loss'] = reduce_tensor(loss_dict['loss'] .data, num_gpus) + loss_dict['stopnet_loss'] = reduce_tensor(loss_dict['stopnet_loss'].data, num_gpus) if config.stopnet else loss_dict['stopnet_loss'] # detach loss values loss_dict_new = dict() @@ -306,7 +295,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, keep_avg.update_values(update_train_values) # print training progress - if global_step % c.print_step == 0: + if global_step % config.print_step == 0: log_dict = { "max_spec_length": [max_spec_length, 1], # value, precision "max_text_length": [max_text_length, 1], @@ -319,7 +308,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, if args.rank == 0: # Plot Training Iter Stats # reduce TB load - if global_step % c.tb_plot_step == 0: + if global_step % config.tb_plot_step == 0: iter_stats = { "lr": current_lr, "grad_norm": grad_norm, @@ -329,29 +318,20 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, iter_stats.update(loss_dict) tb_logger.tb_train_iter_stats(global_step, iter_stats) - if global_step % c.save_step == 0: - if c.checkpoint: + if global_step % config.save_step == 0: + if config.checkpoint: # save model - save_checkpoint( - model, - optimizer, - global_step, - epoch, - model.decoder.r, - OUT_PATH, - optimizer_st=optimizer_st, - model_loss=loss_dict["postnet_loss"], - characters=model_characters, - scaler=scaler.state_dict() if c.mixed_precision else None, - ) + save_checkpoint(model, optimizer, global_step, epoch, model.decoder.r, OUT_PATH, + optimizer_st=optimizer_st, + model_loss=loss_dict['postnet_loss'], + characters=model_characters, + scaler=scaler.state_dict() if config.mixed_precision else None) # Diagnostic visualizations const_spec = postnet_output[0].data.cpu().numpy() - gt_spec = ( - linear_input[0].data.cpu().numpy() - if c.model in ["Tacotron", "TacotronGST"] - else mel_input[0].data.cpu().numpy() - ) + gt_spec = linear_input[0].data.cpu().numpy() if config.model in [ + "Tacotron", "TacotronGST" + ] else mel_input[0].data.cpu().numpy() align_img = alignments[0].data.cpu().numpy() figures = { @@ -360,19 +340,19 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, "alignment": plot_alignment(align_img, output_fig=False), } - if c.bidirectional_decoder or c.double_decoder_consistency: - figures["alignment_backward"] = plot_alignment( - alignments_backward[0].data.cpu().numpy(), output_fig=False - ) + if config.bidirectional_decoder or config.double_decoder_consistency: + figures["alignment_backward"] = plot_alignment(alignments_backward[0].data.cpu().numpy(), output_fig=False) tb_logger.tb_train_figures(global_step, figures) # Sample audio - if c.model in ["Tacotron", "TacotronGST"]: - train_audio = ap.inv_spectrogram(const_spec.T) + if config.model in ["Tacotron", "TacotronGST"]: + train_audio = ap.inv_spectrogram(const_speconfig.T) else: - train_audio = ap.inv_melspectrogram(const_spec.T) - tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"]) + train_audio = ap.inv_melspectrogram(const_speconfig.T) + tb_logger.tb_train_audios(global_step, + {'TrainAudio': train_audio}, + config.audio["sample_rate"]) end_time = time.time() # print epoch stats @@ -383,7 +363,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, epoch_stats = {"epoch_time": epoch_time} epoch_stats.update(keep_avg.avg_values) tb_logger.tb_train_epoch_stats(global_step, epoch_stats) - if c.tb_model_param_stats: + if config.tb_model_param_stats: tb_logger.tb_model_weights(model, global_step) return keep_avg.avg_values, global_step @@ -414,17 +394,9 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): assert mel_input.shape[1] % model.decoder.r == 0 # forward pass model - if c.bidirectional_decoder or c.double_decoder_consistency: - ( - decoder_output, - postnet_output, - alignments, - stop_tokens, - decoder_backward_output, - alignments_backward, - ) = model( - text_input, text_lengths, mel_input, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings - ) + if config.bidirectional_decoder or config.double_decoder_consistency: + decoder_output, postnet_output, alignments, stop_tokens, decoder_backward_output, alignments_backward = model( + text_input, text_lengths, mel_input, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings) else: decoder_output, postnet_output, alignments, stop_tokens = model( text_input, text_lengths, mel_input, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings @@ -466,10 +438,10 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # aggregate losses from processes if num_gpus > 1: - loss_dict["postnet_loss"] = reduce_tensor(loss_dict["postnet_loss"].data, num_gpus) - loss_dict["decoder_loss"] = reduce_tensor(loss_dict["decoder_loss"].data, num_gpus) - if c.stopnet: - loss_dict["stopnet_loss"] = reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) + loss_dict['postnet_loss'] = reduce_tensor(loss_dict['postnet_loss'].data, num_gpus) + loss_dict['decoder_loss'] = reduce_tensor(loss_dict['decoder_loss'].data, num_gpus) + if config.stopnet: + loss_dict['stopnet_loss'] = reduce_tensor(loss_dict['stopnet_loss'].data, num_gpus) # detach loss values loss_dict_new = dict() @@ -486,18 +458,16 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): update_train_values["avg_" + key] = value keep_avg.update_values(update_train_values) - if c.print_eval: + if config.print_eval: c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) if args.rank == 0: # Diagnostic visualizations idx = np.random.randint(mel_input.shape[0]) const_spec = postnet_output[idx].data.cpu().numpy() - gt_spec = ( - linear_input[idx].data.cpu().numpy() - if c.model in ["Tacotron", "TacotronGST"] - else mel_input[idx].data.cpu().numpy() - ) + gt_spec = linear_input[idx].data.cpu().numpy() if config.model in [ + "Tacotron", "TacotronGST" + ] else mel_input[idx].data.cpu().numpy() align_img = alignments[idx].data.cpu().numpy() eval_figures = { @@ -507,22 +477,23 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): } # Sample audio - if c.model in ["Tacotron", "TacotronGST"]: - eval_audio = ap.inv_spectrogram(const_spec.T) + if config.model in ["Tacotron", "TacotronGST"]: + eval_audio = ap.inv_spectrogram(const_speconfig.T) else: - eval_audio = ap.inv_melspectrogram(const_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"]) + eval_audio = ap.inv_melspectrogram(const_speconfig.T) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, + config.audio["sample_rate"]) # Plot Validation Stats - if c.bidirectional_decoder or c.double_decoder_consistency: + if config.bidirectional_decoder or config.double_decoder_consistency: align_b_img = alignments_backward[idx].data.cpu().numpy() eval_figures["alignment2"] = plot_alignment(align_b_img, output_fig=False) tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) tb_logger.tb_eval_figures(global_step, eval_figures) - if args.rank == 0 and epoch > c.test_delay_epochs: - if c.test_sentences_file is None: + if args.rank == 0 and epoch > config.test_delay_epochs: + if config.test_sentences_file is None: test_sentences = [ "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", "Be a voice, not an echo.", @@ -531,40 +502,36 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): "Prior to November 22, 1963.", ] else: - with open(c.test_sentences_file, "r") as f: + with open(config.test_sentences_file, "r") as f: test_sentences = [s.strip() for s in f.readlines()] # test sentences test_audios = {} test_figures = {} print(" | > Synthesizing test sentences") - speaker_id = 0 if c.use_speaker_embedding else None - speaker_embedding = ( - speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]]["embedding"] - if c.use_external_speaker_embedding_file and c.use_speaker_embedding - else None - ) - style_wav = c.get("gst_style_input") - if style_wav is None and c.use_gst: + speaker_id = 0 if config.use_speaker_embedding else None + speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping)-1)]]['embedding'] if config.use_external_speaker_embedding_file and config.use_speaker_embedding else None + style_wav = config.get("gst_style_input") + if style_wav is None and config.use_gst: # inicialize GST with zero dict. style_wav = {} print("WARNING: You don't provided a gst style wav, for this reason we use a zero tensor!") - for i in range(c.gst['gst_num_style_tokens']): + for i in range(config.gst['gst_num_style_tokens']): style_wav[str(i)] = 0 - style_wav = c.get("gst_style_input", style_wav) + style_wav = config.get("gst_style_input") for idx, test_sentence in enumerate(test_sentences): try: wav, alignment, decoder_output, postnet_output, stop_tokens, _ = synthesis( model, test_sentence, - c, + config, use_cuda, ap, speaker_id=speaker_id, speaker_embedding=speaker_embedding, style_wav=style_wav, truncated=False, - enable_eos_bos_chars=c.enable_eos_bos_chars, # pylint: disable=unused-argument + enable_eos_bos_chars=config.enable_eos_bos_chars, #pylint: disable=unused-argument use_griffin_lim=True, do_trim_silence=False, ) @@ -579,7 +546,8 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): except: # pylint: disable=bare-except print(" !! Error creating Test Sentence -", idx) traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"]) + tb_logger.tb_test_audios(global_step, test_audios, + config.audio['sample_rate']) tb_logger.tb_test_figures(global_step, test_figures) return keep_avg.avg_values @@ -588,45 +556,48 @@ def main(args): # pylint: disable=redefined-outer-name # pylint: disable=global-variable-undefined global meta_data_train, meta_data_eval, speaker_mapping, symbols, phonemes, model_characters # Audio processor - ap = AudioProcessor(**c.audio) + ap = AudioProcessor(**config.audio.to_dict()) # setup custom characters if set in config file. - if "characters" in c.keys(): - symbols, phonemes = make_symbols(**c.characters) + if config.characters is not None: + symbols, phonemes = make_symbols(**config.characters.to_dict()) # DISTRUBUTED if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"]) - num_chars = len(phonemes) if c.use_phonemes else len(symbols) - model_characters = phonemes if c.use_phonemes else symbols + init_distributed(args.rank, num_gpus, args.group_id, + config.distributed["backend"], config.distributed["url"]) + num_chars = len(phonemes) if config.use_phonemes else len(symbols) + model_characters = phonemes if config.use_phonemes else symbols # load data instances - meta_data_train, meta_data_eval = load_meta_data(c.datasets) + meta_data_train, meta_data_eval = load_meta_data(config.datasets) # set the portion of the data used for training - if "train_portion" in c.keys(): - meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)] - if "eval_portion" in c.keys(): - meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)] + if config.has('train_portion'): + meta_data_train = meta_data_train[:int(len(meta_data_train) * config.train_portion)] + if config.has('eval_portion'): + meta_data_eval = meta_data_eval[:int(len(meta_data_eval) * config.eval_portion)] # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH) + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) - model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim) + model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim) # scalers for mixed precision training - scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None - scaler_st = torch.cuda.amp.GradScaler() if c.mixed_precision and c.separate_stopnet else None + scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None + scaler_st = torch.cuda.amp.GradScaler() if config.mixed_precision and config.separate_stopnet else None - params = set_weight_decay(model, c.wd) - optimizer = RAdam(params, lr=c.lr, weight_decay=0) - if c.stopnet and c.separate_stopnet: - optimizer_st = RAdam(model.decoder.stopnet.parameters(), lr=c.lr, weight_decay=0) + params = set_weight_decay(model, config.wd) + optimizer = RAdam(params, lr=config.lr, weight_decay=0) + if config.stopnet and config.separate_stopnet: + optimizer_st = RAdam(model.decoder.stopnet.parameters(), + lr=config.lr, + weight_decay=0) else: optimizer_st = None # setup criterion - criterion = TacotronLoss(c, stopnet_pos_weight=c.stopnet_pos_weight, ga_sigma=0.4) + criterion = TacotronLoss(config, stopnet_pos_weight=config.stopnet_pos_weight, ga_sigma=0.4) if args.restore_path: print(f" > Restoring from {os.path.basename(args.restore_path)}...") checkpoint = torch.load(args.restore_path, map_location="cpu") @@ -635,11 +606,11 @@ def main(args): # pylint: disable=redefined-outer-name model.load_state_dict(checkpoint["model"]) # optimizer restore print(" > Restoring Optimizer...") - optimizer.load_state_dict(checkpoint["optimizer"]) - if "scaler" in checkpoint and c.mixed_precision: + optimizer.load_state_dict(checkpoint['optimizer']) + if "scaler" in checkpoint and config.mixed_precision: print(" > Restoring AMP Scaler...") scaler.load_state_dict(checkpoint["scaler"]) - if c.reinit_layers: + if config.reinit_layers: raise RuntimeError except (KeyError, RuntimeError): print(" > Partial model initialization...") @@ -651,9 +622,10 @@ def main(args): # pylint: disable=redefined-outer-name del model_dict for group in optimizer.param_groups: - group["lr"] = c.lr - print(" > Model restored from step %d" % checkpoint["step"], flush=True) - args.restore_step = checkpoint["step"] + group['lr'] = config.lr + print(" > Model restored from step %d" % checkpoint['step'], + flush=True) + args.restore_step = checkpoint['step'] else: args.restore_step = 0 @@ -665,8 +637,10 @@ def main(args): # pylint: disable=redefined-outer-name if num_gpus > 1: model = apply_gradient_allreduce(model) - if c.noam_schedule: - scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1) + if config.noam_schedule: + scheduler = NoamLR(optimizer, + warmup_steps=config.warmup_steps, + last_epoch=args.restore_step - 1) else: scheduler = None @@ -680,22 +654,22 @@ def main(args): # pylint: disable=redefined-outer-name print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] print(f" > Starting with loaded last best loss {best_loss}.") - keep_all_best = c.get("keep_all_best", False) - keep_after = c.get("keep_after", 10000) # void if keep_all_best False + keep_all_best = config.keep_all_best + keep_after = config.keep_after # void if keep_all_best False # define data loaders train_loader = setup_loader(ap, model.decoder.r, is_val=False, verbose=True) eval_loader = setup_loader(ap, model.decoder.r, is_val=True) global_step = args.restore_step - for epoch in range(0, c.epochs): - c_logger.print_epoch_start(epoch, c.epochs) + for epoch in range(0, config.epochs): + c_logger.print_epoch_start(epoch, config.epochs) # set gradual training - if c.gradual_training is not None: - r, c.batch_size = gradual_training_scheduler(global_step, c) - c.r = r + if config.gradual_training is not None: + r, config.batch_size = gradual_training_scheduler(global_step, c) + config.r = r model.decoder.set_r(r) - if c.bidirectional_decoder: + if config.bidirectional_decoder: model.decoder_backward.set_r(r) train_loader.dataset.outputs_per_step = r eval_loader.dataset.outputs_per_step = r @@ -719,9 +693,9 @@ def main(args): # pylint: disable=redefined-outer-name # eval one epoch eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) - target_loss = train_avg_loss_dict["avg_postnet_loss"] - if c.run_eval: - target_loss = eval_avg_loss_dict["avg_postnet_loss"] + target_loss = train_avg_loss_dict['avg_postnet_loss'] + if config.run_eval: + target_loss = eval_avg_loss_dict['avg_postnet_loss'] best_loss = save_best_model( target_loss, best_loss, @@ -729,31 +703,26 @@ def main(args): # pylint: disable=redefined-outer-name optimizer, global_step, epoch, - c.r, + config.r, OUT_PATH, model_characters, keep_all_best=keep_all_best, keep_after=keep_after, - scaler=scaler.state_dict() if c.mixed_precision else None, + scaler=scaler.state_dict() if config.mixed_precision else None ) -if __name__ == "__main__": - args = parse_arguments(sys.argv) - c = TacotronConfig() - args = c.init_argparse(args) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args( - args, c, model_type='tacotron') - +if __name__ == '__main__': + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) except KeyboardInterrupt: - remove_experiment_folder(OUT_PATH) + # remove_experiment_folder(OUT_PATH) try: sys.exit(0) except SystemExit: os._exit(0) # pylint: disable=protected-access except Exception: # pylint: disable=broad-except - remove_experiment_folder(OUT_PATH) + # remove_experiment_folder(OUT_PATH) traceback.print_exc() sys.exit(1) diff --git a/TTS/tts/datasets/preprocess.py b/TTS/tts/datasets/preprocess.py index 0f82814d..d6040493 100644 --- a/TTS/tts/datasets/preprocess.py +++ b/TTS/tts/datasets/preprocess.py @@ -37,8 +37,8 @@ def load_meta_data(datasets, eval_split=True): meta_data_eval_all += meta_data_eval meta_data_train_all += meta_data_train # load attention masks for duration predictor training - if "meta_file_attn_mask" in dataset and dataset["meta_file_attn_mask"] is not None: - meta_data = dict(load_attention_mask_meta_data(dataset["meta_file_attn_mask"])) + if dataset.meta_file_attn_mask is not None: + meta_data = dict(load_attention_mask_meta_data(dataset['meta_file_attn_mask'])) for idx, ins in enumerate(meta_data_train_all): attn_file = meta_data[ins[1]].strip() meta_data_train_all[idx].append(attn_file) diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index 0e80111b..9711c868 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -38,7 +38,7 @@ def sequence_mask(sequence_length, max_len=None): def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): print(" > Using model: {}".format(c.model)) - find_module("TTS.tts.models", c.model.lower()) + MyModel = find_module("TTS.tts.models", c.model.lower()) if c.model.lower() in "tacotron": model = MyModel( num_chars=num_chars + getattr(c, "add_blank", False), @@ -76,11 +76,11 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): r=c.r, postnet_output_dim=c.audio["num_mels"], decoder_output_dim=c.audio["num_mels"], - gst=c.use_gst, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"], + gst=c.gst is not None, + gst_embedding_dim=None if c.gst is None else c.gst['gst_embedding_dim'], + gst_num_heads=None if c.gst is None else c.gst['gst_num_heads'], + gst_num_style_tokens=None if c.gst is None else c.gst['gst_num_style_tokens'], + gst_use_speaker_embedding=None if c.gst is None else c.gst['gst_use_speaker_embedding'], attn_type=c.attention_type, attn_win=c.windowing, attn_norm=c.attention_norm, diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 364baaf9..55717c7f 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -6,16 +6,17 @@ import argparse import glob import json import os +import sys import re from TTS.tts.utils.text.symbols import parse_symbols from TTS.utils.console_logger import ConsoleLogger from TTS.utils.generic_utils import create_experiment_folder, get_git_branch -from TTS.utils.io import copy_model_files +from TTS.utils.io import copy_model_files, load_config from TTS.utils.tensorboard_logger import TensorboardLogger -def parse_arguments(argv): +def init_arguments(argv): """Parse command line arguments of training scripts. Args: @@ -45,16 +46,26 @@ def parse_arguments(argv): "Best model file to be used for extracting best loss." "If not specified, the latest best model in continue path is used" ), - default="", - ) + default="") + parser.add_argument("--config_path", + type=str, + help="Path to config file for training.", + required="--continue_path" not in argv) + parser.add_argument("--debug", + type=bool, + default=False, + help="Do not verify commit integrity to run training.") parser.add_argument( - "--config_path", type=str, help="Path to config file for training.", required="--continue_path" not in argv - ) - parser.add_argument("--debug", type=bool, default=False, help="Do not verify commit integrity to run training.") - parser.add_argument("--rank", type=int, default=0, help="DISTRIBUTED: process rank for distributed training.") - parser.add_argument("--group_id", type=str, default="", help="DISTRIBUTED: process group id.") + "--rank", + type=int, + default=0, + help="DISTRIBUTED: process rank for distributed training.") + parser.add_argument("--group_id", + type=str, + default="", + help="DISTRIBUTED: process group id.") - return parser.parse_args() + return parser def get_last_checkpoint(path): @@ -115,7 +126,7 @@ def get_last_checkpoint(path): return last_models["checkpoint"], last_models["best_model"] -def process_args(args, config, tb_prefix): +def process_args(args): """Process parsed comand line arguments. Args: @@ -130,21 +141,27 @@ def process_args(args, config, tb_prefix): tb_logger (TTS.utils.tensorboard.TensorboardLogger): Class that does the TensorBoard loggind. """ + if isinstance(args, tuple): + args, coqpit_overrides = args if args.continue_path: # continue a previous training from its output folder - args.output_path = args.continue_path + experiment_path = args.continue_path args.config_path = os.path.join(args.continue_path, "config.json") args.restore_path, best_model = get_last_checkpoint(args.continue_path) if not args.best_path: args.best_path = best_model # setup output paths and read configs - config.load_json(args.config_path) + config = load_config(args.config_path) + # override values from command-line args + config.parse_args(coqpit_overrides) if config.mixed_precision: print(" > Mixed precision mode is ON") if not os.path.exists(config.output_path): - out_path = create_experiment_folder(config.output_path, config.run_name, - args.debug) - audio_path = os.path.join(out_path, "test_audios") + experiment_path = create_experiment_folder(config.output_path, + config.run_name, args.debug) + else: + experiment_path = config.output_path + audio_path = os.path.join(experiment_path, "test_audios") # setup rank 0 process in distributed training if args.rank == 0: os.makedirs(audio_path, exist_ok=True) @@ -157,13 +174,22 @@ def process_args(args, config, tb_prefix): # compatibility. if config.has('characters_config'): used_characters = parse_symbols() - new_fields["characters"] = used_characters - copy_model_files(c, args.config_path, out_path, new_fields) + new_fields['characters'] = used_characters + copy_model_files(config, args.config_path, experiment_path, new_fields) os.chmod(audio_path, 0o775) - os.chmod(out_path, 0o775) - log_path = out_path - tb_logger = TensorboardLogger(log_path, model_name=tb_prefix) + os.chmod(experiment_path, 0o775) + tb_logger = TensorboardLogger(experiment_path, + model_name=config.model) # write model desc to tensorboard - tb_logger.tb_add_text("model-description", config["run_description"], 0) + tb_logger.tb_add_text("model-description", config["run_description"], + 0) c_logger = ConsoleLogger() - return c, out_path, audio_path, c_logger, tb_logger + return config, experiment_path, audio_path, c_logger, tb_logger + + +def init_training(argv): + """Initialization of a training run.""" + parser = init_arguments(argv) + args = parser.parse_known_args() + config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args) + return args[0], config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger diff --git a/TTS/utils/io.py b/TTS/utils/io.py index 2d5662eb..58e6dd69 100644 --- a/TTS/utils/io.py +++ b/TTS/utils/io.py @@ -3,9 +3,11 @@ import os import pickle as pickle_tts import re from shutil import copyfile -from TTS.utils.generic_utils import find_module import yaml +from TTS.utils.generic_utils import find_module + +from .generic_utils import find_module class RenamingUnpickler(pickle_tts.Unpickler): @@ -35,26 +37,25 @@ def read_json_with_comments(json_path): data = json.loads(input_str) return data -def load_config(config_path: str) -> AttrDict: - """DEPRECATED: Load config files and discard comments - Args: - config_path (str): path to config file. - """ - config_dict = AttrDict() +def load_config(config_path: str) -> None: + config_dict = {} ext = os.path.splitext(config_path)[1] if ext in (".yml", ".yaml"): with open(config_path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) - else: + elif ext == '.json': with open(config_path, "r", encoding="utf-8") as f: input_str = f.read() data = json.loads(input_str) + else: + raise TypeError(f' [!] Unknown config file type {ext}') config_dict.update(data) - config_class = find_module('TTS.tts.configs', config_dict.model.lower()+'_config') + config_class = find_module('TTS.tts.configs', config_dict['model'].lower()+'_config') config = config_class() config.from_dict(config_dict) - return + return config + def copy_model_files(c, config_file, out_path, new_fields): From c34c8137d73dce183ce7ee1b28f92f506c3e1945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 3 May 2021 16:39:55 +0200 Subject: [PATCH 20/87] update compute_statistics for coqpit --- TTS/bin/compute_statistics.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/TTS/bin/compute_statistics.py b/TTS/bin/compute_statistics.py index f333e55b..d87ecf95 100755 --- a/TTS/bin/compute_statistics.py +++ b/TTS/bin/compute_statistics.py @@ -10,19 +10,22 @@ from tqdm import tqdm from TTS.tts.datasets.preprocess import load_meta_data from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config + +# from TTS.utils.io import load_config +from TTS.utils.config import load_config def main(): """Run preprocessing process.""" - parser = argparse.ArgumentParser( - description="Compute mean and variance of spectrogtram features.") - parser.add_argument("config_path", type=str, - help="TTS config file path to define audio processin parameters.") - parser.add_argument("out_path", type=str, - help="save path (directory and filename).") - parser.add_argument("--data_path", type=str, required=False, - help="folder including the target set of wavs overriding dataset config.") + parser = argparse.ArgumentParser(description="Compute mean and variance of spectrogtram features.") + parser.add_argument("config_path", type=str, help="TTS config file path to define audio processin parameters.") + parser.add_argument("out_path", type=str, help="save path (directory and filename).") + parser.add_argument( + "--data_path", + type=str, + required=False, + help="folder including the target set of wavs overriding dataset config.", + ) args, overrides = parser.parse_known_args() CONFIG = load_config(args.config_path) @@ -37,7 +40,7 @@ def main(): # load the meta data of target dataset if args.data_path: - dataset_items = glob.glob(os.path.join(args.data_path, '**', '*.wav'), recursive=True) + dataset_items = glob.glob(os.path.join(args.data_path, "**", "*.wav"), recursive=True) else: dataset_items = load_meta_data(CONFIG.dataset_config)[0] # take only train data print(f" > There are {len(dataset_items)} files.") @@ -85,7 +88,7 @@ def main(): del CONFIG.audio.min_level_db del CONFIG.audio.symmetric_norm del CONFIG.audio.clip_norm - stats['audio_config'] = CONFIG.audio.to_dict() + stats["audio_config"] = CONFIG.audio.to_dict() np.save(output_file_path, stats, allow_pickle=True) print(f" > stats saved to {output_file_path}") From 9c18e40f64cae1c2e25fa0e2f89d2905094d9a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 3 May 2021 16:42:15 +0200 Subject: [PATCH 21/87] black formatting --- TTS/bin/train_tacotron.py | 182 +++++++++++++++++++++---------------- TTS/utils/arguments.py | 39 +++----- TTS/utils/generic_utils.py | 7 +- TTS/utils/io.py | 12 +-- 4 files changed, 126 insertions(+), 114 deletions(-) diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index a69008b8..f5d74099 100755 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -10,6 +10,7 @@ from random import randrange import numpy as np import torch from torch.utils.data import DataLoader + from TTS.tts.datasets.preprocess import load_meta_data from TTS.tts.datasets.TTSDataset import MyDataset from TTS.tts.layers.losses import TacotronLoss @@ -22,10 +23,8 @@ from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor -from TTS.utils.distribute import (DistributedSampler, apply_gradient_allreduce, - init_distributed, reduce_tensor) -from TTS.utils.generic_utils import (KeepAverage, count_parameters, - remove_experiment_folder, set_init_dict) +from TTS.utils.distribute import DistributedSampler, apply_gradient_allreduce, init_distributed, reduce_tensor +from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict from TTS.utils.radam import RAdam from TTS.utils.training import ( NoamLR, @@ -47,13 +46,12 @@ def setup_loader(ap, r, is_val=False, verbose=False, dataset=None): dataset = MyDataset( r, config.text_cleaner, - compute_linear_spec=config.model.lower() == 'tacotron', + compute_linear_spec=config.model.lower() == "tacotron", meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, tp=config.characters, - add_blank=config['add_blank'], - batch_group_size=0 if is_val else config.batch_group_size * - config.batch_size, + add_blank=config["add_blank"], + batch_group_size=0 if is_val else config.batch_group_size * config.batch_size, min_seq_len=config.min_seq_len, max_seq_len=config.max_seq_len, phoneme_cache_path=config.phoneme_cache_path, @@ -61,11 +59,12 @@ def setup_loader(ap, r, is_val=False, verbose=False, dataset=None): phoneme_language=config.phoneme_language, enable_eos_bos=config.enable_eos_bos_chars, verbose=verbose, - speaker_mapping=(speaker_mapping if ( - config.use_speaker_embedding - and config.use_external_speaker_embedding_file - ) else None) - ) + speaker_mapping=( + speaker_mapping + if (config.use_speaker_embedding and config.use_external_speaker_embedding_file) + else None + ), + ) if config.use_phonemes and config.compute_input_seq_cache: # precompute phonemes to have a better estimate of sequence lengths. @@ -80,9 +79,9 @@ def setup_loader(ap, r, is_val=False, verbose=False, dataset=None): collate_fn=dataset.collate_fn, drop_last=False, sampler=sampler, - num_workers=config.num_val_loader_workers - if is_val else config.num_loader_workers, - pin_memory=False) + num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers, + pin_memory=False, + ) return loader @@ -111,10 +110,8 @@ def format_data(data): speaker_ids = None # set stop targets view, we predict a single stop token per iteration. - stop_targets = stop_targets.view(text_input.shape[0], - stop_targets.size(1) // config.r, -1) - stop_targets = (stop_targets.sum(2) > - 0.0).unsqueeze(2).float().squeeze(2) + stop_targets = stop_targets.view(text_input.shape[0], stop_targets.size(1) // config.r, -1) + stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float().squeeze(2) # dispatch data to GPU if use_cuda: @@ -148,8 +145,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, epoch_time = 0 keep_avg = KeepAverage() if use_cuda: - batch_n_iter = int( - len(data_loader.dataset) / (config.batch_size * num_gpus)) + batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus)) else: batch_n_iter = int(len(data_loader.dataset) / config.batch_size) end_time = time.time() @@ -185,8 +181,21 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, with torch.cuda.amp.autocast(enabled=config.mixed_precision): # forward pass model if config.bidirectional_decoder or config.double_decoder_consistency: - decoder_output, postnet_output, alignments, stop_tokens, decoder_backward_output, alignments_backward = model( - text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings) + ( + decoder_output, + postnet_output, + alignments, + stop_tokens, + decoder_backward_output, + alignments_backward, + ) = model( + text_input, + text_lengths, + mel_input, + mel_lengths, + speaker_ids=speaker_ids, + speaker_embeddings=speaker_embeddings, + ) else: decoder_output, postnet_output, alignments, stop_tokens = model( text_input, @@ -239,7 +248,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, # stopnet optimizer step if config.separate_stopnet: - scaler_st.scale(loss_dict['stopnet_loss']).backward() + scaler_st.scale(loss_dict["stopnet_loss"]).backward() scaler.unscale_(optimizer_st) optimizer_st, _ = adam_weight_decay(optimizer_st) grad_norm_st, _ = check_update(model.decoder.stopnet, 1.0) @@ -256,7 +265,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, # stopnet optimizer step if config.separate_stopnet: - loss_dict['stopnet_loss'].backward() + loss_dict["stopnet_loss"].backward() optimizer_st, _ = adam_weight_decay(optimizer_st) grad_norm_st, _ = check_update(model.decoder.stopnet, 1.0) optimizer_st.step() @@ -272,10 +281,12 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, # aggregate losses from processes if num_gpus > 1: - loss_dict['postnet_loss'] = reduce_tensor(loss_dict['postnet_loss'].data, num_gpus) - loss_dict['decoder_loss'] = reduce_tensor(loss_dict['decoder_loss'].data, num_gpus) - loss_dict['loss'] = reduce_tensor(loss_dict['loss'] .data, num_gpus) - loss_dict['stopnet_loss'] = reduce_tensor(loss_dict['stopnet_loss'].data, num_gpus) if config.stopnet else loss_dict['stopnet_loss'] + loss_dict["postnet_loss"] = reduce_tensor(loss_dict["postnet_loss"].data, num_gpus) + loss_dict["decoder_loss"] = reduce_tensor(loss_dict["decoder_loss"].data, num_gpus) + loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) + loss_dict["stopnet_loss"] = ( + reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) if config.stopnet else loss_dict["stopnet_loss"] + ) # detach loss values loss_dict_new = dict() @@ -321,17 +332,26 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, if global_step % config.save_step == 0: if config.checkpoint: # save model - save_checkpoint(model, optimizer, global_step, epoch, model.decoder.r, OUT_PATH, - optimizer_st=optimizer_st, - model_loss=loss_dict['postnet_loss'], - characters=model_characters, - scaler=scaler.state_dict() if config.mixed_precision else None) + save_checkpoint( + model, + optimizer, + global_step, + epoch, + model.decoder.r, + OUT_PATH, + optimizer_st=optimizer_st, + model_loss=loss_dict["postnet_loss"], + characters=model_characters, + scaler=scaler.state_dict() if config.mixed_precision else None, + ) # Diagnostic visualizations const_spec = postnet_output[0].data.cpu().numpy() - gt_spec = linear_input[0].data.cpu().numpy() if config.model in [ - "Tacotron", "TacotronGST" - ] else mel_input[0].data.cpu().numpy() + gt_spec = ( + linear_input[0].data.cpu().numpy() + if config.model in ["Tacotron", "TacotronGST"] + else mel_input[0].data.cpu().numpy() + ) align_img = alignments[0].data.cpu().numpy() figures = { @@ -341,7 +361,9 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, } if config.bidirectional_decoder or config.double_decoder_consistency: - figures["alignment_backward"] = plot_alignment(alignments_backward[0].data.cpu().numpy(), output_fig=False) + figures["alignment_backward"] = plot_alignment( + alignments_backward[0].data.cpu().numpy(), output_fig=False + ) tb_logger.tb_train_figures(global_step, figures) @@ -350,9 +372,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, train_audio = ap.inv_spectrogram(const_speconfig.T) else: train_audio = ap.inv_melspectrogram(const_speconfig.T) - tb_logger.tb_train_audios(global_step, - {'TrainAudio': train_audio}, - config.audio["sample_rate"]) + tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() # print epoch stats @@ -395,8 +415,16 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # forward pass model if config.bidirectional_decoder or config.double_decoder_consistency: - decoder_output, postnet_output, alignments, stop_tokens, decoder_backward_output, alignments_backward = model( - text_input, text_lengths, mel_input, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings) + ( + decoder_output, + postnet_output, + alignments, + stop_tokens, + decoder_backward_output, + alignments_backward, + ) = model( + text_input, text_lengths, mel_input, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings + ) else: decoder_output, postnet_output, alignments, stop_tokens = model( text_input, text_lengths, mel_input, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings @@ -438,10 +466,10 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # aggregate losses from processes if num_gpus > 1: - loss_dict['postnet_loss'] = reduce_tensor(loss_dict['postnet_loss'].data, num_gpus) - loss_dict['decoder_loss'] = reduce_tensor(loss_dict['decoder_loss'].data, num_gpus) + loss_dict["postnet_loss"] = reduce_tensor(loss_dict["postnet_loss"].data, num_gpus) + loss_dict["decoder_loss"] = reduce_tensor(loss_dict["decoder_loss"].data, num_gpus) if config.stopnet: - loss_dict['stopnet_loss'] = reduce_tensor(loss_dict['stopnet_loss'].data, num_gpus) + loss_dict["stopnet_loss"] = reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) # detach loss values loss_dict_new = dict() @@ -465,9 +493,11 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # Diagnostic visualizations idx = np.random.randint(mel_input.shape[0]) const_spec = postnet_output[idx].data.cpu().numpy() - gt_spec = linear_input[idx].data.cpu().numpy() if config.model in [ - "Tacotron", "TacotronGST" - ] else mel_input[idx].data.cpu().numpy() + gt_spec = ( + linear_input[idx].data.cpu().numpy() + if config.model in ["Tacotron", "TacotronGST"] + else mel_input[idx].data.cpu().numpy() + ) align_img = alignments[idx].data.cpu().numpy() eval_figures = { @@ -481,8 +511,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): eval_audio = ap.inv_spectrogram(const_speconfig.T) else: eval_audio = ap.inv_melspectrogram(const_speconfig.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, - config.audio["sample_rate"]) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) # Plot Validation Stats @@ -510,13 +539,17 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): test_figures = {} print(" | > Synthesizing test sentences") speaker_id = 0 if config.use_speaker_embedding else None - speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping)-1)]]['embedding'] if config.use_external_speaker_embedding_file and config.use_speaker_embedding else None + speaker_embedding = ( + speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]]["embedding"] + if config.use_external_speaker_embedding_file and config.use_speaker_embedding + else None + ) style_wav = config.get("gst_style_input") if style_wav is None and config.use_gst: # inicialize GST with zero dict. style_wav = {} print("WARNING: You don't provided a gst style wav, for this reason we use a zero tensor!") - for i in range(config.gst['gst_num_style_tokens']): + for i in range(config.gst["gst_num_style_tokens"]): style_wav[str(i)] = 0 style_wav = config.get("gst_style_input") for idx, test_sentence in enumerate(test_sentences): @@ -531,7 +564,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): speaker_embedding=speaker_embedding, style_wav=style_wav, truncated=False, - enable_eos_bos_chars=config.enable_eos_bos_chars, #pylint: disable=unused-argument + enable_eos_bos_chars=config.enable_eos_bos_chars, # pylint: disable=unused-argument use_griffin_lim=True, do_trim_silence=False, ) @@ -546,8 +579,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): except: # pylint: disable=bare-except print(" !! Error creating Test Sentence -", idx) traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, - config.audio['sample_rate']) + tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"]) tb_logger.tb_test_figures(global_step, test_figures) return keep_avg.avg_values @@ -564,8 +596,7 @@ def main(args): # pylint: disable=redefined-outer-name # DISTRUBUTED if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, - config.distributed["backend"], config.distributed["url"]) + init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"]) num_chars = len(phonemes) if config.use_phonemes else len(symbols) model_characters = phonemes if config.use_phonemes else symbols @@ -573,10 +604,10 @@ def main(args): # pylint: disable=redefined-outer-name meta_data_train, meta_data_eval = load_meta_data(config.datasets) # set the portion of the data used for training - if config.has('train_portion'): - meta_data_train = meta_data_train[:int(len(meta_data_train) * config.train_portion)] - if config.has('eval_portion'): - meta_data_eval = meta_data_eval[:int(len(meta_data_eval) * config.eval_portion)] + if config.has("train_portion"): + meta_data_train = meta_data_train[: int(len(meta_data_train) * config.train_portion)] + if config.has("eval_portion"): + meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * config.eval_portion)] # parse speakers num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) @@ -590,9 +621,7 @@ def main(args): # pylint: disable=redefined-outer-name params = set_weight_decay(model, config.wd) optimizer = RAdam(params, lr=config.lr, weight_decay=0) if config.stopnet and config.separate_stopnet: - optimizer_st = RAdam(model.decoder.stopnet.parameters(), - lr=config.lr, - weight_decay=0) + optimizer_st = RAdam(model.decoder.stopnet.parameters(), lr=config.lr, weight_decay=0) else: optimizer_st = None @@ -606,7 +635,7 @@ def main(args): # pylint: disable=redefined-outer-name model.load_state_dict(checkpoint["model"]) # optimizer restore print(" > Restoring Optimizer...") - optimizer.load_state_dict(checkpoint['optimizer']) + optimizer.load_state_dict(checkpoint["optimizer"]) if "scaler" in checkpoint and config.mixed_precision: print(" > Restoring AMP Scaler...") scaler.load_state_dict(checkpoint["scaler"]) @@ -622,10 +651,9 @@ def main(args): # pylint: disable=redefined-outer-name del model_dict for group in optimizer.param_groups: - group['lr'] = config.lr - print(" > Model restored from step %d" % checkpoint['step'], - flush=True) - args.restore_step = checkpoint['step'] + group["lr"] = config.lr + print(" > Model restored from step %d" % checkpoint["step"], flush=True) + args.restore_step = checkpoint["step"] else: args.restore_step = 0 @@ -638,9 +666,7 @@ def main(args): # pylint: disable=redefined-outer-name model = apply_gradient_allreduce(model) if config.noam_schedule: - scheduler = NoamLR(optimizer, - warmup_steps=config.warmup_steps, - last_epoch=args.restore_step - 1) + scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1) else: scheduler = None @@ -693,9 +719,9 @@ def main(args): # pylint: disable=redefined-outer-name # eval one epoch eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) - target_loss = train_avg_loss_dict['avg_postnet_loss'] + target_loss = train_avg_loss_dict["avg_postnet_loss"] if config.run_eval: - target_loss = eval_avg_loss_dict['avg_postnet_loss'] + target_loss = eval_avg_loss_dict["avg_postnet_loss"] best_loss = save_best_model( target_loss, best_loss, @@ -708,11 +734,11 @@ def main(args): # pylint: disable=redefined-outer-name model_characters, keep_all_best=keep_all_best, keep_after=keep_after, - scaler=scaler.state_dict() if config.mixed_precision else None + scaler=scaler.state_dict() if config.mixed_precision else None, ) -if __name__ == '__main__': +if __name__ == "__main__": args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 55717c7f..344a8ba6 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -6,8 +6,8 @@ import argparse import glob import json import os -import sys import re +import sys from TTS.tts.utils.text.symbols import parse_symbols from TTS.utils.console_logger import ConsoleLogger @@ -46,24 +46,14 @@ def init_arguments(argv): "Best model file to be used for extracting best loss." "If not specified, the latest best model in continue path is used" ), - default="") - parser.add_argument("--config_path", - type=str, - help="Path to config file for training.", - required="--continue_path" not in argv) - parser.add_argument("--debug", - type=bool, - default=False, - help="Do not verify commit integrity to run training.") + default="", + ) parser.add_argument( - "--rank", - type=int, - default=0, - help="DISTRIBUTED: process rank for distributed training.") - parser.add_argument("--group_id", - type=str, - default="", - help="DISTRIBUTED: process group id.") + "--config_path", type=str, help="Path to config file for training.", required="--continue_path" not in argv + ) + parser.add_argument("--debug", type=bool, default=False, help="Do not verify commit integrity to run training.") + parser.add_argument("--rank", type=int, default=0, help="DISTRIBUTED: process rank for distributed training.") + parser.add_argument("--group_id", type=str, default="", help="DISTRIBUTED: process group id.") return parser @@ -157,8 +147,7 @@ def process_args(args): if config.mixed_precision: print(" > Mixed precision mode is ON") if not os.path.exists(config.output_path): - experiment_path = create_experiment_folder(config.output_path, - config.run_name, args.debug) + experiment_path = create_experiment_folder(config.output_path, config.run_name, args.debug) else: experiment_path = config.output_path audio_path = os.path.join(experiment_path, "test_audios") @@ -172,17 +161,15 @@ def process_args(args): # if model characters are not set in the config file # save the default set to the config file for future # compatibility. - if config.has('characters_config'): + if config.has("characters_config"): used_characters = parse_symbols() - new_fields['characters'] = used_characters + new_fields["characters"] = used_characters copy_model_files(config, args.config_path, experiment_path, new_fields) os.chmod(audio_path, 0o775) os.chmod(experiment_path, 0o775) - tb_logger = TensorboardLogger(experiment_path, - model_name=config.model) + tb_logger = TensorboardLogger(experiment_path, model_name=config.model) # write model desc to tensorboard - tb_logger.tb_add_text("model-description", config["run_description"], - 0) + tb_logger.tb_add_text("model-description", config["run_description"], 0) c_logger = ConsoleLogger() return config, experiment_path, audio_path, c_logger, tb_logger diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index 87307032..274fb634 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -73,14 +73,14 @@ def count_parameters(model): def to_camel(text): text = text.capitalize() - text = re.sub(r'(?!^)_([a-zA-Z])', lambda m: m.group(1).upper(), text) - text = text.replace('Tts', 'TTS') + text = re.sub(r"(?!^)_([a-zA-Z])", lambda m: m.group(1).upper(), text) + text = text.replace("Tts", "TTS") return text def find_module(module_path: str, module_name: str) -> object: module_name = module_name.lower() - module = importlib.import_module(module_path+'.'+module_name) + module = importlib.import_module(module_path + "." + module_name) class_name = to_camel(module_name) return getattr(module, class_name) @@ -156,4 +156,3 @@ class KeepAverage: def update_values(self, value_dict): for key, value in value_dict.items(): self.update_value(key, value) - diff --git a/TTS/utils/io.py b/TTS/utils/io.py index 58e6dd69..0a352d0d 100644 --- a/TTS/utils/io.py +++ b/TTS/utils/io.py @@ -5,6 +5,7 @@ import re from shutil import copyfile import yaml + from TTS.utils.generic_utils import find_module from .generic_utils import find_module @@ -32,8 +33,8 @@ def read_json_with_comments(json_path): with open(json_path, "r", encoding="utf-8") as f: input_str = f.read() # handle comments - input_str = re.sub(r'\\\n', '', input_str) - input_str = re.sub(r'//.*\n', '\n', input_str) + input_str = re.sub(r"\\\n", "", input_str) + input_str = re.sub(r"//.*\n", "\n", input_str) data = json.loads(input_str) return data @@ -44,20 +45,19 @@ def load_config(config_path: str) -> None: if ext in (".yml", ".yaml"): with open(config_path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) - elif ext == '.json': + elif ext == ".json": with open(config_path, "r", encoding="utf-8") as f: input_str = f.read() data = json.loads(input_str) else: - raise TypeError(f' [!] Unknown config file type {ext}') + raise TypeError(f" [!] Unknown config file type {ext}") config_dict.update(data) - config_class = find_module('TTS.tts.configs', config_dict['model'].lower()+'_config') + config_class = find_module("TTS.tts.configs", config_dict["model"].lower() + "_config") config = config_class() config.from_dict(config_dict) return config - def copy_model_files(c, config_file, out_path, new_fields): """Copy config.json and other model files to training folder and add new fields. From 93a00373f6d5d15eb2d7f616cfed56725d9f8421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 3 May 2021 16:43:54 +0200 Subject: [PATCH 22/87] move split_dataset --- TTS/tts/datasets/preprocess.py | 28 +++++++++++++++++++++++++--- TTS/tts/utils/generic_utils.py | 26 +------------------------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/TTS/tts/datasets/preprocess.py b/TTS/tts/datasets/preprocess.py index d6040493..4523d70b 100644 --- a/TTS/tts/datasets/preprocess.py +++ b/TTS/tts/datasets/preprocess.py @@ -2,19 +2,41 @@ import os import re import sys import xml.etree.ElementTree as ET +from collections import Counter from glob import glob from pathlib import Path from typing import List +import numpy as np from tqdm import tqdm -from TTS.tts.utils.generic_utils import split_dataset - #################### # UTILITIES #################### +def split_dataset(items): + speakers = [item[-1] for item in items] + is_multi_speaker = len(set(speakers)) > 1 + eval_split_size = min(500, int(len(items) * 0.01)) + assert eval_split_size > 0, " [!] You do not have enough samples to train. You need at least 100 samples." + np.random.seed(0) + np.random.shuffle(items) + if is_multi_speaker: + items_eval = [] + speakers = [item[-1] for item in items] + speaker_counter = Counter(speakers) + while len(items_eval) < eval_split_size: + item_idx = np.random.randint(0, len(items)) + speaker_to_be_removed = items[item_idx][-1] + if speaker_counter[speaker_to_be_removed] > 1: + items_eval.append(items[item_idx]) + speaker_counter[speaker_to_be_removed] -= 1 + del items[item_idx] + return items_eval, items + return items[:eval_split_size], items[eval_split_size:] + + def load_meta_data(datasets, eval_split=True): meta_data_train_all = [] meta_data_eval_all = [] if eval_split else None @@ -38,7 +60,7 @@ def load_meta_data(datasets, eval_split=True): meta_data_train_all += meta_data_train # load attention masks for duration predictor training if dataset.meta_file_attn_mask is not None: - meta_data = dict(load_attention_mask_meta_data(dataset['meta_file_attn_mask'])) + meta_data = dict(load_attention_mask_meta_data(dataset["meta_file_attn_mask"])) for idx, ins in enumerate(meta_data_train_all): attn_file = meta_data[ins[1]].strip() meta_data_train_all[idx].append(attn_file) diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index 9711c868..9f17da0b 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -1,32 +1,8 @@ -import importlib -import re -from collections import Counter +import torch from TTS.utils.generic_utils import find_module -def split_dataset(items): - speakers = [item[-1] for item in items] - is_multi_speaker = len(set(speakers)) > 1 - eval_split_size = min(500, int(len(items) * 0.01)) - assert eval_split_size > 0, " [!] You do not have enough samples to train. You need at least 100 samples." - np.random.seed(0) - np.random.shuffle(items) - if is_multi_speaker: - items_eval = [] - speakers = [item[-1] for item in items] - speaker_counter = Counter(speakers) - while len(items_eval) < eval_split_size: - item_idx = np.random.randint(0, len(items)) - speaker_to_be_removed = items[item_idx][-1] - if speaker_counter[speaker_to_be_removed] > 1: - items_eval.append(items[item_idx]) - speaker_counter[speaker_to_be_removed] -= 1 - del items[item_idx] - return items_eval, items - return items[:eval_split_size], items[eval_split_size:] - - # from https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1 def sequence_mask(sequence_length, max_len=None): if max_len is None: From 05d9543ed8732a7aa9e1bfb21aaa39ad118108e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 3 May 2021 16:44:34 +0200 Subject: [PATCH 23/87] init GST module using gst config in Tacotron models --- TTS/tts/models/tacotron.py | 36 +++++++------------------ TTS/tts/models/tacotron2.py | 41 ++++++++--------------------- TTS/tts/models/tacotron_abstract.py | 12 ++------- TTS/tts/utils/generic_utils.py | 16 +++-------- 4 files changed, 26 insertions(+), 79 deletions(-) diff --git a/TTS/tts/models/tacotron.py b/TTS/tts/models/tacotron.py index 85d90116..1ffe9786 100644 --- a/TTS/tts/models/tacotron.py +++ b/TTS/tts/models/tacotron.py @@ -41,11 +41,7 @@ class Tacotron(TacotronAbstract): encoder_in_features (int, optional): input channels for the encoder. Defaults to 512. decoder_in_features (int, optional): input channels for the decoder. Defaults to 512. speaker_embedding_dim (int, optional): external speaker conditioning vector channels. Defaults to None. - gst (bool, optional): enable/disable global style token learning. Defaults to False. - gst_embedding_dim (int, optional): size of channels for GST vectors. Defaults to 512. - gst_num_heads (int, optional): number of attention heads for GST. Defaults to 4. - gst_num_style_tokens (int, optional): number of GST tokens. Defaults to 10. - gst_use_speaker_embedding (bool, optional): enable/disable inputing speaker embedding to GST. Defaults to False. + gst (Coqpit, optional): Coqpit to initialize the GST module. If `None`, GST is disabled. Defaults to None. memory_size (int, optional): size of the history queue fed to the prenet. Model feeds the last ```memory_size``` output frames to the prenet. """ @@ -75,12 +71,8 @@ class Tacotron(TacotronAbstract): encoder_in_features=256, decoder_in_features=256, speaker_embedding_dim=None, - gst=False, - gst_embedding_dim=256, - gst_num_heads=4, - gst_style_tokens=10, + gst=None, memory_size=5, - gst_use_speaker_embedding=False, ): super().__init__( num_chars, @@ -107,10 +99,6 @@ class Tacotron(TacotronAbstract): decoder_in_features, speaker_embedding_dim, gst, - gst_embedding_dim, - gst_num_heads, - gst_style_tokens, - gst_use_speaker_embedding, ) # speaker embedding layers @@ -156,13 +144,11 @@ class Tacotron(TacotronAbstract): # global style token layers if self.gst: self.gst_layer = GST( - num_mel=80, - num_heads=gst_num_heads, - num_style_tokens=gst_style_tokens, - gst_embedding_dim=self.gst_embedding_dim, - speaker_embedding_dim=speaker_embedding_dim - if self.embeddings_per_sample and self.gst_use_speaker_embedding - else None, + num_mel=decoder_output_dim, + speaker_embedding_dim=speaker_embedding_dim, + num_heads=gst.gst_num_heads, + num_style_tokens=gst.gst_num_style_tokens, + gst_embedding_dim=gst.gst_embedding_dim, ) # backward pass decoder if self.bidirectional_decoder: @@ -207,9 +193,7 @@ class Tacotron(TacotronAbstract): # global style token if self.gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, mel_specs, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, mel_specs, speaker_embeddings) # speaker embedding if self.num_speakers > 1: if not self.embeddings_per_sample: @@ -265,9 +249,7 @@ class Tacotron(TacotronAbstract): encoder_outputs = self.encoder(inputs) if self.gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings) if self.num_speakers > 1: if not self.embeddings_per_sample: # B x 1 x speaker_embed_dim diff --git a/TTS/tts/models/tacotron2.py b/TTS/tts/models/tacotron2.py index 44c81735..1945a6f7 100644 --- a/TTS/tts/models/tacotron2.py +++ b/TTS/tts/models/tacotron2.py @@ -41,11 +41,7 @@ class Tacotron2(TacotronAbstract): encoder_in_features (int, optional): input channels for the encoder. Defaults to 512. decoder_in_features (int, optional): input channels for the decoder. Defaults to 512. speaker_embedding_dim (int, optional): external speaker conditioning vector channels. Defaults to None. - gst (bool, optional): enable/disable global style token learning. Defaults to False. - gst_embedding_dim (int, optional): size of channels for GST vectors. Defaults to 512. - gst_num_heads (int, optional): number of attention heads for GST. Defaults to 4. - gst_num_style_tokens (int, optional): number of GST tokens. Defaults to 10. - gst_use_speaker_embedding (bool, optional): enable/disable inputing speaker embedding to GST. Defaults to False. + gst (Coqpit, optional): Coqpit to initialize the GST module. If `None`, GST is disabled. Defaults to None. """ def __init__( @@ -73,11 +69,7 @@ class Tacotron2(TacotronAbstract): encoder_in_features=512, decoder_in_features=512, speaker_embedding_dim=None, - gst=False, - gst_embedding_dim=512, - gst_num_heads=4, - gst_style_tokens=10, - gst_use_speaker_embedding=False, + gst=None, ): super().__init__( num_chars, @@ -104,10 +96,6 @@ class Tacotron2(TacotronAbstract): decoder_in_features, speaker_embedding_dim, gst, - gst_embedding_dim, - gst_num_heads, - gst_style_tokens, - gst_use_speaker_embedding, ) # speaker embedding layer @@ -150,14 +138,13 @@ class Tacotron2(TacotronAbstract): # global style token layers if self.gst: self.gst_layer = GST( - num_mel=80, - num_heads=self.gst_num_heads, - num_style_tokens=self.gst_style_tokens, - gst_embedding_dim=self.gst_embedding_dim, - speaker_embedding_dim=speaker_embedding_dim - if self.embeddings_per_sample and self.gst_use_speaker_embedding - else None, + num_mel=decoder_output_dim, + speaker_embedding_dim=speaker_embedding_dim, + num_heads=gst.gst_num_heads, + num_style_tokens=gst.gst_num_style_tokens, + gst_embedding_dim=gst.gst_embedding_dim, ) + # backward pass decoder if self.bidirectional_decoder: self._init_backward_decoder() @@ -205,9 +192,7 @@ class Tacotron2(TacotronAbstract): encoder_outputs = self.encoder(embedded_inputs, text_lengths) if self.gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, mel_specs, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, mel_specs, speaker_embeddings) if self.num_speakers > 1: if not self.embeddings_per_sample: # B x 1 x speaker_embed_dim @@ -263,9 +248,7 @@ class Tacotron2(TacotronAbstract): if self.gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings) if self.num_speakers > 1: if not self.embeddings_per_sample: speaker_embeddings = self.speaker_embedding(speaker_ids)[:, None] @@ -286,9 +269,7 @@ class Tacotron2(TacotronAbstract): if self.gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings) if self.num_speakers > 1: if not self.embeddings_per_sample: diff --git a/TTS/tts/models/tacotron_abstract.py b/TTS/tts/models/tacotron_abstract.py index c6bdb19e..42411656 100644 --- a/TTS/tts/models/tacotron_abstract.py +++ b/TTS/tts/models/tacotron_abstract.py @@ -33,11 +33,7 @@ class TacotronAbstract(ABC, nn.Module): encoder_in_features=512, decoder_in_features=512, speaker_embedding_dim=None, - gst=False, - gst_embedding_dim=512, - gst_num_heads=4, - gst_style_tokens=10, - gst_use_speaker_embedding=False, + gst=None, ): """Abstract Tacotron class""" super().__init__() @@ -46,10 +42,6 @@ class TacotronAbstract(ABC, nn.Module): self.decoder_output_dim = decoder_output_dim self.postnet_output_dim = postnet_output_dim self.gst = gst - self.gst_embedding_dim = gst_embedding_dim - self.gst_num_heads = gst_num_heads - self.gst_num_style_tokens = gst_num_style_tokens - self.gst_use_speaker_embedding = gst_use_speaker_embedding self.num_speakers = num_speakers self.bidirectional_decoder = bidirectional_decoder self.double_decoder_consistency = double_decoder_consistency @@ -86,7 +78,7 @@ class TacotronAbstract(ABC, nn.Module): # global style token if self.gst: - self.decoder_in_features += gst_embedding_dim # add gst embedding dim + self.decoder_in_features += self.gst.gst_embedding_dim # add gst embedding dim self.gst_layer = None # model states diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index 9f17da0b..e6934bc9 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -14,7 +14,7 @@ def sequence_mask(sequence_length, max_len=None): def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): print(" > Using model: {}".format(c.model)) - MyModel = find_module("TTS.tts.models", c.model.lower()) + MyModel = find_module("TTS.tts.models", c.model.lower()) if c.model.lower() in "tacotron": model = MyModel( num_chars=num_chars + getattr(c, "add_blank", False), @@ -23,17 +23,13 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): postnet_output_dim=int(c.audio["fft_size"] / 2 + 1), decoder_output_dim=c.audio["num_mels"], gst=c.use_gst, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"], memory_size=c.memory_size, attn_type=c.attention_type, attn_win=c.windowing, attn_norm=c.attention_norm, prenet_type=c.prenet_type, prenet_dropout=c.prenet_dropout, - prenet_dropout_at_inference=c.prenet_dropout_at_inference if "prenet_dropout_at_inference" in c else False, + prenet_dropout_at_inference=c.prenet_dropout_at_inference, forward_attn=c.use_forward_attn, trans_agent=c.transition_agent, forward_attn_mask=c.forward_attn_mask, @@ -52,17 +48,13 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): r=c.r, postnet_output_dim=c.audio["num_mels"], decoder_output_dim=c.audio["num_mels"], - gst=c.gst is not None, - gst_embedding_dim=None if c.gst is None else c.gst['gst_embedding_dim'], - gst_num_heads=None if c.gst is None else c.gst['gst_num_heads'], - gst_num_style_tokens=None if c.gst is None else c.gst['gst_num_style_tokens'], - gst_use_speaker_embedding=None if c.gst is None else c.gst['gst_use_speaker_embedding'], + gst=c.gst, attn_type=c.attention_type, attn_win=c.windowing, attn_norm=c.attention_norm, prenet_type=c.prenet_type, prenet_dropout=c.prenet_dropout, - prenet_dropout_at_inference=c.prenet_dropout_at_inference if "prenet_dropout_at_inference" in c else False, + prenet_dropout_at_inference=c.prenet_dropout_at_inference, forward_attn=c.use_forward_attn, trans_agent=c.transition_agent, forward_attn_mask=c.forward_attn_mask, From 4a58fdfd592b5c7ed4966a0e77aa8bf0aa51ed2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 3 May 2021 16:48:32 +0200 Subject: [PATCH 24/87] comment out check-arguments before copying fields to the configs --- TTS/speaker_encoder/utils/generic_utils.py | 124 ++++++++++----------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/TTS/speaker_encoder/utils/generic_utils.py b/TTS/speaker_encoder/utils/generic_utils.py index c9bfa679..69ff25b7 100644 --- a/TTS/speaker_encoder/utils/generic_utils.py +++ b/TTS/speaker_encoder/utils/generic_utils.py @@ -5,7 +5,6 @@ import re import torch from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.utils.generic_utils import check_argument def to_camel(text): @@ -54,70 +53,71 @@ def save_best_model(model, optimizer, model_loss, best_loss, out_path, current_s def check_config_speaker_encoder(c): - """Check the config.json file of the speaker encoder""" - check_argument("run_name", c, restricted=True, val_type=str) - check_argument("run_description", c, val_type=str) + ... + # """Check the config.json file of the speaker encoder""" + # check_argument("run_name", c, restricted=True, val_type=str) + # check_argument("run_description", c, val_type=str) - # audio processing parameters - check_argument("audio", c, restricted=True, val_type=dict) - check_argument("num_mels", c["audio"], restricted=True, val_type=int, min_val=10, max_val=2056) - check_argument("fft_size", c["audio"], restricted=True, val_type=int, min_val=128, max_val=4058) - check_argument("sample_rate", c["audio"], restricted=True, val_type=int, min_val=512, max_val=100000) - check_argument( - "frame_length_ms", - c["audio"], - restricted=True, - val_type=float, - min_val=10, - max_val=1000, - alternative="win_length", - ) - check_argument( - "frame_shift_ms", c["audio"], restricted=True, val_type=float, min_val=1, max_val=1000, alternative="hop_length" - ) - check_argument("preemphasis", c["audio"], restricted=True, val_type=float, min_val=0, max_val=1) - check_argument("min_level_db", c["audio"], restricted=True, val_type=int, min_val=-1000, max_val=10) - check_argument("ref_level_db", c["audio"], restricted=True, val_type=int, min_val=0, max_val=1000) - check_argument("power", c["audio"], restricted=True, val_type=float, min_val=1, max_val=5) - check_argument("griffin_lim_iters", c["audio"], restricted=True, val_type=int, min_val=10, max_val=1000) + # # audio processing parameters + # check_argument("audio", c, restricted=True, val_type=dict) + # check_argument("num_mels", c["audio"], restricted=True, val_type=int, min_val=10, max_val=2056) + # check_argument("fft_size", c["audio"], restricted=True, val_type=int, min_val=128, max_val=4058) + # check_argument("sample_rate", c["audio"], restricted=True, val_type=int, min_val=512, max_val=100000) + # check_argument( + # "frame_length_ms", + # c["audio"], + # restricted=True, + # val_type=float, + # min_val=10, + # max_val=1000, + # alternative="win_length", + # ) + # check_argument( + # "frame_shift_ms", c["audio"], restricted=True, val_type=float, min_val=1, max_val=1000, alternative="hop_length" + # ) + # check_argument("preemphasis", c["audio"], restricted=True, val_type=float, min_val=0, max_val=1) + # check_argument("min_level_db", c["audio"], restricted=True, val_type=int, min_val=-1000, max_val=10) + # check_argument("ref_level_db", c["audio"], restricted=True, val_type=int, min_val=0, max_val=1000) + # check_argument("power", c["audio"], restricted=True, val_type=float, min_val=1, max_val=5) + # check_argument("griffin_lim_iters", c["audio"], restricted=True, val_type=int, min_val=10, max_val=1000) - # training parameters - check_argument("loss", c, enum_list=["ge2e", "angleproto"], restricted=True, val_type=str) - check_argument("grad_clip", c, restricted=True, val_type=float) - check_argument("epochs", c, restricted=True, val_type=int, min_val=1) - check_argument("lr", c, restricted=True, val_type=float, min_val=0) - check_argument("lr_decay", c, restricted=True, val_type=bool) - check_argument("warmup_steps", c, restricted=True, val_type=int, min_val=0) - check_argument("tb_model_param_stats", c, restricted=True, val_type=bool) - check_argument("num_speakers_in_batch", c, restricted=True, val_type=int) - check_argument("num_loader_workers", c, restricted=True, val_type=int) - check_argument("wd", c, restricted=True, val_type=float, min_val=0.0, max_val=1.0) + # # training parameters + # check_argument("loss", c, enum_list=["ge2e", "angleproto"], restricted=True, val_type=str) + # check_argument("grad_clip", c, restricted=True, val_type=float) + # check_argument("epochs", c, restricted=True, val_type=int, min_val=1) + # check_argument("lr", c, restricted=True, val_type=float, min_val=0) + # check_argument("lr_decay", c, restricted=True, val_type=bool) + # check_argument("warmup_steps", c, restricted=True, val_type=int, min_val=0) + # check_argument("tb_model_param_stats", c, restricted=True, val_type=bool) + # check_argument("num_speakers_in_batch", c, restricted=True, val_type=int) + # check_argument("num_loader_workers", c, restricted=True, val_type=int) + # check_argument("wd", c, restricted=True, val_type=float, min_val=0.0, max_val=1.0) - # checkpoint and output parameters - check_argument("steps_plot_stats", c, restricted=True, val_type=int) - check_argument("checkpoint", c, restricted=True, val_type=bool) - check_argument("save_step", c, restricted=True, val_type=int) - check_argument("print_step", c, restricted=True, val_type=int) - check_argument("output_path", c, restricted=True, val_type=str) + # # checkpoint and output parameters + # check_argument("steps_plot_stats", c, restricted=True, val_type=int) + # check_argument("checkpoint", c, restricted=True, val_type=bool) + # check_argument("save_step", c, restricted=True, val_type=int) + # check_argument("print_step", c, restricted=True, val_type=int) + # check_argument("output_path", c, restricted=True, val_type=str) - # model parameters - check_argument("model", c, restricted=True, val_type=dict) - check_argument("input_dim", c["model"], restricted=True, val_type=int) - check_argument("proj_dim", c["model"], restricted=True, val_type=int) - check_argument("lstm_dim", c["model"], restricted=True, val_type=int) - check_argument("num_lstm_layers", c["model"], restricted=True, val_type=int) - check_argument("use_lstm_with_projection", c["model"], restricted=True, val_type=bool) + # # model parameters + # check_argument("model", c, restricted=True, val_type=dict) + # check_argument("input_dim", c["model"], restricted=True, val_type=int) + # check_argument("proj_dim", c["model"], restricted=True, val_type=int) + # check_argument("lstm_dim", c["model"], restricted=True, val_type=int) + # check_argument("num_lstm_layers", c["model"], restricted=True, val_type=int) + # check_argument("use_lstm_with_projection", c["model"], restricted=True, val_type=bool) - # in-memory storage parameters - check_argument("storage", c, restricted=True, val_type=dict) - check_argument("sample_from_storage_p", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0) - check_argument("storage_size", c["storage"], restricted=True, val_type=int, min_val=1, max_val=100) - check_argument("additive_noise", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0) + # # in-memory storage parameters + # check_argument("storage", c, restricted=True, val_type=dict) + # check_argument("sample_from_storage_p", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0) + # check_argument("storage_size", c["storage"], restricted=True, val_type=int, min_val=1, max_val=100) + # check_argument("additive_noise", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0) - # datasets - checking only the first entry - check_argument("datasets", c, restricted=True, val_type=list) - for dataset_entry in c["datasets"]: - check_argument("name", dataset_entry, restricted=True, val_type=str) - check_argument("path", dataset_entry, restricted=True, val_type=str) - check_argument("meta_file_train", dataset_entry, restricted=True, val_type=[str, list]) - check_argument("meta_file_val", dataset_entry, restricted=True, val_type=str) + # # datasets - checking only the first entry + # check_argument("datasets", c, restricted=True, val_type=list) + # for dataset_entry in c["datasets"]: + # check_argument("name", dataset_entry, restricted=True, val_type=str) + # check_argument("path", dataset_entry, restricted=True, val_type=str) + # check_argument("meta_file_train", dataset_entry, restricted=True, val_type=[str, list]) + # check_argument("meta_file_val", dataset_entry, restricted=True, val_type=str) From 65d7ad4250ee65434b8a06c676e592f626588170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 5 May 2021 02:29:29 +0200 Subject: [PATCH 25/87] refactor train_speedy_speech.py for coqpit --- TTS/bin/train_speedy_speech.py | 149 ++++++++++++++++----------------- 1 file changed, 74 insertions(+), 75 deletions(-) diff --git a/TTS/bin/train_speedy_speech.py b/TTS/bin/train_speedy_speech.py index 3adbe513..b9bdc6d1 100644 --- a/TTS/bin/train_speedy_speech.py +++ b/TTS/bin/train_speedy_speech.py @@ -25,7 +25,7 @@ from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.synthesis import synthesis from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed, reduce_tensor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -36,45 +36,45 @@ use_cuda, num_gpus = setup_torch_training_env(True, False) def setup_loader(ap, r, is_val=False, verbose=False): - if is_val and not c.run_eval: + if is_val and not config.run_eval: loader = None else: dataset = MyDataset( r, - c.text_cleaner, + config.text_cleaner, compute_linear_spec=False, meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, - tp=c.characters if "characters" in c.keys() else None, - add_blank=c["add_blank"] if "add_blank" in c.keys() else False, - batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, - min_seq_len=c.min_seq_len, - max_seq_len=c.max_seq_len, - phoneme_cache_path=c.phoneme_cache_path, - use_phonemes=c.use_phonemes, - phoneme_language=c.phoneme_language, - enable_eos_bos=c.enable_eos_bos_chars, + tp=config.characters, + add_blank=config["add_blank"], + batch_group_size=0 if is_val else config.batch_group_size * config.batch_size, + min_seq_len=config.min_seq_len, + max_seq_len=config.max_seq_len, + phoneme_cache_path=config.phoneme_cache_path, + use_phonemes=config.use_phonemes, + phoneme_language=config.phoneme_language, + enable_eos_bos=config.enable_eos_bos_chars, use_noise_augment=not is_val, verbose=verbose, speaker_mapping=speaker_mapping - if c.use_speaker_embedding and c.use_external_speaker_embedding_file + if config.use_speaker_embedding and config.use_external_speaker_embedding_file else None, ) - if c.use_phonemes and c.compute_input_seq_cache: + if config.use_phonemes and config.compute_input_seq_cache: # precompute phonemes to have a better estimate of sequence lengths. - dataset.compute_input_seq(c.num_loader_workers) + dataset.compute_input_seq(config.num_loader_workers) dataset.sort_items() sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader( dataset, - batch_size=c.eval_batch_size if is_val else c.batch_size, + batch_size=config.eval_batch_size if is_val else config.batch_size, shuffle=False, collate_fn=dataset.collate_fn, drop_last=False, sampler=sampler, - num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers, + num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers, pin_memory=False, ) return loader @@ -92,8 +92,8 @@ def format_data(data): avg_text_length = torch.mean(text_lengths.float()) avg_spec_length = torch.mean(mel_lengths.float()) - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: # return precomputed embedding vector speaker_c = data[8] else: @@ -150,12 +150,12 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch_time = 0 keep_avg = KeepAverage() if use_cuda: - batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus)) + batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus)) else: - batch_n_iter = int(len(data_loader.dataset) / c.batch_size) + batch_n_iter = int(len(data_loader.dataset) / config.batch_size) end_time = time.time() c_logger.print_train_start() - scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None + scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None for num_iter, data in enumerate(data_loader): start_time = time.time() @@ -179,7 +179,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, optimizer.zero_grad() # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): decoder_output, dur_output, alignments = model.forward( text_input, text_lengths, mel_lengths, dur_target, g=speaker_c ) @@ -190,19 +190,19 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, ) # backward pass with loss scaling - if c.mixed_precision: + if config.mixed_precision: scaler.scale(loss_dict["loss"]).backward() scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) scaler.step(optimizer) scaler.update() else: loss_dict["loss"].backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) optimizer.step() # setup lr - if c.noam_schedule: + if config.noam_schedule: scheduler.step() # current_lr @@ -240,7 +240,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, keep_avg.update_values(update_train_values) # print training progress - if global_step % c.print_step == 0: + if global_step % config.print_step == 0: log_dict = { "avg_spec_length": [avg_spec_length, 1], # value, precision "avg_text_length": [avg_text_length, 1], @@ -253,13 +253,13 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, if args.rank == 0: # Plot Training Iter Stats # reduce TB load - if global_step % c.tb_plot_step == 0: + if global_step % config.tb_plot_step == 0: iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time} iter_stats.update(loss_dict) tb_logger.tb_train_iter_stats(global_step, iter_stats) - if global_step % c.save_step == 0: - if c.checkpoint: + if global_step % config.save_step == 0: + if config.checkpoint: # save model save_checkpoint( model, @@ -291,7 +291,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, # Sample audio train_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"]) + tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() # print epoch stats @@ -302,7 +302,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch_stats = {"epoch_time": epoch_time} epoch_stats.update(keep_avg.avg_values) tb_logger.tb_train_epoch_stats(global_step, epoch_stats) - if c.tb_model_param_stats: + if config.tb_model_param_stats: tb_logger.tb_model_weights(model, global_step) return keep_avg.avg_values, global_step @@ -321,7 +321,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _, dur_target, _ = format_data(data) # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): decoder_output, dur_output, alignments = model.forward( text_input, text_lengths, mel_lengths, dur_target, g=speaker_c ) @@ -361,7 +361,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): update_train_values["avg_" + key] = value keep_avg.update_values(update_train_values) - if c.print_eval: + if config.print_eval: c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) if args.rank == 0: @@ -379,14 +379,17 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # Sample audio eval_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"]) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) # Plot Validation Stats tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) tb_logger.tb_eval_figures(global_step, eval_figures) - if args.rank == 0 and epoch >= c.test_delay_epochs: - if c.test_sentences_file is None: + if args.rank == 0 and epoch >= config.test_delay_epochs: + if config.test_sentences_file: + with open(config.test_sentences_file, "r") as f: + test_sentences = [s.strip() for s in f.readlines()] + else: test_sentences = [ "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", "Be a voice, not an echo.", @@ -394,16 +397,14 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): "This cake is great. It's so delicious and moist.", "Prior to November 22, 1963.", ] - else: - with open(c.test_sentences_file, "r") as f: - test_sentences = [s.strip() for s in f.readlines()] + # test sentences test_audios = {} test_figures = {} print(" | > Synthesizing test sentences") - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][ "embedding" ] @@ -415,20 +416,19 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): speaker_id = None speaker_embedding = None - style_wav = c.get("style_wav_for_test") for idx, test_sentence in enumerate(test_sentences): try: wav, alignment, _, postnet_output, _, _ = synthesis( model, test_sentence, - c, + config, use_cuda, ap, speaker_id=speaker_id, speaker_embedding=speaker_embedding, - style_wav=style_wav, + style_wav=None, truncated=False, - enable_eos_bos_chars=c.enable_eos_bos_chars, # pylint: disable=unused-argument + enable_eos_bos_chars=config.enable_eos_bos_chars, # pylint: disable=unused-argument use_griffin_lim=True, do_trim_silence=False, ) @@ -443,7 +443,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): except: # pylint: disable=bare-except print(" !! Error creating Test Sentence -", idx) traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"]) + tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"]) tb_logger.tb_test_figures(global_step, test_figures) return keep_avg.avg_values @@ -453,34 +453,34 @@ def main(args): # pylint: disable=redefined-outer-name # pylint: disable=global-variable-undefined global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping # Audio processor - ap = AudioProcessor(**c.audio) - if "characters" in c.keys(): - symbols, phonemes = make_symbols(**c.characters) + ap = AudioProcessor(**config.audio.to_dict()) + if config.characters is not None: + symbols, phonemes = make_symbols(**config.characters.to_dict()) # DISTRUBUTED if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"]) + init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"]) # set model characters - model_characters = phonemes if c.use_phonemes else symbols + model_characters = phonemes if config.use_phonemes else symbols num_chars = len(model_characters) # load data instances - meta_data_train, meta_data_eval = load_meta_data(c.datasets, eval_split=True) + meta_data_train, meta_data_eval = load_meta_data(config.datasets, eval_split=True) # set the portion of the data used for training if set in config.json - if "train_portion" in c.keys(): - meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)] - if "eval_portion" in c.keys(): - meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)] + if config.has("train_portion"): + meta_data_train = meta_data_train[: int(len(meta_data_train) * config.train_portion)] + if config.has("eval_portion"): + meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * config.eval_portion)] # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH) + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) # setup model - model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim) - optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) - criterion = SpeedySpeechLoss(c) + model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim) + optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) + criterion = SpeedySpeechLoss(config) if args.restore_path: print(f" > Restoring from {os.path.basename(args.restore_path)} ...") @@ -489,18 +489,18 @@ def main(args): # pylint: disable=redefined-outer-name # TODO: fix optimizer init, model.cuda() needs to be called before # optimizer restore optimizer.load_state_dict(checkpoint["optimizer"]) - if c.reinit_layers: + if config.reinit_layers: raise RuntimeError model.load_state_dict(checkpoint["model"]) except: # pylint: disable=bare-except print(" > Partial model initialization.") model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint["model"], c) + model_dict = set_init_dict(model_dict, checkpoint["model"], config) model.load_state_dict(model_dict) del model_dict for group in optimizer.param_groups: - group["initial_lr"] = c.lr + group["initial_lr"] = config.lr print(" > Model restored from step %d" % checkpoint["step"], flush=True) args.restore_step = checkpoint["step"] else: @@ -514,8 +514,8 @@ def main(args): # pylint: disable=redefined-outer-name if num_gpus > 1: model = DDP_th(model, device_ids=[args.rank]) - if c.noam_schedule: - scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1) + if config.noam_schedule: + scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1) else: scheduler = None @@ -529,23 +529,23 @@ def main(args): # pylint: disable=redefined-outer-name print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] print(f" > Starting with loaded last best loss {best_loss}.") - keep_all_best = c.get("keep_all_best", False) - keep_after = c.get("keep_after", 10000) # void if keep_all_best False + keep_all_best = config.keep_all_best + keep_after = config.keep_after # void if keep_all_best False # define dataloaders train_loader = setup_loader(ap, 1, is_val=False, verbose=True) eval_loader = setup_loader(ap, 1, is_val=True, verbose=True) global_step = args.restore_step - for epoch in range(0, c.epochs): - c_logger.print_epoch_start(epoch, c.epochs) + for epoch in range(0, config.epochs): + c_logger.print_epoch_start(epoch, config.epochs) train_avg_loss_dict, global_step = train( train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch ) eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) target_loss = train_avg_loss_dict["avg_loss"] - if c.run_eval: + if config.run_eval: target_loss = eval_avg_loss_dict["avg_loss"] best_loss = save_best_model( target_loss, @@ -554,7 +554,7 @@ def main(args): # pylint: disable=redefined-outer-name optimizer, global_step, epoch, - c.r, + config.r, OUT_PATH, model_characters, keep_all_best=keep_all_best, @@ -563,8 +563,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts") + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) From eaa130e813cf2f5c05e5664e331cbe4f723c64b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 5 May 2021 02:30:25 +0200 Subject: [PATCH 26/87] fix tacotron for coqpit --- TTS/bin/train_tacotron.py | 33 ++++++++++++---------------- TTS/tts/utils/generic_utils.py | 2 +- TTS/tts/utils/synthesis.py | 39 ++++++++++++++++++++-------------- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index f5d74099..edf89858 100755 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -90,7 +90,7 @@ def format_data(data): text_input = data[0] text_lengths = data[1] speaker_names = data[2] - linear_input = data[3] if config.model in ["Tacotron"] else None + linear_input = data[3] if config.model.lower() in ["tacotron"] else None mel_input = data[4] mel_lengths = data[5] stop_targets = data[6] @@ -369,9 +369,9 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, # Sample audio if config.model in ["Tacotron", "TacotronGST"]: - train_audio = ap.inv_spectrogram(const_speconfig.T) + train_audio = ap.inv_spectrogram(const_spec.T) else: - train_audio = ap.inv_melspectrogram(const_speconfig.T) + train_audio = ap.inv_melspectrogram(const_spec.T) tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() @@ -507,10 +507,10 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): } # Sample audio - if config.model in ["Tacotron", "TacotronGST"]: - eval_audio = ap.inv_spectrogram(const_speconfig.T) + if config.model.lower() in ["tacotron"]: + eval_audio = ap.inv_spectrogram(const_spec.T) else: - eval_audio = ap.inv_melspectrogram(const_speconfig.T) + eval_audio = ap.inv_melspectrogram(const_spec.T) tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) # Plot Validation Stats @@ -522,7 +522,10 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): tb_logger.tb_eval_figures(global_step, eval_figures) if args.rank == 0 and epoch > config.test_delay_epochs: - if config.test_sentences_file is None: + if config.test_sentences_file: + with open(config.test_sentences_file, "r") as f: + test_sentences = [s.strip() for s in f.readlines()] + else: test_sentences = [ "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", "Be a voice, not an echo.", @@ -530,9 +533,6 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): "This cake is great. It's so delicious and moist.", "Prior to November 22, 1963.", ] - else: - with open(config.test_sentences_file, "r") as f: - test_sentences = [s.strip() for s in f.readlines()] # test sentences test_audios = {} @@ -544,14 +544,13 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): if config.use_external_speaker_embedding_file and config.use_speaker_embedding else None ) - style_wav = config.get("gst_style_input") - if style_wav is None and config.use_gst: + style_wav = config.gst_style_input + if style_wav is None and config.gst is not None: # inicialize GST with zero dict. style_wav = {} print("WARNING: You don't provided a gst style wav, for this reason we use a zero tensor!") for i in range(config.gst["gst_num_style_tokens"]): style_wav[str(i)] = 0 - style_wav = config.get("gst_style_input") for idx, test_sentence in enumerate(test_sentences): try: wav, alignment, decoder_output, postnet_output, stop_tokens, _ = synthesis( @@ -639,14 +638,10 @@ def main(args): # pylint: disable=redefined-outer-name if "scaler" in checkpoint and config.mixed_precision: print(" > Restoring AMP Scaler...") scaler.load_state_dict(checkpoint["scaler"]) - if config.reinit_layers: - raise RuntimeError except (KeyError, RuntimeError): print(" > Partial model initialization...") model_dict = model.state_dict() model_dict = set_init_dict(model_dict, checkpoint["model"], c) - # torch.save(model_dict, os.path.join(OUT_PATH, 'state_dict.pt')) - # print("State Dict saved for debug in: ", os.path.join(OUT_PATH, 'state_dict.pt')) model.load_state_dict(model_dict) del model_dict @@ -743,12 +738,12 @@ if __name__ == "__main__": try: main(args) except KeyboardInterrupt: - # remove_experiment_folder(OUT_PATH) + remove_experiment_folder(OUT_PATH) try: sys.exit(0) except SystemExit: os._exit(0) # pylint: disable=protected-access except Exception: # pylint: disable=broad-except - # remove_experiment_folder(OUT_PATH) + remove_experiment_folder(OUT_PATH) traceback.print_exc() sys.exit(1) diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index e6934bc9..8667b2ec 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -22,7 +22,7 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): r=c.r, postnet_output_dim=int(c.audio["fft_size"] / 2 + 1), decoder_output_dim=c.audio["num_mels"], - gst=c.use_gst, + gst=c.gst, memory_size=c.memory_size, attn_type=c.attention_type, attn_win=c.windowing, diff --git a/TTS/tts/utils/synthesis.py b/TTS/tts/utils/synthesis.py index f2cfbd43..281ef55d 100644 --- a/TTS/tts/utils/synthesis.py +++ b/TTS/tts/utils/synthesis.py @@ -65,28 +65,31 @@ def compute_style_mel(style_wav, ap, cuda=False): def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None, speaker_embeddings=None): - speaker_embedding_g = speaker_id if speaker_id is not None else speaker_embeddings if "tacotron" in CONFIG.model.lower(): - if not CONFIG.use_gst: - style_mel = None - - if truncated: - decoder_output, postnet_output, alignments, stop_tokens = model.inference_truncated( - inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings - ) - else: + if CONFIG.gst: decoder_output, postnet_output, alignments, stop_tokens = model.inference( inputs, style_mel=style_mel, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings ) + else: + if truncated: + decoder_output, postnet_output, alignments, stop_tokens = model.inference_truncated( + inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings + ) + else: + decoder_output, postnet_output, alignments, stop_tokens = model.inference( + inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings + ) elif "glow" in CONFIG.model.lower(): inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) # pylint: disable=not-callable if hasattr(model, "module"): # distributed model postnet_output, _, _, _, alignments, _, _ = model.module.inference( - inputs, inputs_lengths, g=speaker_embedding_g + inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings ) else: - postnet_output, _, _, _, alignments, _, _ = model.inference(inputs, inputs_lengths, g=speaker_embedding_g) + postnet_output, _, _, _, alignments, _, _ = model.inference( + inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings + ) postnet_output = postnet_output.permute(0, 2, 1) # these only belong to tacotron models. decoder_output = None @@ -95,9 +98,13 @@ def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) # pylint: disable=not-callable if hasattr(model, "module"): # distributed model - postnet_output, alignments = model.module.inference(inputs, inputs_lengths, g=speaker_embedding_g) + postnet_output, alignments = model.module.inference( + inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings + ) else: - postnet_output, alignments = model.inference(inputs, inputs_lengths, g=speaker_embedding_g) + postnet_output, alignments = model.inference( + inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings + ) postnet_output = postnet_output.permute(0, 2, 1) # these only belong to tacotron models. decoder_output = None @@ -108,7 +115,7 @@ def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel def run_model_tf(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None): - if CONFIG.use_gst and style_mel is not None: + if CONFIG.gst and style_mel is not None: raise NotImplementedError(" [!] GST inference not implemented for TF") if truncated: raise NotImplementedError(" [!] Truncated inference not implemented for TF") @@ -120,7 +127,7 @@ def run_model_tf(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=No def run_model_tflite(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None): - if CONFIG.use_gst and style_mel is not None: + if CONFIG.gst and style_mel is not None: raise NotImplementedError(" [!] GST inference not implemented for TfLite") if truncated: raise NotImplementedError(" [!] Truncated inference not implemented for TfLite") @@ -249,7 +256,7 @@ def synthesis( """ # GST processing style_mel = None - if "use_gst" in CONFIG.keys() and CONFIG.use_gst and style_wav is not None: + if CONFIG.has('gst') and CONFIG.gst and style_wav is not None: if isinstance(style_wav, dict): style_mel = style_wav else: From 647163397db6f2c94bdd4a4046f7ccfa0154a4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 5 May 2021 02:31:12 +0200 Subject: [PATCH 27/87] coqpit refactoring --- TTS/tts/utils/synthesis.py | 8 ++++---- TTS/utils/arguments.py | 12 +++++------- TTS/utils/audio.py | 1 + TTS/utils/generic_utils.py | 2 +- TTS/utils/io.py | 22 ++++++---------------- 5 files changed, 17 insertions(+), 28 deletions(-) diff --git a/TTS/tts/utils/synthesis.py b/TTS/tts/utils/synthesis.py index 281ef55d..405cf2dc 100644 --- a/TTS/tts/utils/synthesis.py +++ b/TTS/tts/utils/synthesis.py @@ -23,8 +23,8 @@ def text_to_seqvec(text, CONFIG): text_cleaner, CONFIG.phoneme_language, CONFIG.enable_eos_bos_chars, - tp=CONFIG.characters if "characters" in CONFIG.keys() else None, - add_blank=CONFIG["add_blank"] if "add_blank" in CONFIG.keys() else False, + tp=CONFIG.characters, + add_blank=CONFIG.add_blank, ), dtype=np.int32, ) @@ -33,8 +33,8 @@ def text_to_seqvec(text, CONFIG): text_to_sequence( text, text_cleaner, - tp=CONFIG.characters if "characters" in CONFIG.keys() else None, - add_blank=CONFIG["add_blank"] if "add_blank" in CONFIG.keys() else False, + tp=CONFIG.characters, + add_blank=CONFIG.add_blank, ), dtype=np.int32, ) diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 344a8ba6..85e89191 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -2,12 +2,11 @@ # -*- coding: utf-8 -*- """Argument parser for training scripts.""" +import torch import argparse import glob -import json import os import re -import sys from TTS.tts.utils.text.symbols import parse_symbols from TTS.utils.console_logger import ConsoleLogger @@ -146,10 +145,9 @@ def process_args(args): config.parse_args(coqpit_overrides) if config.mixed_precision: print(" > Mixed precision mode is ON") - if not os.path.exists(config.output_path): + experiment_path = args.continue_path + if not experiment_path: experiment_path = create_experiment_folder(config.output_path, config.run_name, args.debug) - else: - experiment_path = config.output_path audio_path = os.path.join(experiment_path, "test_audios") # setup rank 0 process in distributed training if args.rank == 0: @@ -164,12 +162,12 @@ def process_args(args): if config.has("characters_config"): used_characters = parse_symbols() new_fields["characters"] = used_characters - copy_model_files(config, args.config_path, experiment_path, new_fields) + copy_model_files(config, experiment_path, new_fields) os.chmod(audio_path, 0o775) os.chmod(experiment_path, 0o775) tb_logger = TensorboardLogger(experiment_path, model_name=config.model) # write model desc to tensorboard - tb_logger.tb_add_text("model-description", config["run_description"], 0) + tb_logger.tb_add_text("model-config", f"
{config.to_json()}
", 0) c_logger = ConsoleLogger() return config, experiment_path, audio_path, c_logger, tb_logger diff --git a/TTS/utils/audio.py b/TTS/utils/audio.py index cb1341f4..222b4c74 100644 --- a/TTS/utils/audio.py +++ b/TTS/utils/audio.py @@ -21,6 +21,7 @@ class AudioProcessor(object): sample_rate (int, optional): target audio sampling rate. Defaults to None. resample (bool, optional): enable/disable resampling of the audio clips when the target sampling rate does not match the original sampling rate. Defaults to False. num_mels (int, optional): number of melspectrogram dimensions. Defaults to None. + log_func (int, optional): log exponent used for converting spectrogram aplitude to DB. min_level_db (int, optional): minimum db threshold for the computed melspectrograms. Defaults to None. frame_shift_ms (int, optional): milliseconds of frames between STFT columns. Defaults to None. frame_length_ms (int, optional): milliseconds of STFT window length. Defaults to None. diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index 274fb634..92244b8b 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -111,7 +111,7 @@ def set_init_dict(model_dict, checkpoint_state, c): # 2. filter out different size layers pretrained_dict = {k: v for k, v in pretrained_dict.items() if v.numel() == model_dict[k].numel()} # 3. skip reinit layers - if c.reinit_layers is not None: + if c.has('reinit_layers') and c.reinit_layers is not None: for reinit_layer_name in c.reinit_layers: pretrained_dict = {k: v for k, v in pretrained_dict.items() if reinit_layer_name not in k} # 4. overwrite entries in the existing state dict diff --git a/TTS/utils/io.py b/TTS/utils/io.py index 0a352d0d..6d233d24 100644 --- a/TTS/utils/io.py +++ b/TTS/utils/io.py @@ -58,35 +58,25 @@ def load_config(config_path: str) -> None: return config -def copy_model_files(c, config_file, out_path, new_fields): +def copy_model_files(config, out_path, new_fields): """Copy config.json and other model files to training folder and add new fields. Args: - c (dict): model config from config.json. - config_file (str): path to config file. + config (Coqpit): Coqpit config defining the training run. out_path (str): output path to copy the file. new_fields (dict): new fileds to be added or edited in the config file. """ - # copy config.json copy_config_path = os.path.join(out_path, "config.json") - config_lines = open(config_file, "r", encoding="utf-8").readlines() # add extra information fields - for key, value in new_fields.items(): - if isinstance(value, str): - new_line = '"{}":"{}",\n'.format(key, value) - else: - new_line = '"{}":{},\n'.format(key, json.dumps(value, ensure_ascii=False)) - config_lines.insert(1, new_line) - config_out_file = open(copy_config_path, "w", encoding="utf-8") - config_out_file.writelines(config_lines) - config_out_file.close() + config.update(new_fields, allow_new=True) + config.save_json(copy_config_path) # copy model stats file if available - if c.audio["stats_path"] is not None: + if config.audio.stats_path is not None: copy_stats_path = os.path.join(out_path, "scale_stats.npy") if not os.path.exists(copy_stats_path): copyfile( - c.audio["stats_path"], + config.audio.stats_path, copy_stats_path, ) From 35341d54826b43976cd95a836bdd6c777ffc8a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 5 May 2021 02:38:19 +0200 Subject: [PATCH 28/87] move bash script based tests to python with coqpit --- TTS/bin/train_speedy_speech.py | 1 - TTS/utils/arguments.py | 3 +- TTS/utils/generic_utils.py | 2 +- tests/tts_tests/__init__.py | 0 tests/tts_tests/test_speedy_speech_train.py | 48 +++++++++++++++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 tests/tts_tests/__init__.py create mode 100644 tests/tts_tests/test_speedy_speech_train.py diff --git a/TTS/bin/train_speedy_speech.py b/TTS/bin/train_speedy_speech.py index b9bdc6d1..2fba3df1 100644 --- a/TTS/bin/train_speedy_speech.py +++ b/TTS/bin/train_speedy_speech.py @@ -398,7 +398,6 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): "Prior to November 22, 1963.", ] - # test sentences test_audios = {} test_figures = {} diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 85e89191..35fa80eb 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -2,12 +2,13 @@ # -*- coding: utf-8 -*- """Argument parser for training scripts.""" -import torch import argparse import glob import os import re +import torch + from TTS.tts.utils.text.symbols import parse_symbols from TTS.utils.console_logger import ConsoleLogger from TTS.utils.generic_utils import create_experiment_folder, get_git_branch diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index 92244b8b..e8beff88 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -111,7 +111,7 @@ def set_init_dict(model_dict, checkpoint_state, c): # 2. filter out different size layers pretrained_dict = {k: v for k, v in pretrained_dict.items() if v.numel() == model_dict[k].numel()} # 3. skip reinit layers - if c.has('reinit_layers') and c.reinit_layers is not None: + if c.has("reinit_layers") and c.reinit_layers is not None: for reinit_layer_name in c.reinit_layers: pretrained_dict = {k: v for k, v in pretrained_dict.items() if reinit_layer_name not in k} # 4. overwrite entries in the existing state dict diff --git a/tests/tts_tests/__init__.py b/tests/tts_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/tts_tests/test_speedy_speech_train.py b/tests/tts_tests/test_speedy_speech_train.py new file mode 100644 index 00000000..341174f7 --- /dev/null +++ b/tests/tts_tests/test_speedy_speech_train.py @@ -0,0 +1,48 @@ +import glob +import os + +from tests import get_tests_output_path, run_cli +from TTS.config import BaseDatasetConfig +from TTS.tts.configs import SpeedySpeechConfig + +config_path = os.path.join(get_tests_output_path(), "test_speedy_speech_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = SpeedySpeechConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=True, + phoneme_language="en-us", + phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_speedy_speech.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " + "--coqpit.datasets.0.meta_file_attn_mask tests/data/ljspeech/metadata_attn_mask.txt" +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_speedy_speech.py --continue_path {continue_path} " +run_cli(command_train) From 816e7ee69803285566eeb0d25f534633326b4e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 5 May 2021 15:23:11 +0200 Subject: [PATCH 29/87] remove default configs.json as replacing with Coqpit configs --- .../ljspeech_tacotron2_dynamic_conv_attn.json | 173 ------------------ TTS/tts/configs/speedy_speech_ljspeech.json | 153 ---------------- 2 files changed, 326 deletions(-) delete mode 100644 TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json delete mode 100644 TTS/tts/configs/speedy_speech_ljspeech.json diff --git a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json b/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json deleted file mode 100644 index 947462aa..00000000 --- a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "model": "Tacotron2", - "run_name": "ljspeech-dcattn", - "run_description": "tacotron2 with dynamic convolution attention.", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. - "mixed_precision": true, // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate. - - // LOSS SETTINGS - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled - "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled - "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled - "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled - "ga_alpha": 0.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. - - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TACOTRON PRENET - "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. - "prenet_type": "original", // "original" or "bn". - "prenet_dropout": false, // enable/disable dropout at prenet. - - // TACOTRON ATTENTION - "attention_type": "dynamic_convolution", // 'original' , 'graves', 'dynamic_convolution' - "attention_heads": 4, // number of attention heads (only for 'graves') - "attention_norm": "softmax", // softmax or sigmoid. - "windowing": false, // Enables attention windowing. Used only in eval mode. - "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. - "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. - "transition_agent": false, // enable/disable transition agent of forward attention. - "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. - "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. - "double_decoder_consistency": false, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ - "ddc_r": 7, // reduction rate for coarse decoder. - - // STOPNET - "stopnet": true, // Train stopnet predicting the end of synthesis. - "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 4, //Number of batches to shuffle after bucketing. - "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 153, // DATASET-RELATED: maximum text length - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": false, // use global style tokens - "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_num_style_tokens). - "gst_embedding_dim": 512, - "gst_num_heads": 4, - "gst_num_style_tokens": 10, - "gst_use_speaker_embedding": false - }, - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers - "meta_file_val": null - } - ] -} - diff --git a/TTS/tts/configs/speedy_speech_ljspeech.json b/TTS/tts/configs/speedy_speech_ljspeech.json deleted file mode 100644 index f61f35cd..00000000 --- a/TTS/tts/configs/speedy_speech_ljspeech.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "model": "speedy_speech", - "run_name": "speedy-speech-ljspeech", - "run_description": "speedy-speech model for LJSpeech dataset.", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "&", - // "bos": "*", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZÇÃÀÁÂÊÉÍÓÔÕÚÛabcdefghijklmnopqrstuvwxyzçãàáâêéíóôõúû!(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ'̃' " - // }, - - "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model. - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // MODEL PARAMETERS - "positional_encoding": true, - "hidden_channels": 128, // defined globally all the hidden channels of the model - 128 default - "encoder_type": "residual_conv_bn", - "encoder_params":{ - "kernel_size": 4, - "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1], - "num_conv_blocks": 2, - "num_res_blocks": 13 - }, - "decoder_type": "residual_conv_bn", - "decoder_params":{ - "kernel_size": 4, - "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1], - "num_conv_blocks": 2, - "num_res_blocks": 17 - }, - - // TRAINING - "batch_size":64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":32, - "r": 1, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "loss_masking": true, // enable / disable loss masking against the sequence padding. - - // LOSS PARAMETERS - "ssim_alpha": 1, - "l1_alpha": 1, - "huber_alpha": 1, - - // VALIDATION - "run_eval": true, - "test_delay_epochs": -1, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": true, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 10000, // total number of epochs to train. - "lr": 0.002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.:set n - "mixed_precision": false, - - // DATA LOADING - "text_cleaner": "english_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 8, // number of evaluation data loader processes. - "batch_group_size": 4, //Number of batches to shuffle after bucketing. - "min_seq_len": 2, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 300, // DATASET-RELATED: maximum text length - "compute_f0": false, // compute f0 values in data-loader - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - - // PATHS - "output_path": "/home/erogol/Models/ljspeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/ljspeech_phonemes/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronoun[ciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "external_speaker_embedding_file": "/home/erogol/Data/libritts/speakers.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - - - // DATASETS - "datasets": // List of datasets. They all merged and they get different s$ - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", - "meta_file_val": null, - "meta_file_attn_mask": "/home/erogol/Data/LJSpeech-1.1/metadata_attn_mask.txt" // created by bin/compute_attention_masks.py - } - ] -} \ No newline at end of file From 720fe1305669eeacc7b63b4ad1023d1620112fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 6 May 2021 03:39:50 +0200 Subject: [PATCH 30/87] update glow_tts modules and training script for coqpit use --- TTS/bin/train_glow_tts.py | 145 ++++++++++++++++----------------- TTS/tts/utils/generic_utils.py | 2 +- 2 files changed, 69 insertions(+), 78 deletions(-) diff --git a/TTS/bin/train_glow_tts.py b/TTS/bin/train_glow_tts.py index d3b3d0e2..e93a4e8a 100644 --- a/TTS/bin/train_glow_tts.py +++ b/TTS/bin/train_glow_tts.py @@ -24,7 +24,7 @@ from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.synthesis import synthesis from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed, reduce_tensor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -35,45 +35,45 @@ use_cuda, num_gpus = setup_torch_training_env(True, False) def setup_loader(ap, r, is_val=False, verbose=False): - if is_val and not c.run_eval: + if is_val and not config.run_eval: loader = None else: dataset = MyDataset( r, - c.text_cleaner, + config.text_cleaner, compute_linear_spec=False, meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, - tp=c.characters if "characters" in c.keys() else None, - add_blank=c["add_blank"] if "add_blank" in c.keys() else False, - batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, - min_seq_len=c.min_seq_len, - max_seq_len=c.max_seq_len, - phoneme_cache_path=c.phoneme_cache_path, - use_phonemes=c.use_phonemes, - phoneme_language=c.phoneme_language, - enable_eos_bos=c.enable_eos_bos_chars, - use_noise_augment=c["use_noise_augment"] and not is_val, + tp=config.characters, + add_blank=config["add_blank"], + batch_group_size=0 if is_val else config.batch_group_size * config.batch_size, + min_seq_len=config.min_seq_len, + max_seq_len=config.max_seq_len, + phoneme_cache_path=config.phoneme_cache_path, + use_phonemes=config.use_phonemes, + phoneme_language=config.phoneme_language, + enable_eos_bos=config.enable_eos_bos_chars, + use_noise_augment=not is_val, verbose=verbose, speaker_mapping=speaker_mapping - if c.use_speaker_embedding and c.use_external_speaker_embedding_file + if config.use_speaker_embedding and config.use_external_speaker_embedding_file else None, ) - if c.use_phonemes and c.compute_input_seq_cache: + if config.use_phonemes and config.compute_input_seq_cache: # precompute phonemes to have a better estimate of sequence lengths. - dataset.compute_input_seq(c.num_loader_workers) + dataset.compute_input_seq(config.num_loader_workers) dataset.sort_items() sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader( dataset, - batch_size=c.eval_batch_size if is_val else c.batch_size, + batch_size=config.eval_batch_size if is_val else config.batch_size, shuffle=False, collate_fn=dataset.collate_fn, drop_last=False, sampler=sampler, - num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers, + num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers, pin_memory=False, ) return loader @@ -91,8 +91,8 @@ def format_data(data): avg_text_length = torch.mean(text_lengths.float()) avg_spec_length = torch.mean(mel_lengths.float()) - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: # return precomputed embedding vector speaker_c = data[8] else: @@ -147,7 +147,7 @@ def data_depended_init(data_loader, model): # forward pass model _ = model.forward(text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=spekaer_embed) - if num_iter == c.data_dep_init_iter: + if num_iter == config.data_dep_init_steps: break num_iter += 1 @@ -168,12 +168,12 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch_time = 0 keep_avg = KeepAverage() if use_cuda: - batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus)) + batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus)) else: - batch_n_iter = int(len(data_loader.dataset) / c.batch_size) + batch_n_iter = int(len(data_loader.dataset) / config.batch_size) end_time = time.time() c_logger.print_train_start() - scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None + scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None for num_iter, data in enumerate(data_loader): start_time = time.time() @@ -196,7 +196,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, optimizer.zero_grad() # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): z, logdet, y_mean, y_log_scale, alignments, o_dur_log, o_total_dur = model.forward( text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c ) @@ -205,19 +205,19 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, loss_dict = criterion(z, y_mean, y_log_scale, logdet, mel_lengths, o_dur_log, o_total_dur, text_lengths) # backward pass with loss scaling - if c.mixed_precision: + if config.mixed_precision: scaler.scale(loss_dict["loss"]).backward() scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) scaler.step(optimizer) scaler.update() else: loss_dict["loss"].backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) optimizer.step() # setup lr - if c.noam_schedule: + if config.noam_schedule: scheduler.step() # current_lr @@ -254,7 +254,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, keep_avg.update_values(update_train_values) # print training progress - if global_step % c.print_step == 0: + if global_step % config.print_step == 0: log_dict = { "avg_spec_length": [avg_spec_length, 1], # value, precision "avg_text_length": [avg_text_length, 1], @@ -267,13 +267,13 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, if args.rank == 0: # Plot Training Iter Stats # reduce TB load - if global_step % c.tb_plot_step == 0: + if global_step % config.tb_plot_step == 0: iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time} iter_stats.update(loss_dict) tb_logger.tb_train_iter_stats(global_step, iter_stats) - if global_step % c.save_step == 0: - if c.checkpoint: + if global_step % config.save_step == 0: + if config.checkpoint: # save model save_checkpoint( model, @@ -314,7 +314,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, # Sample audio train_audio = ap.inv_melspectrogram(const_spec.T) - tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"]) + tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() # print epoch stats @@ -325,7 +325,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch_stats = {"epoch_time": epoch_time} epoch_stats.update(keep_avg.avg_values) tb_logger.tb_train_epoch_stats(global_step, epoch_stats) - if c.tb_model_param_stats: + if config.tb_model_param_stats: tb_logger.tb_model_weights(model, global_step) return keep_avg.avg_values, global_step @@ -380,7 +380,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): update_train_values["avg_" + key] = value keep_avg.update_values(update_train_values) - if c.print_eval: + if config.print_eval: c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) if args.rank == 0: @@ -406,14 +406,17 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # Sample audio eval_audio = ap.inv_melspectrogram(const_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"]) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) # Plot Validation Stats tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) tb_logger.tb_eval_figures(global_step, eval_figures) - if args.rank == 0 and epoch >= c.test_delay_epochs: - if c.test_sentences_file is None: + if args.rank == 0 and epoch >= config.test_delay_epochs: + if config.test_sentences_file: + with open(config.test_sentences_file, "r") as f: + test_sentences = [s.strip() for s in f.readlines()] + else: test_sentences = [ "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", "Be a voice, not an echo.", @@ -421,16 +424,13 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): "This cake is great. It's so delicious and moist.", "Prior to November 22, 1963.", ] - else: - with open(c.test_sentences_file, "r") as f: - test_sentences = [s.strip() for s in f.readlines()] # test sentences test_audios = {} test_figures = {} print(" | > Synthesizing test sentences") - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][ "embedding" ] @@ -442,20 +442,20 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): speaker_id = None speaker_embedding = None - style_wav = c.get("style_wav_for_test") + style_wav = config.style_wav_for_test for idx, test_sentence in enumerate(test_sentences): try: wav, alignment, _, postnet_output, _, _ = synthesis( model, test_sentence, - c, + config, use_cuda, ap, speaker_id=speaker_id, speaker_embedding=speaker_embedding, style_wav=style_wav, truncated=False, - enable_eos_bos_chars=c.enable_eos_bos_chars, # pylint: disable=unused-argument + enable_eos_bos_chars=config.enable_eos_bos_chars, # pylint: disable=unused-argument use_griffin_lim=True, do_trim_silence=False, ) @@ -470,7 +470,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): except: # pylint: disable=bare-except print(" !! Error creating Test Sentence -", idx) traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"]) + tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"]) tb_logger.tb_test_figures(global_step, test_figures) return keep_avg.avg_values @@ -479,33 +479,27 @@ def main(args): # pylint: disable=redefined-outer-name # pylint: disable=global-variable-undefined global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping # Audio processor - ap = AudioProcessor(**c.audio) - if "characters" in c.keys(): - symbols, phonemes = make_symbols(**c.characters) + ap = AudioProcessor(**config.audio.to_dict()) + if config.has("characters") and config.characters: + symbols, phonemes = make_symbols(**config.characters.to_dict()) # DISTRUBUTED if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"]) + init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"]) # set model characters - model_characters = phonemes if c.use_phonemes else symbols + model_characters = phonemes if config.use_phonemes else symbols num_chars = len(model_characters) # load data instances - meta_data_train, meta_data_eval = load_meta_data(c.datasets) - - # set the portion of the data used for training - if "train_portion" in c.keys(): - meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)] - if "eval_portion" in c.keys(): - meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)] + meta_data_train, meta_data_eval = load_meta_data(config.datasets) # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH) + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) # setup model - model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim) - optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) + model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim) + optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) criterion = GlowTTSLoss() if args.restore_path: @@ -515,18 +509,16 @@ def main(args): # pylint: disable=redefined-outer-name # TODO: fix optimizer init, model.cuda() needs to be called before # optimizer restore optimizer.load_state_dict(checkpoint["optimizer"]) - if c.reinit_layers: - raise RuntimeError model.load_state_dict(checkpoint["model"]) except: # pylint: disable=bare-except print(" > Partial model initialization.") model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint["model"], c) + model_dict = set_init_dict(model_dict, checkpoint["model"], config) model.load_state_dict(model_dict) del model_dict for group in optimizer.param_groups: - group["initial_lr"] = c.lr + group["initial_lr"] = config.lr print(f" > Model restored from step {checkpoint['step']:d}", flush=True) args.restore_step = checkpoint["step"] else: @@ -540,8 +532,8 @@ def main(args): # pylint: disable=redefined-outer-name if num_gpus > 1: model = DDP_th(model, device_ids=[args.rank]) - if c.noam_schedule: - scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1) + if config.noam_schedule: + scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1) else: scheduler = None @@ -555,8 +547,8 @@ def main(args): # pylint: disable=redefined-outer-name print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] print(f" > Starting with loaded last best loss {best_loss}.") - keep_all_best = c.get("keep_all_best", False) - keep_after = c.get("keep_after", 10000) # void if keep_all_best False + keep_all_best = config.keep_all_best + keep_after = config.keep_after # void if keep_all_best False # define dataloaders train_loader = setup_loader(ap, 1, is_val=False, verbose=True) @@ -564,15 +556,15 @@ def main(args): # pylint: disable=redefined-outer-name global_step = args.restore_step model = data_depended_init(train_loader, model) - for epoch in range(0, c.epochs): - c_logger.print_epoch_start(epoch, c.epochs) + for epoch in range(0, config.epochs): + c_logger.print_epoch_start(epoch, config.epochs) train_avg_loss_dict, global_step = train( train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch ) eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) target_loss = train_avg_loss_dict["avg_loss"] - if c.run_eval: + if config.run_eval: target_loss = eval_avg_loss_dict["avg_loss"] best_loss = save_best_model( target_loss, @@ -581,7 +573,7 @@ def main(args): # pylint: disable=redefined-outer-name optimizer, global_step, epoch, - c.r, + config.r, OUT_PATH, model_characters, keep_all_best=keep_all_best, @@ -590,8 +582,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts") + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index 8667b2ec..b81a75ff 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -76,7 +76,7 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): encoder_type=c.encoder_type, encoder_params=c.encoder_params, use_encoder_prenet=c["use_encoder_prenet"], - inference_noise_scale=c.get("inference_noise_scale", 0.33), + inference_noise_scale=c.inference_noise_scale, num_flow_blocks_dec=12, kernel_size_dec=5, dilation_rate=1, From c6df8de80aca83a860e590ef96c9eb6830b6c820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 6 May 2021 03:40:26 +0200 Subject: [PATCH 31/87] remove output train folder at the end of the test --- tests/tts_tests/test_speedy_speech_train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tts_tests/test_speedy_speech_train.py b/tests/tts_tests/test_speedy_speech_train.py index 341174f7..1b356985 100644 --- a/tests/tts_tests/test_speedy_speech_train.py +++ b/tests/tts_tests/test_speedy_speech_train.py @@ -1,8 +1,8 @@ import glob import os +import shutil from tests import get_tests_output_path, run_cli -from TTS.config import BaseDatasetConfig from TTS.tts.configs import SpeedySpeechConfig config_path = os.path.join(get_tests_output_path(), "test_speedy_speech_config.json") @@ -46,3 +46,4 @@ continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getm # restore the model and continue training for one more epoch command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_speedy_speech.py --continue_path {continue_path} " run_cli(command_train) +shutil.rmtree(continue_path) From 51a7e0694586394c1fb8cd28af6da2eea5a0ab26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 6 May 2021 03:41:06 +0200 Subject: [PATCH 32/87] glow_tts_config.py and train test on python --- TTS/tts/configs/glow_tts_config.py | 50 ++++++++++++++++++++++++++ tests/tts_tests/test_glow_tts_train.py | 49 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 TTS/tts/configs/glow_tts_config.py create mode 100644 tests/tts_tests/test_glow_tts_train.py diff --git a/TTS/tts/configs/glow_tts_config.py b/TTS/tts/configs/glow_tts_config.py new file mode 100644 index 00000000..8474caae --- /dev/null +++ b/TTS/tts/configs/glow_tts_config.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass, field + +from .shared_configs import BaseTTSConfig + + +@dataclass +class GlowTTSConfig(BaseTTSConfig): + """Defines parameters for GlowTTS model.""" + + model: str = "glow_tts" + + # model params + encoder_type: str = "rel_pos_transformer" + encoder_params: dict = field( + default_factory=lambda: { + "kernel_size": 3, + "dropout_p": 0.1, + "num_layers": 6, + "num_heads": 2, + "hidden_channels_ffn": 768, + } + ) + use_encoder_prenet: bool = True + hidden_channels_encoder: int = 192 + hidden_channels_decoder: int = 192 + hidden_channels_duration_predictor: int = 256 + + # training params + data_dep_init_steps: int = 10 + + # inference params + style_wav_for_test: str = None + inference_noise_scale: float = 0.0 + + # multi-speaker settings + use_speaker_embedding: bool = False + use_external_speaker_embedding_file: bool = False + external_speaker_embedding_file: str = False + + # optimizer params + noam_schedule: bool = True + warmup_steps: int = 4000 + grad_clip: float = 5.0 + lr: float = 1e-3 + wd: float = 0.000001 + + # overrides + min_seq_len: int = 3 + max_seq_len: int = 500 + r: int = 1 diff --git a/tests/tts_tests/test_glow_tts_train.py b/tests/tts_tests/test_glow_tts_train.py new file mode 100644 index 00000000..bb630aef --- /dev/null +++ b/tests/tts_tests/test_glow_tts_train.py @@ -0,0 +1,49 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.tts.configs import GlowTTSConfig + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = GlowTTSConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=True, + phoneme_language="en-us", + phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_glow_tts.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " + "--coqpit.datasets.0.meta_file_attn_mask tests/data/ljspeech/metadata_attn_mask.txt" +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_glow_tts.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) From 7227e8f1d28388f0401a681bdf645e32b4fd7718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 6 May 2021 15:53:32 +0200 Subject: [PATCH 33/87] update train_align_tts.py for coqpit --- TTS/bin/train_align_tts.py | 986 +++++++++++++++------------- TTS/tts/configs/align_tts_config.py | 53 ++ 2 files changed, 575 insertions(+), 464 deletions(-) create mode 100644 TTS/tts/configs/align_tts_config.py diff --git a/TTS/bin/train_align_tts.py b/TTS/bin/train_align_tts.py index 6f268ed3..206d8b03 100644 --- a/TTS/bin/train_align_tts.py +++ b/TTS/bin/train_align_tts.py @@ -23,175 +23,342 @@ from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.synthesis import synthesis from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed, reduce_tensor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict from TTS.utils.radam import RAdam from TTS.utils.training import NoamLR, setup_torch_training_env -if __name__ == "__main__": - use_cuda, num_gpus = setup_torch_training_env(True, False) - # torch.autograd.set_detect_anomaly(True) +use_cuda, num_gpus = setup_torch_training_env(True, False) +# torch.autograd.set_detect_anomaly(True) - def setup_loader(ap, r, is_val=False, verbose=False): - if is_val and not c.run_eval: - loader = None + +def setup_loader(ap, r, is_val=False, verbose=False): + if is_val and not config.run_eval: + loader = None + else: + dataset = MyDataset( + r, + config.text_cleaner, + compute_linear_spec=False, + meta_data=meta_data_eval if is_val else meta_data_train, + ap=ap, + tp=config.characters, + add_blank=config["add_blank"], + batch_group_size=0 if is_val else config.batch_group_size * + config.batch_size, + min_seq_len=config.min_seq_len, + max_seq_len=config.max_seq_len, + phoneme_cache_path=config.phoneme_cache_path, + use_phonemes=config.use_phonemes, + phoneme_language=config.phoneme_language, + enable_eos_bos=config.enable_eos_bos_chars, + use_noise_augment=not is_val, + verbose=verbose, + speaker_mapping=speaker_mapping if config.use_speaker_embedding + and config.use_external_speaker_embedding_file else None, + ) + + if config.use_phonemes and config.compute_input_seq_cache: + # precompute phonemes to have a better estimate of sequence lengths. + dataset.compute_input_seq(config.num_loader_workers) + dataset.sort_items() + + sampler = DistributedSampler(dataset) if num_gpus > 1 else None + loader = DataLoader( + dataset, + batch_size=config.eval_batch_size if is_val else config.batch_size, + shuffle=False, + collate_fn=dataset.collate_fn, + drop_last=False, + sampler=sampler, + num_workers=config.num_val_loader_workers + if is_val else config.num_loader_workers, + pin_memory=False, + ) + return loader + + +def format_data(data): + # setup input data + text_input = data[0] + text_lengths = data[1] + speaker_names = data[2] + mel_input = data[4].permute(0, 2, 1) # B x D x T + mel_lengths = data[5] + item_idx = data[7] + avg_text_length = torch.mean(text_lengths.float()) + avg_spec_length = torch.mean(mel_lengths.float()) + + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: + # return precomputed embedding vector + speaker_c = data[8] else: - dataset = MyDataset( - r, - c.text_cleaner, - compute_linear_spec=False, - meta_data=meta_data_eval if is_val else meta_data_train, - ap=ap, - tp=c.characters if "characters" in c.keys() else None, - add_blank=c["add_blank"] if "add_blank" in c.keys() else False, - batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, - min_seq_len=c.min_seq_len, - max_seq_len=c.max_seq_len, - phoneme_cache_path=c.phoneme_cache_path, - use_phonemes=c.use_phonemes, - phoneme_language=c.phoneme_language, - enable_eos_bos=c.enable_eos_bos_chars, - use_noise_augment=not is_val, - verbose=verbose, - speaker_mapping=speaker_mapping - if c.use_speaker_embedding and c.use_external_speaker_embedding_file - else None, - ) + # return speaker_id to be used by an embedding layer + speaker_c = [ + speaker_mapping[speaker_name] for speaker_name in speaker_names + ] + speaker_c = torch.LongTensor(speaker_c) + else: + speaker_c = None + # dispatch data to GPU + if use_cuda: + text_input = text_input.cuda(non_blocking=True) + text_lengths = text_lengths.cuda(non_blocking=True) + mel_input = mel_input.cuda(non_blocking=True) + mel_lengths = mel_lengths.cuda(non_blocking=True) + if speaker_c is not None: + speaker_c = speaker_c.cuda(non_blocking=True) + return text_input, text_lengths, mel_input, mel_lengths, speaker_c, avg_text_length, avg_spec_length, item_idx - if c.use_phonemes and c.compute_input_seq_cache: - # precompute phonemes to have a better estimate of sequence lengths. - dataset.compute_input_seq(c.num_loader_workers) - dataset.sort_items() - sampler = DistributedSampler(dataset) if num_gpus > 1 else None - loader = DataLoader( - dataset, - batch_size=c.eval_batch_size if is_val else c.batch_size, - shuffle=False, - collate_fn=dataset.collate_fn, - drop_last=False, - sampler=sampler, - num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers, - pin_memory=False, - ) - return loader +def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, + epoch, training_phase): - def format_data(data): - # setup input data - text_input = data[0] - text_lengths = data[1] - speaker_names = data[2] - mel_input = data[4].permute(0, 2, 1) # B x D x T - mel_lengths = data[5] - item_idx = data[7] - avg_text_length = torch.mean(text_lengths.float()) - avg_spec_length = torch.mean(mel_lengths.float()) + model.train() + epoch_time = 0 + keep_avg = KeepAverage() + if use_cuda: + batch_n_iter = int( + len(data_loader.dataset) / (config.batch_size * num_gpus)) + else: + batch_n_iter = int(len(data_loader.dataset) / config.batch_size) + end_time = time.time() + c_logger.print_train_start() + scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None + for num_iter, data in enumerate(data_loader): + start_time = time.time() - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: - # return precomputed embedding vector - speaker_c = data[8] - else: - # return speaker_id to be used by an embedding layer - speaker_c = [speaker_mapping[speaker_name] for speaker_name in speaker_names] - speaker_c = torch.LongTensor(speaker_c) - else: - speaker_c = None - # dispatch data to GPU - if use_cuda: - text_input = text_input.cuda(non_blocking=True) - text_lengths = text_lengths.cuda(non_blocking=True) - mel_input = mel_input.cuda(non_blocking=True) - mel_lengths = mel_lengths.cuda(non_blocking=True) - if speaker_c is not None: - speaker_c = speaker_c.cuda(non_blocking=True) - return text_input, text_lengths, mel_input, mel_lengths, speaker_c, avg_text_length, avg_spec_length, item_idx + # format data + ( + text_input, + text_lengths, + mel_targets, + mel_lengths, + speaker_c, + avg_text_length, + avg_spec_length, + _, + ) = format_data(data) - def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, training_phase): + loader_time = time.time() - end_time - model.train() - epoch_time = 0 - keep_avg = KeepAverage() - if use_cuda: - batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus)) - else: - batch_n_iter = int(len(data_loader.dataset) / c.batch_size) - end_time = time.time() - c_logger.print_train_start() - scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None - for num_iter, data in enumerate(data_loader): - start_time = time.time() + global_step += 1 + optimizer.zero_grad() - # format data - ( + # forward pass model + with torch.cuda.amp.autocast(enabled=config.mixed_precision): + decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward( text_input, text_lengths, mel_targets, mel_lengths, - speaker_c, - avg_text_length, - avg_spec_length, - _, - ) = format_data(data) + g=speaker_c, + phase=training_phase) - loader_time = time.time() - end_time + # compute loss + loss_dict = criterion( + logp, + decoder_output, + mel_targets, + mel_lengths, + dur_output, + dur_mas_output, + text_lengths, + global_step, + phase=training_phase, + ) - global_step += 1 - optimizer.zero_grad() + # backward pass with loss scaling + if config.mixed_precision: + scaler.scale(loss_dict["loss"]).backward() + scaler.unscale_(optimizer) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), + config.grad_clip) + scaler.step(optimizer) + scaler.update() + else: + loss_dict["loss"].backward() + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), + config.grad_clip) + optimizer.step() + + # setup lr + if config.noam_schedule: + scheduler.step() + + # current_lr + current_lr = optimizer.param_groups[0]["lr"] + + # compute alignment error (the lower the better ) + align_error = 1 - alignment_diagonal_score(alignments, binary=True) + loss_dict["align_error"] = align_error + + step_time = time.time() - start_time + epoch_time += step_time + + # aggregate losses from processes + if num_gpus > 1: + loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, + num_gpus) + loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, + num_gpus) + loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, + num_gpus) + loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) + + # detach loss values + loss_dict_new = dict() + for key, value in loss_dict.items(): + if isinstance(value, (int, float)): + loss_dict_new[key] = value + else: + loss_dict_new[key] = value.item() + loss_dict = loss_dict_new + + # update avg stats + update_train_values = dict() + for key, value in loss_dict.items(): + update_train_values["avg_" + key] = value + update_train_values["avg_loader_time"] = loader_time + update_train_values["avg_step_time"] = step_time + keep_avg.update_values(update_train_values) + + # print training progress + if global_step % config.print_step == 0: + log_dict = { + "avg_spec_length": [avg_spec_length, 1], # value, precision + "avg_text_length": [avg_text_length, 1], + "step_time": [step_time, 4], + "loader_time": [loader_time, 2], + "current_lr": current_lr, + } + c_logger.print_train_step(batch_n_iter, num_iter, global_step, + log_dict, loss_dict, keep_avg.avg_values) + + if args.rank == 0: + # Plot Training Iter Stats + # reduce TB load + if global_step % config.tb_plot_step == 0: + iter_stats = { + "lr": current_lr, + "grad_norm": grad_norm, + "step_time": step_time + } + iter_stats.update(loss_dict) + tb_logger.tb_train_iter_stats(global_step, iter_stats) + + if global_step % config.save_step == 0: + if config.checkpoint: + # save model + save_checkpoint( + model, + optimizer, + global_step, + epoch, + 1, + OUT_PATH, + model_characters, + model_loss=loss_dict["loss"], + ) + + # wait all kernels to be completed + torch.cuda.synchronize() + + # Diagnostic visualizations + if decoder_output is not None: + idx = np.random.randint(mel_targets.shape[0]) + pred_spec = decoder_output[idx].detach().data.cpu().numpy( + ).T + gt_spec = mel_targets[idx].data.cpu().numpy().T + align_img = alignments[idx].data.cpu() + + figures = { + "prediction": plot_spectrogram(pred_spec, ap), + "ground_truth": plot_spectrogram(gt_spec, ap), + "alignment": plot_alignment(align_img), + } + + tb_logger.tb_train_figures(global_step, figures) + + # Sample audio + train_audio = ap.inv_melspectrogram(pred_spec.T) + tb_logger.tb_train_audios(global_step, + {"TrainAudio": train_audio}, + config.audio["sample_rate"]) + end_time = time.time() + + # print epoch stats + c_logger.print_train_epoch_end(global_step, epoch, epoch_time, keep_avg) + + # Plot Epoch Stats + if args.rank == 0: + epoch_stats = {"epoch_time": epoch_time} + epoch_stats.update(keep_avg.avg_values) + tb_logger.tb_train_epoch_stats(global_step, epoch_stats) + if config.tb_model_param_stats: + tb_logger.tb_model_weights(model, global_step) + return keep_avg.avg_values, global_step + + +@torch.no_grad() +def evaluate(data_loader, model, criterion, ap, global_step, epoch, + training_phase): + model.eval() + epoch_time = 0 + keep_avg = KeepAverage() + c_logger.print_eval_start() + if data_loader is not None: + for num_iter, data in enumerate(data_loader): + start_time = time.time() + + # format data + text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _ = format_data( + data) # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward( - text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase - ) - - # compute loss - loss_dict = criterion( - logp, - decoder_output, + text_input, + text_lengths, mel_targets, mel_lengths, - dur_output, - dur_mas_output, - text_lengths, - global_step, - phase=training_phase, - ) + g=speaker_c, + phase=training_phase) - # backward pass with loss scaling - if c.mixed_precision: - scaler.scale(loss_dict["loss"]).backward() - scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) - scaler.step(optimizer) - scaler.update() - else: - loss_dict["loss"].backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) - optimizer.step() - - # setup lr - if c.noam_schedule: - scheduler.step() - - # current_lr - current_lr = optimizer.param_groups[0]["lr"] - - # compute alignment error (the lower the better ) - align_error = 1 - alignment_diagonal_score(alignments, binary=True) - loss_dict["align_error"] = align_error + # compute loss + loss_dict = criterion( + logp, + decoder_output, + mel_targets, + mel_lengths, + dur_output, + dur_mas_output, + text_lengths, + global_step, + phase=training_phase, + ) + # step time step_time = time.time() - start_time epoch_time += step_time + # compute alignment score + align_error = 1 - alignment_diagonal_score(alignments, binary=True) + loss_dict["align_error"] = align_error + # aggregate losses from processes if num_gpus > 1: - loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus) - loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, num_gpus) - loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, num_gpus) - loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) + loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, + num_gpus) + loss_dict["loss_ssim"] = reduce_tensor( + loss_dict["loss_ssim"].data, num_gpus) + loss_dict["loss_dur"] = reduce_tensor( + loss_dict["loss_dur"].data, num_gpus) + loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, + num_gpus) # detach loss values loss_dict_new = dict() @@ -206,357 +373,248 @@ if __name__ == "__main__": update_train_values = dict() for key, value in loss_dict.items(): update_train_values["avg_" + key] = value - update_train_values["avg_loader_time"] = loader_time - update_train_values["avg_step_time"] = step_time keep_avg.update_values(update_train_values) - # print training progress - if global_step % c.print_step == 0: - log_dict = { - "avg_spec_length": [avg_spec_length, 1], # value, precision - "avg_text_length": [avg_text_length, 1], - "step_time": [step_time, 4], - "loader_time": [loader_time, 2], - "current_lr": current_lr, - } - c_logger.print_train_step(batch_n_iter, num_iter, global_step, log_dict, loss_dict, keep_avg.avg_values) + if config.print_eval: + c_logger.print_eval_step(num_iter, loss_dict, + keep_avg.avg_values) - if args.rank == 0: - # Plot Training Iter Stats - # reduce TB load - if global_step % c.tb_plot_step == 0: - iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time} - iter_stats.update(loss_dict) - tb_logger.tb_train_iter_stats(global_step, iter_stats) - - if global_step % c.save_step == 0: - if c.checkpoint: - # save model - save_checkpoint( - model, - optimizer, - global_step, - epoch, - 1, - OUT_PATH, - model_characters, - model_loss=loss_dict["loss"], - ) - - # wait all kernels to be completed - torch.cuda.synchronize() - - # Diagnostic visualizations - if decoder_output is not None: - idx = np.random.randint(mel_targets.shape[0]) - pred_spec = decoder_output[idx].detach().data.cpu().numpy().T - gt_spec = mel_targets[idx].data.cpu().numpy().T - align_img = alignments[idx].data.cpu() - - figures = { - "prediction": plot_spectrogram(pred_spec, ap), - "ground_truth": plot_spectrogram(gt_spec, ap), - "alignment": plot_alignment(align_img), - } - - tb_logger.tb_train_figures(global_step, figures) - - # Sample audio - train_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"]) - end_time = time.time() - - # print epoch stats - c_logger.print_train_epoch_end(global_step, epoch, epoch_time, keep_avg) - - # Plot Epoch Stats if args.rank == 0: - epoch_stats = {"epoch_time": epoch_time} - epoch_stats.update(keep_avg.avg_values) - tb_logger.tb_train_epoch_stats(global_step, epoch_stats) - if c.tb_model_param_stats: - tb_logger.tb_model_weights(model, global_step) - return keep_avg.avg_values, global_step + # Diagnostic visualizations + idx = np.random.randint(mel_targets.shape[0]) + pred_spec = decoder_output[idx].detach().data.cpu().numpy().T + gt_spec = mel_targets[idx].data.cpu().numpy().T + align_img = alignments[idx].data.cpu() - @torch.no_grad() - def evaluate(data_loader, model, criterion, ap, global_step, epoch, training_phase): - model.eval() - epoch_time = 0 - keep_avg = KeepAverage() - c_logger.print_eval_start() - if data_loader is not None: - for num_iter, data in enumerate(data_loader): - start_time = time.time() + eval_figures = { + "prediction": plot_spectrogram(pred_spec, ap, + output_fig=False), + "ground_truth": plot_spectrogram(gt_spec, ap, + output_fig=False), + "alignment": plot_alignment(align_img, output_fig=False), + } - # format data - text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _ = format_data(data) + # Sample audio + eval_audio = ap.inv_melspectrogram(pred_spec.T) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, + config.audio["sample_rate"]) - # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): - decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward( - text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase - ) + # Plot Validation Stats + tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) + tb_logger.tb_eval_figures(global_step, eval_figures) - # compute loss - loss_dict = criterion( - logp, - decoder_output, - mel_targets, - mel_lengths, - dur_output, - dur_mas_output, - text_lengths, - global_step, - phase=training_phase, + if args.rank == 0 and epoch >= config.test_delay_epochs: + if config.test_sentences_file: + with open(config.test_sentences_file, "r") as f: + test_sentences = [s.strip() for s in f.readlines()] + else: + test_sentences = [ + "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", + "Be a voice, not an echo.", + "I'm sorry Dave. I'm afraid I can't do that.", + "This cake is great. It's so delicious and moist.", + "Prior to November 22, 1963.", + ] + + # test sentences + test_audios = {} + test_figures = {} + print(" | > Synthesizing test sentences") + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: + speaker_embedding = speaker_mapping[list( + speaker_mapping.keys())[randrange( + len(speaker_mapping) - 1)]]["embedding"] + speaker_id = None + else: + speaker_id = 0 + speaker_embedding = None + else: + speaker_id = None + speaker_embedding = None + + for idx, test_sentence in enumerate(test_sentences): + try: + wav, alignment, _, postnet_output, _, _ = synthesis( + model, + test_sentence, + config, + use_cuda, + ap, + speaker_id=speaker_id, + speaker_embedding=speaker_embedding, + style_wav=None, + truncated=False, + enable_eos_bos_chars=config.enable_eos_bos_chars, # pylint: disable=unused-argument + use_griffin_lim=True, + do_trim_silence=False, ) - # step time - step_time = time.time() - start_time - epoch_time += step_time - - # compute alignment score - align_error = 1 - alignment_diagonal_score(alignments, binary=True) - loss_dict["align_error"] = align_error - - # aggregate losses from processes - if num_gpus > 1: - loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus) - loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, num_gpus) - loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, num_gpus) - loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) - - # detach loss values - loss_dict_new = dict() - for key, value in loss_dict.items(): - if isinstance(value, (int, float)): - loss_dict_new[key] = value - else: - loss_dict_new[key] = value.item() - loss_dict = loss_dict_new - - # update avg stats - update_train_values = dict() - for key, value in loss_dict.items(): - update_train_values["avg_" + key] = value - keep_avg.update_values(update_train_values) - - if c.print_eval: - c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) - - if args.rank == 0: - # Diagnostic visualizations - idx = np.random.randint(mel_targets.shape[0]) - pred_spec = decoder_output[idx].detach().data.cpu().numpy().T - gt_spec = mel_targets[idx].data.cpu().numpy().T - align_img = alignments[idx].data.cpu() - - eval_figures = { - "prediction": plot_spectrogram(pred_spec, ap, output_fig=False), - "ground_truth": plot_spectrogram(gt_spec, ap, output_fig=False), - "alignment": plot_alignment(align_img, output_fig=False), - } - - # Sample audio - eval_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"]) - - # Plot Validation Stats - tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) - tb_logger.tb_eval_figures(global_step, eval_figures) - - if args.rank == 0 and epoch >= c.test_delay_epochs: - if c.test_sentences_file is None: - test_sentences = [ - "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", - "Be a voice, not an echo.", - "I'm sorry Dave. I'm afraid I can't do that.", - "This cake is great. It's so delicious and moist.", - "Prior to November 22, 1963.", - ] - else: - with open(c.test_sentences_file, "r") as f: - test_sentences = [s.strip() for s in f.readlines()] - - # test sentences - test_audios = {} - test_figures = {} - print(" | > Synthesizing test sentences") - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: - speaker_embedding = speaker_mapping[ - list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)] - ]["embedding"] - speaker_id = None - else: - speaker_id = 0 - speaker_embedding = None - else: - speaker_id = None - speaker_embedding = None - - style_wav = c.get("style_wav_for_test") - for idx, test_sentence in enumerate(test_sentences): - try: - wav, alignment, _, postnet_output, _, _ = synthesis( - model, - test_sentence, - c, - use_cuda, - ap, - speaker_id=speaker_id, - speaker_embedding=speaker_embedding, - style_wav=style_wav, - truncated=False, - enable_eos_bos_chars=c.enable_eos_bos_chars, # pylint: disable=unused-argument - use_griffin_lim=True, - do_trim_silence=False, - ) - - file_path = os.path.join(AUDIO_PATH, str(global_step)) - os.makedirs(file_path, exist_ok=True) - file_path = os.path.join(file_path, "TestSentence_{}.wav".format(idx)) - ap.save_wav(wav, file_path) - test_audios["{}-audio".format(idx)] = wav - test_figures["{}-prediction".format(idx)] = plot_spectrogram(postnet_output, ap) - test_figures["{}-alignment".format(idx)] = plot_alignment(alignment) - except: # pylint: disable=bare-except - print(" !! Error creating Test Sentence -", idx) - traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"]) - tb_logger.tb_test_figures(global_step, test_figures) - return keep_avg.avg_values - - def main(args): # pylint: disable=redefined-outer-name - # pylint: disable=global-variable-undefined - global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping - # Audio processor - ap = AudioProcessor(**c.audio) - if "characters" in c.keys(): - symbols, phonemes = make_symbols(**c.characters) - - # DISTRUBUTED - if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"]) - - # set model characters - model_characters = phonemes if c.use_phonemes else symbols - num_chars = len(model_characters) - - # load data instances - meta_data_train, meta_data_eval = load_meta_data(c.datasets, eval_split=True) - - # set the portion of the data used for training if set in config.json - if "train_portion" in c.keys(): - meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)] - if "eval_portion" in c.keys(): - meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)] - - # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH) - - # setup model - model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim) - optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) - criterion = AlignTTSLoss(c) - - if args.restore_path: - print(f" > Restoring from {os.path.basename(args.restore_path)} ...") - checkpoint = torch.load(args.restore_path, map_location="cpu") - try: - # TODO: fix optimizer init, model.cuda() needs to be called before - # optimizer restore - optimizer.load_state_dict(checkpoint["optimizer"]) - if c.reinit_layers: - raise RuntimeError - model.load_state_dict(checkpoint["model"]) + file_path = os.path.join(AUDIO_PATH, str(global_step)) + os.makedirs(file_path, exist_ok=True) + file_path = os.path.join(file_path, + "TestSentence_{}.wav".format(idx)) + ap.save_wav(wav, file_path) + test_audios["{}-audio".format(idx)] = wav + test_figures["{}-prediction".format(idx)] = plot_spectrogram( + postnet_output, ap) + test_figures["{}-alignment".format(idx)] = plot_alignment( + alignment) except: # pylint: disable=bare-except - print(" > Partial model initialization.") - model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint["model"], c) - model.load_state_dict(model_dict) - del model_dict + print(" !! Error creating Test Sentence -", idx) + traceback.print_exc() + tb_logger.tb_test_audios(global_step, test_audios, + config.audio["sample_rate"]) + tb_logger.tb_test_figures(global_step, test_figures) + return keep_avg.avg_values - for group in optimizer.param_groups: - group["initial_lr"] = c.lr - print(" > Model restored from step %d" % checkpoint["step"], flush=True) - args.restore_step = checkpoint["step"] - else: - args.restore_step = 0 - if use_cuda: - model.cuda() - criterion.cuda() +def main(args): # pylint: disable=redefined-outer-name + # pylint: disable=global-variable-undefined + global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping + # Audio processor + ap = AudioProcessor(**config.audio.to_dict()) + if config.has("characters") and config.characters: + symbols, phonemes = make_symbols(**config.characters.to_dict()) - # DISTRUBUTED - if num_gpus > 1: - model = DDP_th(model, device_ids=[args.rank]) + # DISTRUBUTED + if num_gpus > 1: + init_distributed(args.rank, num_gpus, args.group_id, + config.distributed["backend"], + config.distributed["url"]) - if c.noam_schedule: - scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1) - else: - scheduler = None + # set model characters + model_characters = phonemes if config.use_phonemes else symbols + num_chars = len(model_characters) - num_params = count_parameters(model) - print("\n > Model has {} parameters".format(num_params), flush=True) + # load data instances + meta_data_train, meta_data_eval = load_meta_data(config.datasets, + eval_split=True) - if args.restore_step == 0 or not args.best_path: - best_loss = float("inf") - print(" > Starting with inf best loss.") - else: - print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") - best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] - print(f" > Starting with loaded last best loss {best_loss}.") - keep_all_best = c.get("keep_all_best", False) - keep_after = c.get("keep_after", 10000) # void if keep_all_best False + # parse speakers + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers( + config, args, meta_data_train, OUT_PATH) - # define dataloaders - train_loader = setup_loader(ap, 1, is_val=False, verbose=True) - eval_loader = setup_loader(ap, 1, is_val=True, verbose=True) + # setup model + model = setup_model(num_chars, + num_speakers, + config, + speaker_embedding_dim=speaker_embedding_dim) + optimizer = RAdam(model.parameters(), + lr=config.lr, + weight_decay=0, + betas=(0.9, 0.98), + eps=1e-9) + criterion = AlignTTSLoss(config) - global_step = args.restore_step + if args.restore_path: + print(f" > Restoring from {os.path.basename(args.restore_path)} ...") + checkpoint = torch.load(args.restore_path, map_location="cpu") + try: + # TODO: fix optimizer init, model.cuda() needs to be called before + # optimizer restore + optimizer.load_state_dict(checkpoint["optimizer"]) + if config.reinit_layers: + raise RuntimeError + model.load_state_dict(checkpoint["model"]) + except: # pylint: disable=bare-except + print(" > Partial model initialization.") + model_dict = model.state_dict() + model_dict = set_init_dict(model_dict, checkpoint["model"], config) + model.load_state_dict(model_dict) + del model_dict - def set_phase(): - """Set AlignTTS training phase""" - if isinstance(c.phase_start_steps, list): - vals = [i < global_step for i in c.phase_start_steps] - if not True in vals: - phase = 0 - else: - phase = ( - len(c.phase_start_steps) - [i < global_step for i in c.phase_start_steps][::-1].index(True) - 1 - ) + for group in optimizer.param_groups: + group["initial_lr"] = config.lr + print(" > Model restored from step %d" % checkpoint["step"], + flush=True) + args.restore_step = checkpoint["step"] + else: + args.restore_step = 0 + + if use_cuda: + model.cuda() + criterion.cuda() + + # DISTRUBUTED + if num_gpus > 1: + model = DDP_th(model, device_ids=[args.rank]) + + if config.noam_schedule: + scheduler = NoamLR(optimizer, + warmup_steps=config.warmup_steps, + last_epoch=args.restore_step - 1) + else: + scheduler = None + + num_params = count_parameters(model) + print("\n > Model has {} parameters".format(num_params), flush=True) + + if args.restore_step == 0 or not args.best_path: + best_loss = float("inf") + print(" > Starting with inf best loss.") + else: + print(" > Restoring best loss from " + f"{os.path.basename(args.best_path)} ...") + best_loss = torch.load(args.best_path, + map_location="cpu")["model_loss"] + print(f" > Starting with loaded last best loss {best_loss}.") + keep_all_best = config.keep_all_best + keep_after = config.keep_after # void if keep_all_best False + + # define dataloaders + train_loader = setup_loader(ap, 1, is_val=False, verbose=True) + eval_loader = setup_loader(ap, 1, is_val=True, verbose=True) + + global_step = args.restore_step + + def set_phase(): + """Set AlignTTS training phase""" + if isinstance(config.phase_start_steps, list): + vals = [i < global_step for i in config.phase_start_steps] + if not True in vals: + phase = 0 else: - phase = None - return phase + phase = ( + len(config.phase_start_steps) - + [i < global_step + for i in config.phase_start_steps][::-1].index(True) - 1) + else: + phase = None + return phase - for epoch in range(0, c.epochs): - cur_phase = set_phase() - print(f"\n > Current AlignTTS phase: {cur_phase}") - c_logger.print_epoch_start(epoch, c.epochs) - train_avg_loss_dict, global_step = train( - train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, cur_phase - ) - eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch, cur_phase) - c_logger.print_epoch_end(epoch, eval_avg_loss_dict) - target_loss = train_avg_loss_dict["avg_loss"] - if c.run_eval: - target_loss = eval_avg_loss_dict["avg_loss"] - best_loss = save_best_model( - target_loss, - best_loss, - model, - optimizer, - global_step, - epoch, - 1, - OUT_PATH, - model_characters, - keep_all_best=keep_all_best, - keep_after=keep_after, - ) + for epoch in range(0, config.epochs): + cur_phase = set_phase() + print(f"\n > Current AlignTTS phase: {cur_phase}") + c_logger.print_epoch_start(epoch, config.epochs) + train_avg_loss_dict, global_step = train(train_loader, model, + criterion, optimizer, + scheduler, ap, global_step, + epoch, cur_phase) + eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, + global_step, epoch, cur_phase) + c_logger.print_epoch_end(epoch, eval_avg_loss_dict) + target_loss = train_avg_loss_dict["avg_loss"] + if config.run_eval: + target_loss = eval_avg_loss_dict["avg_loss"] + best_loss = save_best_model( + target_loss, + best_loss, + model, + optimizer, + global_step, + epoch, + 1, + OUT_PATH, + model_characters, + keep_all_best=keep_all_best, + keep_after=keep_after, + ) - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts") + +if __name__ == "__main__": + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training( + sys.argv) try: main(args) diff --git a/TTS/tts/configs/align_tts_config.py b/TTS/tts/configs/align_tts_config.py new file mode 100644 index 00000000..fae4c608 --- /dev/null +++ b/TTS/tts/configs/align_tts_config.py @@ -0,0 +1,53 @@ +from dataclasses import dataclass, field + +from .shared_configs import BaseTTSConfig + + +@dataclass +class AlignTTSConfig(BaseTTSConfig): + """Defines parameters for AlignTTS model.""" + + model: str = "align_tts" + # model specific params + positional_encoding: bool = True + hidden_channels_dp: int = 256 + hidden_channels: int = 256 + encoder_type: str = "fftransformer" + encoder_params: dict = field( + default_factory=lambda: { + "hidden_channels_ffn": 1024, + "num_heads": 2, + "num_layers": 6, + "dropout_p": 0.1 + }) + decoder_type: str = "fftransformer" + decoder_params: dict = field( + default_factory=lambda: { + "hidden_channels_ffn": 1024, + "num_heads": 2, + "num_layers": 6, + "dropout_p": 0.1 + }) + phase_start_steps: list = None + + ssim_alpha: float = 1.0 + spec_loss_alpha: float = 1.0 + dur_loss_alpha: float = 1.0 + mdn_alpha: float = 1.0 + + # multi-speaker settings + use_speaker_embedding: bool = False + use_external_speaker_embedding_file: bool = False + external_speaker_embedding_file: str = False + + # optimizer parameters + noam_schedule: bool = False + warmup_steps: int = 4000 + lr: float = 1e-4 + wd: float = 1e-6 + grad_clip: float = 5.0 + + # overrides + min_seq_len: int = 13 + max_seq_len: int = 200 + r: int = 1 From 7663bc63c16617c1be467a3fe1db840b512a0020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 6 May 2021 16:03:09 +0200 Subject: [PATCH 34/87] add Coqpit configs for the TTS models --- TTS/tts/configs/__init__.py | 17 +++ TTS/tts/configs/aligntts_transformers.json | 0 TTS/tts/configs/config.json | 126 ----------------- TTS/tts/configs/glow_tts_gated_conv.json | 138 ------------------- TTS/tts/configs/glow_tts_ljspeech.json | 151 --------------------- TTS/tts/configs/shared_configs.py | 83 +++++++++++ TTS/tts/configs/speedy_speech_config.py | 53 ++++++++ TTS/tts/configs/tacotron2_config.py | 10 ++ TTS/tts/configs/tacotron_config.py | 70 ++++++++++ 9 files changed, 233 insertions(+), 415 deletions(-) create mode 100644 TTS/tts/configs/__init__.py delete mode 100644 TTS/tts/configs/aligntts_transformers.json delete mode 100644 TTS/tts/configs/config.json delete mode 100644 TTS/tts/configs/glow_tts_gated_conv.json delete mode 100644 TTS/tts/configs/glow_tts_ljspeech.json create mode 100644 TTS/tts/configs/shared_configs.py create mode 100644 TTS/tts/configs/speedy_speech_config.py create mode 100644 TTS/tts/configs/tacotron2_config.py create mode 100644 TTS/tts/configs/tacotron_config.py diff --git a/TTS/tts/configs/__init__.py b/TTS/tts/configs/__init__.py new file mode 100644 index 00000000..5ad4fe8c --- /dev/null +++ b/TTS/tts/configs/__init__.py @@ -0,0 +1,17 @@ +import importlib +import os +from inspect import isclass + +# import all files under configs/ +configs_dir = os.path.dirname(__file__) +for file in os.listdir(configs_dir): + path = os.path.join(configs_dir, file) + if not file.startswith("_") and not file.startswith(".") and (file.endswith(".py") or os.path.isdir(path)): + config_name = file[: file.find(".py")] if file.endswith(".py") else file + module = importlib.import_module("TTS.tts.configs." + config_name) + for attribute_name in dir(module): + attribute = getattr(module, attribute_name) + + if isclass(attribute): + # Add the class to this package's variables + globals()[attribute_name] = attribute diff --git a/TTS/tts/configs/aligntts_transformers.json b/TTS/tts/configs/aligntts_transformers.json deleted file mode 100644 index e69de29b..00000000 diff --git a/TTS/tts/configs/config.json b/TTS/tts/configs/config.json deleted file mode 100644 index 95e787e0..00000000 --- a/TTS/tts/configs/config.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "attention_heads": 4, - "attention_norm": "sigmoid", - "attention_type": "original", - "audio_config": { - "clip_norm": true, - "do_trim_silence": true, - "fft_size": 1024, - "frame_length_ms": null, - "frame_shift_ms": null, - "griffin_lim_iters": 60, - "hop_length": 256, - "max_norm": 4, - "mel_fmax": 7600, - "mel_fmin": 50, - "min_level_db": -100, - "num_mels": 80, - "power": 1.5, - "preemphasis": 0, - "ref_level_db": 20, - "sample_rate": 22050, - "signal_norm": true, - "spec_gain": 1, - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy", - "symmetric_norm": true, - "trim_db": 60, - "win_length": 1024 - }, - "bidirectional_decoder": false, - "compute_input_seq_cache": false, - "ddc_r": 7, - "decoder_diff_spec_alpha": 0.25, - "decoder_loss_alpha": 0.5, - "decoder_ssim_alpha": 0.5, - "double_decoder_consistency": true, - "enable_eos_bos_chars": false, - "forward_attn_mask": false, - "ga_alpha": 5, - "grad_clip": 1, - "gradual_training": [ - [ - 0, - 7, - 64 - ], - [ - 1, - 5, - 64 - ], - [ - 50000, - 3, - 32 - ], - [ - 130000, - 2, - 32 - ], - [ - 290000, - 1, - 32 - ] - ], - "location_attn": true, - "lr": 0.0001, - "memory_size": -1, - "noam_schedule": false, - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", - "phoneme_language": "en-us", - "postnet_diff_spec_alpha": 0.25, - "postnet_loss_alpha": 0.25, - "postnet_ssim_alpha": 0.25, - "prenet_dropout": false, - "prenet_type": "original", - "r": 7, - "separate_stopnet": true, - "seq_len_norm": false, - "stopnet": true, - "stopnet_pos_weight": 15, - "test_sentences_file": null, - "text_cleaner": "phoneme_cleaners", - "training_config": { - "batch_group_size": 4, - "batch_size": 32, - "checkpoint": true, - "datasets": [ - { - "meta_file_train": "metadata.csv", - "meta_file_val": null, - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/" - } - ], - "epochs": 1000, - "eval_batch_size": 16, - "keep_after": 10000, - "keep_all_best": false, - "loss_masking": true, - "max_seq_len": 153, - "min_seq_len": 6, - "mixed_precision": true, - "model": "Tacotron2", - "num_loader_workers": 4, - "num_val_loader_workers": 4, - "output_path": "/home/erogol/Models/LJSpeech/", - "print_eval": false, - "print_step": 25, - "run_description": "tacotron2 with DDC and differential spectral loss.", - "run_eval": true, - "run_name": "ljspeech-ddc", - "save_step": 10000, - "tb_model_param_stats": false, - "tb_plot_step": 100, - "test_delay_epochs": 10, - "use_noise_augment": true - }, - "transition_agent": false, - "use_forward_attn": false, - "use_phonemes": true, - "warmup_steps": 4000, - "wd": 0.000001, - "windowing": false -} diff --git a/TTS/tts/configs/glow_tts_gated_conv.json b/TTS/tts/configs/glow_tts_gated_conv.json deleted file mode 100644 index c4d7b1e5..00000000 --- a/TTS/tts/configs/glow_tts_gated_conv.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "model": "glow_tts", - "run_name": "glow-tts-gatedconv", - "run_description": "glow-tts model training with gated conv.", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Griffin-Lim - "power": 1.1, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 1.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model. - - // DISTRIBUTED TRAINING - "apex_amp_level": null, // APEX amp optimization level. "O1" is currently supported. - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54323" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // MODEL PARAMETERS - "use_mas": false, // use Monotonic Alignment Search if true. Otherwise use pre-computed attention alignments. - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 1, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "loss_masking": true, // enable / disable loss masking against the sequence padding. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 0, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": true, // use noam warmup and lr schedule. - "grad_clip": 5.0, // upper limit for gradients for clipping. - "epochs": 10000, // total number of epochs to train. - "lr": 1e-3, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - "encoder_type": "gatedconv", - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "apex_amp_level": null, - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 0, //Number of batches to shuffle after bucketing. - "min_seq_len": 3, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 500, // DATASET-RELATED: maximum text length - "compute_f0": false, // compute f0 values in data-loader - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "style_wav_for_test": null, // path to style wav file to be used in TacotronGST inference. - "use_gst": false, // TACOTRON ONLY: use global style tokens - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", - "meta_file_val": null - // "path_for_attn": "/home/erogol/Data/LJSpeech-1.1/alignments/" - } - ] -} - - diff --git a/TTS/tts/configs/glow_tts_ljspeech.json b/TTS/tts/configs/glow_tts_ljspeech.json deleted file mode 100644 index 5a4c47c2..00000000 --- a/TTS/tts/configs/glow_tts_ljspeech.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "model": "glow_tts", - "run_name": "glow-tts-residual_bn_conv", - "run_description": "glow-tts model training with residual BN conv.", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Griffin-Lim - "power": 1.1, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram.00 - - // Normalization parameters - "signal_norm": false, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 1.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model. - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // MODEL PARAMETERS - // "use_mas": false, // use Monotonic Alignment Search if true. Otherwise use pre-computed attention alignments. - "hidden_channels_encoder": 192, - "hidden_channels_decoder": 192, - "hidden_channels_duration_predictor": 256, - "use_encoder_prenet": true, - "encoder_type": "rel_pos_transformer", - "encoder_params": { - "kernel_size":3, - "dropout_p": 0.1, - "num_layers": 6, - "num_heads": 2, - "hidden_channels_ffn": 768, - "input_length": null - }, - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 1, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "mixed_precision": true, - "data_dep_init_iter": 10, - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 0, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": true, // use noam warmup and lr schedule. - "grad_clip": 5.0, // upper limit for gradients for clipping. - "epochs": 10000, // total number of epochs to train. - "lr": 1e-3, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 0, //Number of batches to shuffle after bucketing. - "min_seq_len": 3, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 500, // DATASET-RELATED: maximum text length - "compute_f0": false, // compute f0 values in data-loader - "use_noise_augment": true, //add a random noise to audio signal for augmentation at training . - "compute_input_seq_cache": true, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_external_speaker_embedding_file": false, - "style_wav_for_test": null, // path to style wav file to be used in TacotronGST inference. - "use_gst": false, // TACOTRON ONLY: use global style tokens - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", - "meta_file_val": null - // "path_for_attn": "/home/erogol/Data/LJSpeech-1.1/alignments/" - } - ] - } - - diff --git a/TTS/tts/configs/shared_configs.py b/TTS/tts/configs/shared_configs.py new file mode 100644 index 00000000..c82b821e --- /dev/null +++ b/TTS/tts/configs/shared_configs.py @@ -0,0 +1,83 @@ +from dataclasses import asdict, dataclass, field +from typing import List + +from coqpit import MISSING, Coqpit, check_argument + +from TTS.config import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig + + +@dataclass +class GSTConfig(Coqpit): + """Defines Global Style Toke module""" + + gst_style_input_wav: str = None + gst_style_input_weights: dict = None + gst_embedding_dim: int = 256 + gst_use_speaker_embedding: bool = False + gst_num_heads: int = 4 + gst_num_style_tokens: int = 10 + + def check_values( + self, + ): + """Check config fields""" + c = asdict(self) + super().check_values() + check_argument("gst_style_input_weights", c, restricted=False) + check_argument("gst_style_input_wav", c, restricted=False) + check_argument("gst_embedding_dim", c, restricted=True, min_val=0, max_val=1000) + check_argument("gst_use_speaker_embedding", c, restricted=False) + check_argument("gst_num_heads", c, restricted=True, min_val=2, max_val=10) + check_argument("gst_num_style_tokens", c, restricted=True, min_val=1, max_val=1000) + + +@dataclass +class CharactersConfig: + """Defines character or phoneme set used by the model""" + + pad: str = None + eos: str = None + bos: str = None + characters: str = None + punctuations: str = None + phonemes: str = None + + def check_values( + self, + ): + """Check config fields""" + c = asdict(self) + check_argument("pad", c, "characters", restricted=True) + check_argument("eos", c, "characters", restricted=True) + check_argument("bos", c, "characters", restricted=True) + check_argument("characters", c, "characters", restricted=True) + check_argument("phonemes", c, restricted=True) + check_argument("punctuations", c, "characters", restricted=True) + + +@dataclass +class BaseTTSConfig(BaseTrainingConfig): + """Shared parameters among all the tts models.""" + + audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) + # phoneme settings + use_phonemes: bool = False + phoneme_language: str = None + compute_input_seq_cache: bool = False + text_cleaner: str = MISSING + enable_eos_bos_chars: bool = False + test_sentences_file: str = "" + phoneme_cache_path: str = None + # vocabulary parameters + characters: CharactersConfig = None + # training params + batch_group_size: int = 0 + loss_masking: bool = None + # dataloading + min_seq_len: int = 1 + max_seq_len: int = float("inf") + compute_f0: bool = False + use_noise_augment: bool = False + add_blank: bool = False + # dataset + datasets: List[BaseDatasetConfig] = field(default_factory=lambda: [BaseDatasetConfig()]) diff --git a/TTS/tts/configs/speedy_speech_config.py b/TTS/tts/configs/speedy_speech_config.py new file mode 100644 index 00000000..a2e90cb8 --- /dev/null +++ b/TTS/tts/configs/speedy_speech_config.py @@ -0,0 +1,53 @@ +from dataclasses import dataclass, field + +from .shared_configs import BaseTTSConfig + + +@dataclass +class SpeedySpeechConfig(BaseTTSConfig): + """Defines parameters for Speedy Speech (feed-forward encoder-decoder) based models.""" + + model: str = "speedy_speech" + # model specific params + positional_encoding: bool = True + hidden_channels: int = 128 + encoder_type: str = "residual_conv_bn" + encoder_params: dict = field( + default_factory=lambda: { + "kernel_size": 4, + "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1], + "num_conv_blocks": 2, + "num_res_blocks": 13, + } + ) + decoder_type: str = "residual_conv_bn" + decoder_params: dict = field( + default_factory=lambda: { + "kernel_size": 4, + "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1], + "num_conv_blocks": 2, + "num_res_blocks": 17, + } + ) + + # multi-speaker settings + use_speaker_embedding: bool = False + use_external_speaker_embedding_file: bool = False + external_speaker_embedding_file: str = False + + # optimizer parameters + noam_schedule: bool = False + warmup_steps: int = 4000 + lr: float = 1e-4 + wd: float = 1e-6 + grad_clip: float = 5.0 + + # loss params + ssim_alpha: float = 1.0 + huber_alpha: float = 1.0 + l1_alpha: float = 1.0 + + # overrides + min_seq_len: int = 13 + max_seq_len: int = 200 + r: int = 1 diff --git a/TTS/tts/configs/tacotron2_config.py b/TTS/tts/configs/tacotron2_config.py new file mode 100644 index 00000000..e6767d41 --- /dev/null +++ b/TTS/tts/configs/tacotron2_config.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + +from TTS.tts.configs.tacotron_config import TacotronConfig + + +@dataclass +class Tacotron2Config(TacotronConfig): + """Defines parameters for Tacotron2 based models.""" + + model: str = "tacotron2" diff --git a/TTS/tts/configs/tacotron_config.py b/TTS/tts/configs/tacotron_config.py new file mode 100644 index 00000000..8b1ed20c --- /dev/null +++ b/TTS/tts/configs/tacotron_config.py @@ -0,0 +1,70 @@ +from dataclasses import asdict, dataclass +from typing import List + +from coqpit import check_argument + +from .shared_configs import BaseTTSConfig, GSTConfig + + +@dataclass +class TacotronConfig(BaseTTSConfig): + """Defines parameters for Tacotron based models.""" + + model: str = "tacotron" + gst: GSTConfig = None + gst_style_input: str = None + # model specific params + r: int = 2 + gradual_training: List = None + memory_size: int = -1 + prenet_type: str = "original" + prenet_dropout: bool = True + prenet_dropout_at_inference: bool = False + stopnet: bool = True + separate_stopnet: bool = True + stopnet_pos_weight: float = 10.0 + + # attention layers + attention_type: str = "original" + attention_heads: int = None + attention_norm: str = "sigmoid" + windowing: bool = False + use_forward_attn: bool = False + forward_attn_mask: bool = False + transition_agent: bool = False + location_attn: bool = True + + # advance methods + bidirectional_decoder: bool = False + double_decoder_consistency: bool = False + ddc_r: int = 6 + + # multi-speaker settings + use_speaker_embedding: bool = False + use_external_speaker_embedding_file: bool = False + external_speaker_embedding_file: str = False + + # optimizer parameters + noam_schedule: bool = False + warmup_steps: int = 4000 + lr: float = 1e-4 + wd: float = 1e-6 + grad_clip: float = 5.0 + seq_len_norm: bool = False + loss_masking: bool = True + + # loss params + decoder_loss_alpha: float = 0.25 + postnet_loss_alpha: float = 0.25 + postnet_diff_spec_alpha: float = 0.25 + decoder_diff_spec_alpha: float = 0.25 + decoder_ssim_alpha: float = 0.25 + postnet_ssim_alpha: float = 0.25 + ga_alpha: float = 5.0 + + +@dataclass +class Tacotron2Config(TacotronConfig): + """Defines parameters for Tacotron2 based models.""" + + model: str = "tacotron2" From bcebd69d09818ef0df6a6231b6af36edfb8ccf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 6 May 2021 16:27:58 +0200 Subject: [PATCH 35/87] remove bash tts training tests --- TTS/bin/compute_statistics.py | 2 +- tests/bash_tests/test_aligntts_train.sh | 13 ------ tests/bash_tests/test_glow-tts_train.sh | 13 ------ tests/bash_tests/test_speedy_speech_train.sh | 13 ------ tests/bash_tests/test_tacotron_train.sh | 36 -------------- tests/tts_tests/test_align_tts_train.py | 48 +++++++++++++++++++ tests/tts_tests/test_tacotron2_train.py | 49 ++++++++++++++++++++ tests/tts_tests/test_tacotron_train.py | 48 +++++++++++++++++++ 8 files changed, 146 insertions(+), 76 deletions(-) delete mode 100755 tests/bash_tests/test_aligntts_train.sh delete mode 100755 tests/bash_tests/test_glow-tts_train.sh delete mode 100755 tests/bash_tests/test_speedy_speech_train.sh delete mode 100755 tests/bash_tests/test_tacotron_train.sh create mode 100644 tests/tts_tests/test_align_tts_train.py create mode 100644 tests/tts_tests/test_tacotron2_train.py create mode 100644 tests/tts_tests/test_tacotron_train.py diff --git a/TTS/bin/compute_statistics.py b/TTS/bin/compute_statistics.py index d87ecf95..b4ee6df7 100755 --- a/TTS/bin/compute_statistics.py +++ b/TTS/bin/compute_statistics.py @@ -12,7 +12,7 @@ from TTS.tts.datasets.preprocess import load_meta_data from TTS.utils.audio import AudioProcessor # from TTS.utils.io import load_config -from TTS.utils.config import load_config +from TTS.config import load_config def main(): diff --git a/tests/bash_tests/test_aligntts_train.sh b/tests/bash_tests/test_aligntts_train.sh deleted file mode 100755 index 38d46520..00000000 --- a/tests/bash_tests/test_aligntts_train.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_align_tts.py --config_path $BASEDIR/../inputs/test_align_tts.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_align_tts.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ diff --git a/tests/bash_tests/test_glow-tts_train.sh b/tests/bash_tests/test_glow-tts_train.sh deleted file mode 100755 index 04aef2ad..00000000 --- a/tests/bash_tests/test_glow-tts_train.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_glow_tts.py --config_path $BASEDIR/../inputs/test_glow_tts.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_glow_tts.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ diff --git a/tests/bash_tests/test_speedy_speech_train.sh b/tests/bash_tests/test_speedy_speech_train.sh deleted file mode 100755 index 2276034f..00000000 --- a/tests/bash_tests/test_speedy_speech_train.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_speedy_speech.py --config_path $BASEDIR/../inputs/test_speedy_speech.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_speedy_speech.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ diff --git a/tests/bash_tests/test_tacotron_train.sh b/tests/bash_tests/test_tacotron_train.sh deleted file mode 100755 index 4aacf69c..00000000 --- a/tests/bash_tests/test_tacotron_train.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" - -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ - -# run Tacotron bi-directional decoder -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron_bd_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ - -# Tacotron2 -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron2_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ - diff --git a/tests/tts_tests/test_align_tts_train.py b/tests/tts_tests/test_align_tts_train.py new file mode 100644 index 00000000..aefc7dc3 --- /dev/null +++ b/tests/tts_tests/test_align_tts_train.py @@ -0,0 +1,48 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.tts.configs import AlignTTSConfig + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = AlignTTSConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=True, + phoneme_language="en-us", + phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_align_tts.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_align_tts.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_tacotron2_train.py b/tests/tts_tests/test_tacotron2_train.py new file mode 100644 index 00000000..2ac17502 --- /dev/null +++ b/tests/tts_tests/test_tacotron2_train.py @@ -0,0 +1,49 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.tts.configs import Tacotron2Config + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = Tacotron2Config( + r=5, + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=True, + phoneme_language="en-us", + phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_tacotron.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_tacotron_train.py b/tests/tts_tests/test_tacotron_train.py new file mode 100644 index 00000000..b45e4a64 --- /dev/null +++ b/tests/tts_tests/test_tacotron_train.py @@ -0,0 +1,48 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.tts.configs import TacotronConfig + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = TacotronConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=True, + phoneme_language="en-us", + phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_tacotron.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) From 6ee6a563bc1bf4ff5ff495ccd329e246501db03e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 6 May 2021 17:07:26 +0200 Subject: [PATCH 36/87] add coqpit to the requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 204d1191..da04b6b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ torch>=1.7 tqdm umap-learn==0.4.6 unidecode==0.4.20 +coqpit From e6f45b9eb712d3ac7f523552b453e8bda104880d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 03:39:49 +0200 Subject: [PATCH 37/87] update train_vocoder_gan.py for coqpit --- TTS/bin/train_vocoder_gan.py | 13 +++++++------ .../test_vocoder_gan_datasets.py | 0 tests/{ => vocoder_tests}/test_vocoder_losses.py | 0 .../test_vocoder_melgan_discriminator.py | 0 .../test_vocoder_melgan_generator.py | 0 .../test_vocoder_parallel_wavegan_discriminator.py | 0 .../test_vocoder_parallel_wavegan_generator.py | 0 tests/{ => vocoder_tests}/test_vocoder_pqmf.py | 0 tests/{ => vocoder_tests}/test_vocoder_rwd.py | 0 .../test_vocoder_tf_melgan_generator.py | 0 tests/{ => vocoder_tests}/test_vocoder_tf_pqmf.py | 0 tests/{ => vocoder_tests}/test_vocoder_wavernn.py | 0 .../test_vocoder_wavernn_datasets.py | 0 .../test_wavegrad.py} | 0 tests/{ => vocoder_tests}/test_wavegrad_layers.py | 0 15 files changed, 7 insertions(+), 6 deletions(-) rename tests/{ => vocoder_tests}/test_vocoder_gan_datasets.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_losses.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_melgan_discriminator.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_melgan_generator.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_parallel_wavegan_discriminator.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_parallel_wavegan_generator.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_pqmf.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_rwd.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_tf_melgan_generator.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_tf_pqmf.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_wavernn.py (100%) rename tests/{ => vocoder_tests}/test_vocoder_wavernn_datasets.py (100%) rename tests/{test_wavegrad_train.py => vocoder_tests/test_wavegrad.py} (100%) rename tests/{ => vocoder_tests}/test_wavegrad_layers.py (100%) diff --git a/TTS/bin/train_vocoder_gan.py b/TTS/bin/train_vocoder_gan.py index f33df3e8..4159f12f 100755 --- a/TTS/bin/train_vocoder_gan.py +++ b/TTS/bin/train_vocoder_gan.py @@ -16,7 +16,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP_th from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -163,7 +163,6 @@ def train( y_hat_sub=y_hat_sub, y_sub=y_G_sub, ) - loss_G = loss_G_dict["G_loss"] # optimizer generator @@ -469,7 +468,7 @@ def main(args): # pylint: disable=redefined-outer-name eval_data, train_data = load_wav_data(c.data_path, c.eval_split_size) # setup audio processor - ap = AudioProcessor(**c.audio) + ap = AudioProcessor(**c.audio.to_dict()) # DISTRUBUTED if num_gpus > 1: @@ -620,13 +619,15 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder") - + args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) except KeyboardInterrupt: remove_experiment_folder(OUT_PATH) + try: + sys.exit(0) + except SystemExit: + os._exit(0) # pylint: disable=protected-access except Exception: # pylint: disable=broad-except remove_experiment_folder(OUT_PATH) traceback.print_exc() diff --git a/tests/test_vocoder_gan_datasets.py b/tests/vocoder_tests/test_vocoder_gan_datasets.py similarity index 100% rename from tests/test_vocoder_gan_datasets.py rename to tests/vocoder_tests/test_vocoder_gan_datasets.py diff --git a/tests/test_vocoder_losses.py b/tests/vocoder_tests/test_vocoder_losses.py similarity index 100% rename from tests/test_vocoder_losses.py rename to tests/vocoder_tests/test_vocoder_losses.py diff --git a/tests/test_vocoder_melgan_discriminator.py b/tests/vocoder_tests/test_vocoder_melgan_discriminator.py similarity index 100% rename from tests/test_vocoder_melgan_discriminator.py rename to tests/vocoder_tests/test_vocoder_melgan_discriminator.py diff --git a/tests/test_vocoder_melgan_generator.py b/tests/vocoder_tests/test_vocoder_melgan_generator.py similarity index 100% rename from tests/test_vocoder_melgan_generator.py rename to tests/vocoder_tests/test_vocoder_melgan_generator.py diff --git a/tests/test_vocoder_parallel_wavegan_discriminator.py b/tests/vocoder_tests/test_vocoder_parallel_wavegan_discriminator.py similarity index 100% rename from tests/test_vocoder_parallel_wavegan_discriminator.py rename to tests/vocoder_tests/test_vocoder_parallel_wavegan_discriminator.py diff --git a/tests/test_vocoder_parallel_wavegan_generator.py b/tests/vocoder_tests/test_vocoder_parallel_wavegan_generator.py similarity index 100% rename from tests/test_vocoder_parallel_wavegan_generator.py rename to tests/vocoder_tests/test_vocoder_parallel_wavegan_generator.py diff --git a/tests/test_vocoder_pqmf.py b/tests/vocoder_tests/test_vocoder_pqmf.py similarity index 100% rename from tests/test_vocoder_pqmf.py rename to tests/vocoder_tests/test_vocoder_pqmf.py diff --git a/tests/test_vocoder_rwd.py b/tests/vocoder_tests/test_vocoder_rwd.py similarity index 100% rename from tests/test_vocoder_rwd.py rename to tests/vocoder_tests/test_vocoder_rwd.py diff --git a/tests/test_vocoder_tf_melgan_generator.py b/tests/vocoder_tests/test_vocoder_tf_melgan_generator.py similarity index 100% rename from tests/test_vocoder_tf_melgan_generator.py rename to tests/vocoder_tests/test_vocoder_tf_melgan_generator.py diff --git a/tests/test_vocoder_tf_pqmf.py b/tests/vocoder_tests/test_vocoder_tf_pqmf.py similarity index 100% rename from tests/test_vocoder_tf_pqmf.py rename to tests/vocoder_tests/test_vocoder_tf_pqmf.py diff --git a/tests/test_vocoder_wavernn.py b/tests/vocoder_tests/test_vocoder_wavernn.py similarity index 100% rename from tests/test_vocoder_wavernn.py rename to tests/vocoder_tests/test_vocoder_wavernn.py diff --git a/tests/test_vocoder_wavernn_datasets.py b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py similarity index 100% rename from tests/test_vocoder_wavernn_datasets.py rename to tests/vocoder_tests/test_vocoder_wavernn_datasets.py diff --git a/tests/test_wavegrad_train.py b/tests/vocoder_tests/test_wavegrad.py similarity index 100% rename from tests/test_wavegrad_train.py rename to tests/vocoder_tests/test_wavegrad.py diff --git a/tests/test_wavegrad_layers.py b/tests/vocoder_tests/test_wavegrad_layers.py similarity index 100% rename from tests/test_wavegrad_layers.py rename to tests/vocoder_tests/test_wavegrad_layers.py From 757e90b1cc69fb928b869c7246d92b1a5d2aaceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 03:40:34 +0200 Subject: [PATCH 38/87] load_config function to initialize the right Coqpit for the given model --- TTS/config/__init__.py | 40 ++++++++++++++++++++++++++++++++++++++++ TTS/utils/arguments.py | 3 ++- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 TTS/config/__init__.py diff --git a/TTS/config/__init__.py b/TTS/config/__init__.py new file mode 100644 index 00000000..85e7d9b9 --- /dev/null +++ b/TTS/config/__init__.py @@ -0,0 +1,40 @@ +from TTS.config.shared_configs import * + +import json +import os + +import yaml + +from TTS.utils.generic_utils import find_module + + +def _search_configs(model_name): + config_class = None + paths = ["TTS.tts.configs", "TTS.vocoder.configs"] + for path in paths: + try: + config_class = find_module(path, model_name + "_config") + except ModuleNotFoundError: + pass + if config_class is None: + raise ModuleNotFoundError() + return config_class + + +def load_config(config_path: str) -> None: + config_dict = {} + ext = os.path.splitext(config_path)[1] + if ext in (".yml", ".yaml"): + with open(config_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + elif ext == ".json": + with open(config_path, "r", encoding="utf-8") as f: + input_str = f.read() + data = json.loads(input_str) + else: + raise TypeError(f" [!] Unknown config file type {ext}") + config_dict.update(data) + config_class = _search_configs(config_dict["model"].lower()) + config = config_class() + config.from_dict(config_dict) + return config diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 35fa80eb..cf64edae 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -12,7 +12,8 @@ import torch from TTS.tts.utils.text.symbols import parse_symbols from TTS.utils.console_logger import ConsoleLogger from TTS.utils.generic_utils import create_experiment_folder, get_git_branch -from TTS.utils.io import copy_model_files, load_config +from TTS.utils.io import copy_model_files +from TTS.config import load_config from TTS.utils.tensorboard_logger import TensorboardLogger From 045f1c3e76a6b26607e3d131bd353383115eb70c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 09:54:41 +0200 Subject: [PATCH 39/87] add hifigan train test --- tests/vocoder_tests/test_hifigan_train.py | 43 +++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/vocoder_tests/test_hifigan_train.py diff --git a/tests/vocoder_tests/test_hifigan_train.py b/tests/vocoder_tests/test_hifigan_train.py new file mode 100644 index 00000000..83a3f4b8 --- /dev/null +++ b/tests/vocoder_tests/test_hifigan_train.py @@ -0,0 +1,43 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.vocoder.configs import HifiganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = HifiganConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=1024, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) From 78b3825d0b25109d8e7dac07a9244321bac8f4b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 15:31:49 +0200 Subject: [PATCH 40/87] update train scripts for coqpit --- TTS/bin/train_vocoder_wavegrad.py | 12 +- TTS/bin/train_vocoder_wavernn.py | 30 +--- TTS/vocoder/configs/hifigan_ljspeech.json | 164 ---------------------- 3 files changed, 8 insertions(+), 198 deletions(-) delete mode 100644 TTS/vocoder/configs/hifigan_ljspeech.json diff --git a/TTS/bin/train_vocoder_wavegrad.py b/TTS/bin/train_vocoder_wavegrad.py index 1f039a67..c0fcff51 100644 --- a/TTS/bin/train_vocoder_wavegrad.py +++ b/TTS/bin/train_vocoder_wavegrad.py @@ -15,7 +15,7 @@ from torch.optim import Adam from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -131,12 +131,12 @@ def train(model, criterion, optimizer, scheduler, scaler, ap, global_step, epoch if c.mixed_precision: scaler.scale(loss).backward() scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.clip_grad) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) scaler.step(optimizer) scaler.update() else: loss.backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.clip_grad) + grad_norm = torch.nn.utils.grad_clip_norm_(model.parameters(), c.clip_grad) optimizer.step() # schedule update @@ -311,7 +311,7 @@ def main(args): # pylint: disable=redefined-outer-name eval_data, train_data = load_wav_data(c.data_path, c.eval_split_size) # setup audio processor - ap = AudioProcessor(**c.audio) + ap = AudioProcessor(**c.audio.to_dict()) # DISTRUBUTED if num_gpus > 1: @@ -416,9 +416,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder") - + args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) except KeyboardInterrupt: diff --git a/TTS/bin/train_vocoder_wavernn.py b/TTS/bin/train_vocoder_wavernn.py index 8a2d8d3a..bcad9493 100644 --- a/TTS/bin/train_vocoder_wavernn.py +++ b/TTS/bin/train_vocoder_wavernn.py @@ -11,7 +11,7 @@ import torch from torch.utils.data import DataLoader from TTS.tts.utils.visual import plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict from TTS.utils.radam import RAdam @@ -307,29 +307,7 @@ def main(args): # pylint: disable=redefined-outer-name global train_data, eval_data # setup audio processor - ap = AudioProcessor(**c.audio) - - # print(f" > Loading wavs from: {c.data_path}") - # if c.feature_path is not None: - # print(f" > Loading features from: {c.feature_path}") - # eval_data, train_data = load_wav_feat_data( - # c.data_path, c.feature_path, c.eval_split_size - # ) - # else: - # mel_feat_path = os.path.join(OUT_PATH, "mel") - # feat_data = find_feat_files(mel_feat_path) - # if feat_data: - # print(f" > Loading features from: {mel_feat_path}") - # eval_data, train_data = load_wav_feat_data( - # c.data_path, mel_feat_path, c.eval_split_size - # ) - # else: - # print(" > No feature data found. Preprocessing...") - # # preprocessing feature data from given wav files - # preprocess_wav_files(OUT_PATH, CONFIG, ap) - # eval_data, train_data = load_wav_feat_data( - # c.data_path, mel_feat_path, c.eval_split_size - # ) + ap = AudioProcessor(**c.audio.to_dict()) print(f" > Loading wavs from: {c.data_path}") if c.feature_path is not None: @@ -438,9 +416,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder") - + args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) except KeyboardInterrupt: diff --git a/TTS/vocoder/configs/hifigan_ljspeech.json b/TTS/vocoder/configs/hifigan_ljspeech.json deleted file mode 100644 index 23cbf3f8..00000000 --- a/TTS/vocoder/configs/hifigan_ljspeech.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - "run_name": "hifigan", - "run_description": "hifigan mean-var scaling", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "log_func": "np.log10", - "do_sound_norm": true, - - // Silence trimming - "do_trim_silence": false,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/.local/share/tts/tts_models--en--ljspeech--speedy-speech-wn/scale_stats.npy" - }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54324" - }, - - // MODEL PARAMETERS - "use_pqmf": false, - - // LOSS PARAMETERS - "use_stft_loss": false, - "use_subband_stft_loss": false, - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": true, // use only with melgan discriminators - "use_l1_spec_loss": true, - - // loss weights - "stft_loss_weight": 0, - "subband_stft_loss_weight": 0, - "mse_G_loss_weight": 1, - "hinge_G_loss_weight": 0, - "feat_match_loss_weight": 10, - "l1_spec_loss_weight": 45, - - // multiscale stft loss parameters - // "stft_loss_params": { - // "n_ffts": [1024, 2048, 512], - // "hop_lengths": [120, 240, 50], - // "win_lengths": [600, 1200, 240] - // }, - - "l1_spec_loss_params": { - "use_mel": true, - "sample_rate": 22050, - "n_fft": 1024, - "hop_length": 256, - "win_length": 1024, - "n_mels": 80, - "mel_fmin": 0.0, - "mel_fmax": null - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "hifigan_discriminator", - //"discriminator_model_params":{ - // "peroids": [2, 3, 5, 7, 11], - // "base_channels": 16, - // "max_channels":512, - // "downsample_factors":[4, 4, 4] - //}, - "steps_to_start_discriminator": 0, // steps required to start GAN trainining.1 - "diff_samples_for_G_and_D": false, // draw a new sample from the dataset for the D pass. - - // GENERATOR - "generator_model": "hifigan_generator", - "generator_model_params": { - "upsample_factors":[8,8,2,2], - "upsample_kernel_sizes": [16,16,4,4], - "upsample_initial_channel": 512, - "resblock_kernel_sizes": [3,7,11], - "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], - "resblock_type": "1" - }, - - // DATASET - "data_path": "/home/erogol/gdrive/Datasets/LJSpeech-1.1/wavs/", - "feature_path": null, - // "feature_path": "/home/erogol/gdrive/Datasets/non-binary-voice-files/tacotron-DCA/", - "seq_len": 8192, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 16, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/gdrive/Trainings/LJSpeech/" -} - - From 6f4eed94f5fd9f20271b1c77dc6c768c0727794c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 15:33:18 +0200 Subject: [PATCH 41/87] remove *.json vocoder configs --- .../multiband-melgan_and_rwd_config.json | 158 ------------------ .../configs/multiband_melgan_config.json | 148 ---------------- .../configs/parallel_wavegan_config.json | 149 ----------------- .../configs/universal_fullband_melgan.json | 145 ---------------- TTS/vocoder/configs/wavegrad_libritts.json | 118 ------------- TTS/vocoder/configs/wavernn_config.json | 103 ------------ 6 files changed, 821 deletions(-) delete mode 100644 TTS/vocoder/configs/multiband-melgan_and_rwd_config.json delete mode 100644 TTS/vocoder/configs/multiband_melgan_config.json delete mode 100644 TTS/vocoder/configs/parallel_wavegan_config.json delete mode 100644 TTS/vocoder/configs/universal_fullband_melgan.json delete mode 100644 TTS/vocoder/configs/wavegrad_libritts.json delete mode 100644 TTS/vocoder/configs/wavernn_config.json diff --git a/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json b/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json deleted file mode 100644 index d52893f0..00000000 --- a/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "run_name": "multiband-melgan-rwd", - "run_description": "multiband melgan with random window discriminator from https://arxiv.org/pdf/1909.11646.pdf", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "num_freq": 513, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - - // MODEL PARAMETERS - "use_pqmf": true, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": true, - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - // loss weights - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - // multiscale stft loss parameters - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - - // subband multiscale stft loss parameters - "subband_stft_loss_params":{ - "n_ffts": [384, 683, 171], - "hop_lengths": [30, 60, 10], - "win_lengths": [150, 300, 60] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "random_window_discriminator", - "discriminator_model_params":{ - "uncond_disc_donwsample_factors": [8, 4], - "cond_disc_downsample_factors": [[8, 4, 2, 2, 2], [8, 4, 2, 2], [8, 4, 2], [8, 4], [4, 2, 2]], - "cond_disc_out_channels": [[128, 128, 256, 256], [128, 256, 256], [128, 256], [256], [128, 256]], - "window_sizes": [512, 1024, 2048, 4096, 8192] - }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - - // GENERATOR - "generator_model": "multiband_melgan_generator", - "generator_model_params": { - "upsample_factors":[8, 4, 2], - "num_res_blocks": 4 - }, - - // DATASET - "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", - "seq_len": 16384, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "warmup_steps_disc": 4000, - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/TTS/vocoder/configs/multiband_melgan_config.json b/TTS/vocoder/configs/multiband_melgan_config.json deleted file mode 100644 index 5aea4a61..00000000 --- a/TTS/vocoder/configs/multiband_melgan_config.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "run_name": "multiband-melgan", - "run_description": "multiband melgan mean-var scaling", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": true, // use only with multi-band models. - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - // loss weights - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - // multiscale stft loss parameters - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - - // subband multiscale stft loss parameters - "subband_stft_loss_params":{ - "n_ffts": [384, 683, 171], - "hop_lengths": [30, 60, 10], - "win_lengths": [150, 300, 60] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "melgan_multiscale_discriminator", - "discriminator_model_params":{ - "base_channels": 16, - "max_channels":512, - "downsample_factors":[4, 4, 4] - }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - - // GENERATOR - "generator_model": "multiband_melgan_generator", - "generator_model_params": { - "upsample_factors":[8, 4, 2], - "num_res_blocks": 4 - }, - - // DATASET - "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", - "feature_path": null, - "seq_len": 16384, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/TTS/vocoder/configs/parallel_wavegan_config.json b/TTS/vocoder/configs/parallel_wavegan_config.json deleted file mode 100644 index 5ea7dbcd..00000000 --- a/TTS/vocoder/configs/parallel_wavegan_config.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "run_name": "pwgan", - "run_description": "parallel-wavegan training", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - - // MODEL PARAMETERS - "use_pqmf": true, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": false, // USE ONLY WITH MULTIBAND MODELS - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - // loss weights - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - // multiscale stft loss parameters - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - - // subband multiscale stft loss parameters - "subband_stft_loss_params":{ - "n_ffts": [384, 683, 171], - "hop_lengths": [30, 60, 10], - "win_lengths": [150, 300, 60] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "parallel_wavegan_discriminator", - "discriminator_model_params":{ - "num_layers": 10 - }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - - // GENERATOR - "generator_model": "parallel_wavegan_generator", - "generator_model_params": { - "upsample_factors":[4, 4, 4, 4], - "stacks": 3, - "num_res_blocks": 30 - }, - - // DATASET - "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", - "feature_path": null, - "seq_len": 25600, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 6, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/TTS/vocoder/configs/universal_fullband_melgan.json b/TTS/vocoder/configs/universal_fullband_melgan.json deleted file mode 100644 index 245de2d3..00000000 --- a/TTS/vocoder/configs/universal_fullband_melgan.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "run_name": "fullband-melgan", - "run_description": "fullband melgan mean-var scaling", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 24000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/libritts/LibriTTS/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54324" - }, - - // MODEL PARAMETERS - "use_pqmf": false, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": false, - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - // loss weights - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - // multiscale stft loss parameters - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "melgan_multiscale_discriminator", - "discriminator_model_params":{ - "base_channels": 16, - "max_channels":512, - "downsample_factors":[4, 4, 4] - }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - - // GENERATOR - "generator_model": "fullband_melgan_generator", - "generator_model_params": { - "upsample_factors":[8, 8, 4], - "num_res_blocks": 4 - }, - - // DATASET - "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/", - "feature_path": null, - "seq_len": 16384, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 48, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/" -} - - diff --git a/TTS/vocoder/configs/wavegrad_libritts.json b/TTS/vocoder/configs/wavegrad_libritts.json deleted file mode 100644 index ade20a8f..00000000 --- a/TTS/vocoder/configs/wavegrad_libritts.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "run_name": "wavegrad-libritts", - "run_description": "wavegrad libritts", - - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 24000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/libritts/LibriTTS/scale_stats_wavegrad.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - "mixed_precision": true, // enable torch mixed precision training (true, false) - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54322" - }, - - "target_loss": "avg_wavegrad_loss", // loss value to pick the best model to save after each epoch - - // MODEL PARAMETERS - "generator_model": "wavegrad", - "model_params":{ - "use_weight_norm": true, - "y_conv_channels":32, - "x_conv_channels":768, - "ublock_out_channels": [512, 512, 256, 128, 128], - "dblock_out_channels": [128, 128, 256, 512], - "upsample_factors": [4, 4, 4, 2, 2], - "upsample_dilations": [ - [1, 2, 1, 2], - [1, 2, 1, 2], - [1, 2, 4, 8], - [1, 2, 4, 8], - [1, 2, 4, 8]] - }, - - // DATASET - "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/", // root data path. It finds all wav files recursively from there. - "feature_path": null, // if you use precomputed features - "seq_len": 6144, // 24 * hop_length - "pad_short": 0, // additional padding for short wavs - "conv_pad": 0, // additional padding against convolutions applied to spectrograms - "use_noise_augment": false, // add noise to the audio signal for augmentation - "use_cache": false, // use in memory cache to keep the computed features. This might cause OOM. - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 96, // Batch size for training. - - // NOISE SCHEDULE PARAMS - Only effective at training time. - "train_noise_schedule":{ - "min_val": 1e-6, - "max_val": 1e-2, - "num_steps": 1000 - }, - "test_noise_schedule":{ - "min_val": 1e-6, - "max_val": 1e-2, - "num_steps": 50 - }, - - // VALIDATION - "run_eval": true, // enable/disable evaluation run - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "clip_grad": 1.0, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "lr_scheduler": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_params": { - "gamma": 0.5, - "milestones": [100000, 200000, 300000, 400000, 500000, 600000] - }, - "lr": 1e-4, // Initial learning rate. If Noam decay is active, maximum learning rate. - - // TENSORBOARD and LOGGING - "print_step": 50, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 5000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 256, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/TTS/vocoder/configs/wavernn_config.json b/TTS/vocoder/configs/wavernn_config.json deleted file mode 100644 index aa2d7b9f..00000000 --- a/TTS/vocoder/configs/wavernn_config.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "run_name": "wavernn_librittts", - "run_description": "wavernn libritts training from LJSpeech model", - -// AUDIO PARAMETERS - "audio": { - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - // Audio processing parameters - "sample_rate": 24000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - // Silence trimming - "do_trim_silence": false, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 40.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 20.0, // scaler value appplied after log transform of spectrogram. - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - -// Generating / Synthesizing - "batched": true, - "target_samples": 11000, // target number of samples to be generated in each batch entry - "overlap_samples": 550, // number of samples for crossfading between batches - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - -// MODEL MODE - "mode": "mold", // mold [string], gauss [string], bits [int] - "mulaw": true, // apply mulaw if mode is bits - -// MODEL PARAMETERS - "wavernn_model_params": { - "rnn_dims": 512, - "fc_dims": 512, - "compute_dims": 128, - "res_out_dims": 128, - "num_res_blocks": 10, - "use_aux_net": true, - "use_upsample_net": true, - "upsample_factors": [4, 8, 8] // this needs to correctly factorise hop_length - }, - -// GENERATOR - for backward compatibility - "generator_model": "WaveRNN", - -// DATASET - //"use_gta": true, // use computed gta features from the tts model - "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/", // path containing training wav files - "feature_path": null, // path containing computed features from wav files if null compute them - "seq_len": 1280, // has to be devideable by hop_length - "padding": 2, // pad the input for resnet to see wider input length - -// TRAINING - "batch_size": 256, // Batch size for training. - "epochs": 10000, // total number of epochs to train. - "mixed_precision": true, // enable/ disable mixed precision training - -// VALIDATION - "run_eval": true, - "test_every_epochs": 10, // Test after set number of epochs (Test every 10 epochs for example) - -// OPTIMIZER - "grad_clip": 4, // apply gradient clipping if > 0 - "lr_scheduler": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_params": { - "gamma": 0.5, - "milestones": [200000, 400000, 600000] - }, - "lr": 1e-4, // initial learning rate - -// TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - -// DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 50, // number of samples for testing - -// PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} From 3dec62b18386e00d6606cc7fcc773bbe9ca7c279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 15:33:52 +0200 Subject: [PATCH 42/87] add Coqpits for the vocoder models --- TTS/vocoder/configs/__init__.py | 17 ++++ TTS/vocoder/configs/fullband_melgan_config.py | 54 +++++++++++ TTS/vocoder/configs/hifigan_config.py | 53 +++++++++++ TTS/vocoder/configs/melgan_config.py | 54 +++++++++++ .../configs/multiband_melgan_config.py | 79 ++++++++++++++++ .../configs/parallel_wavegan_config.py | 73 +++++++++++++++ TTS/vocoder/configs/shared_configs.py | 92 +++++++++++++++++++ TTS/vocoder/configs/wavegrad_config.py | 58 ++++++++++++ TTS/vocoder/configs/wavernn_config.py | 52 +++++++++++ 9 files changed, 532 insertions(+) create mode 100644 TTS/vocoder/configs/__init__.py create mode 100644 TTS/vocoder/configs/fullband_melgan_config.py create mode 100644 TTS/vocoder/configs/hifigan_config.py create mode 100644 TTS/vocoder/configs/melgan_config.py create mode 100644 TTS/vocoder/configs/multiband_melgan_config.py create mode 100644 TTS/vocoder/configs/parallel_wavegan_config.py create mode 100644 TTS/vocoder/configs/shared_configs.py create mode 100644 TTS/vocoder/configs/wavegrad_config.py create mode 100644 TTS/vocoder/configs/wavernn_config.py diff --git a/TTS/vocoder/configs/__init__.py b/TTS/vocoder/configs/__init__.py new file mode 100644 index 00000000..b5e11b99 --- /dev/null +++ b/TTS/vocoder/configs/__init__.py @@ -0,0 +1,17 @@ +import importlib +import os +from inspect import isclass + +# import all files under configs/ +configs_dir = os.path.dirname(__file__) +for file in os.listdir(configs_dir): + path = os.path.join(configs_dir, file) + if not file.startswith("_") and not file.startswith(".") and (file.endswith(".py") or os.path.isdir(path)): + config_name = file[: file.find(".py")] if file.endswith(".py") else file + module = importlib.import_module("TTS.vocoder.configs." + config_name) + for attribute_name in dir(module): + attribute = getattr(module, attribute_name) + + if isclass(attribute): + # Add the class to this package's variables + globals()[attribute_name] = attribute diff --git a/TTS/vocoder/configs/fullband_melgan_config.py b/TTS/vocoder/configs/fullband_melgan_config.py new file mode 100644 index 00000000..9698d36d --- /dev/null +++ b/TTS/vocoder/configs/fullband_melgan_config.py @@ -0,0 +1,54 @@ +from dataclasses import asdict, dataclass, field + +from .shared_configs import BaseGANVocoderConfig + + +@dataclass +class FullbandMelganConfig(BaseGANVocoderConfig): + """Defines parameters for FullbandMelGAN vocoder.""" + model: str = "melgan" + + # Model specific params + discriminator_model: str = "melgan_multiscale_discriminator" + discriminator_model_params: dict = field( + default_factory=lambda: { + "base_channels": 16, + "max_channels": 512, + "downsample_factors": [4, 4, 4] + }) + generator_model: str = "melgan_generator" + generator_model_params: dict = field( + default_factory=lambda: { + "upsample_factors": [8, 8, 2, 2], + "num_res_blocks": 4 + }) + + # Training - overrides + batch_size: int = 16 + seq_len: int = 8192 + pad_short: int = 2000 + use_noise_augment: bool = True + use_cache: bool = True + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = True + use_subband_stft_loss: bool = False + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = True # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = False + + stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }) + + # loss weights - overrides + stft_loss_weight: float = 0.5 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 2.5 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 108 + l1_spec_loss_weight: float = 0 diff --git a/TTS/vocoder/configs/hifigan_config.py b/TTS/vocoder/configs/hifigan_config.py new file mode 100644 index 00000000..072bd27f --- /dev/null +++ b/TTS/vocoder/configs/hifigan_config.py @@ -0,0 +1,53 @@ +from dataclasses import asdict, dataclass, field + +from .shared_configs import BaseGANVocoderConfig + + +@dataclass +class HifiganConfig(BaseGANVocoderConfig): + """Defines parameters for HifiGAN vocoder.""" + + model: str = "hifigan" + # model specific params + discriminator_model: str = "hifigan_discriminator" + generator_model: str = "hifigan_generator" + generator_model_params: dict = field( + default_factory=lambda: { + "upsample_factors": [8, 8, 2, 2], + "upsample_kernel_sizes": [16, 16, 4, 4], + "upsample_initial_channel": 512, + "resblock_kernel_sizes": [3, 7, 11], + "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]], + "resblock_type": "1" + }) + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = False + use_subband_stft_loss: bool = False + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = True # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = True + + # loss weights - overrides + stft_loss_weight: float = 0 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 1 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 108 + l1_spec_loss_weight: float = 45 + l1_spec_loss_params: dict = field( + default_factory=lambda: { + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None + }) + + # optimizer parameters + lr: float = 1e-4 + wd: float = 1e-6 \ No newline at end of file diff --git a/TTS/vocoder/configs/melgan_config.py b/TTS/vocoder/configs/melgan_config.py new file mode 100644 index 00000000..f000be6a --- /dev/null +++ b/TTS/vocoder/configs/melgan_config.py @@ -0,0 +1,54 @@ +from dataclasses import asdict, dataclass, field + +from .shared_configs import BaseGANVocoderConfig + + +@dataclass +class MelganConfig(BaseGANVocoderConfig): + """Defines parameters for MelGAN vocoder.""" + model: str = "melgan" + + # Model specific params + discriminator_model: str = "melgan_multiscale_discriminator" + discriminator_model_params: dict = field( + default_factory=lambda: { + "base_channels": 16, + "max_channels": 1024, + "downsample_factors": [4, 4, 4, 4] + }) + generator_model: str = "melgan_generator" + generator_model_params: dict = field( + default_factory=lambda: { + "upsample_factors": [8, 8, 2, 2], + "num_res_blocks": 3 + }) + + # Training - overrides + batch_size: int = 16 + seq_len: int = 8192 + pad_short: int = 2000 + use_noise_augment: bool = True + use_cache: bool = True + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = True + use_subband_stft_loss: bool = False + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = True # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = False + + stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }) + + # loss weights - overrides + stft_loss_weight: float = 0.5 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 2.5 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 108 + l1_spec_loss_weight: float = 0 diff --git a/TTS/vocoder/configs/multiband_melgan_config.py b/TTS/vocoder/configs/multiband_melgan_config.py new file mode 100644 index 00000000..70745b5c --- /dev/null +++ b/TTS/vocoder/configs/multiband_melgan_config.py @@ -0,0 +1,79 @@ +from dataclasses import asdict, dataclass, field + +from .shared_configs import BaseGANVocoderConfig + + +@dataclass +class MultibandMelganConfig(BaseGANVocoderConfig): + """Defines parameters for MultiBandMelGAN vocoder.""" + model: str = "multiband_melgan" + + # Model specific params + discriminator_model: str = "melgan_multiscale_discriminator" + discriminator_model_params: dict = field( + default_factory=lambda: { + "base_channels": 16, + "max_channels": 512, + "downsample_factors": [4, 4, 4] + }) + generator_model: str = "multiband_melgan_generator" + generator_model_params: dict = field( + default_factory=lambda: { + "upsample_factors": [8, 4, 2], + "num_res_blocks": 4 + }) + use_pqmf: bool = True + + # optimizer - overrides + lr_gen: float = 0.0001 # Initial learning rate. + lr_disc: float = 0.0001 # Initial learning rate. + optimizer: str = "AdamW" + optimizer_params: dict = field(default_factory=lambda: { + "betas": [0.8, 0.99], + "weight_decay": 0.0 + }) + lr_scheduler_gen: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_gen_params: dict = field(default_factory=lambda: { + "gamma": 0.5, + "milestones": [100000, 200000, 300000, 400000, 500000, 600000] + }) + lr_scheduler_disc: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_disc_params: dict = field(default_factory=lambda: { + "gamma": 0.5, + "milestones": [100000, 200000, 300000, 400000, 500000, 600000] + }) + + # Training - overrides + batch_size: int = 64 + seq_len: int = 16384 + pad_short: int = 2000 + use_noise_augment: bool = False + use_cache: bool = True + steps_to_start_discriminator: bool = 200000 + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = True + use_subband_stft_loss: bool = True + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = False # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = False + + subband_stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [384, 683, 171], + "hop_lengths": [30, 60, 10], + "win_lengths": [150, 300, 60] + }) + + # loss weights - overrides + stft_loss_weight: float = 0.5 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 2.5 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 108 + l1_spec_loss_weight: float = 0 + + # optimizer parameters + lr: float = 1e-4 + wd: float = 1e-6 diff --git a/TTS/vocoder/configs/parallel_wavegan_config.py b/TTS/vocoder/configs/parallel_wavegan_config.py new file mode 100644 index 00000000..28d8107f --- /dev/null +++ b/TTS/vocoder/configs/parallel_wavegan_config.py @@ -0,0 +1,73 @@ +from dataclasses import asdict, dataclass, field + +from .shared_configs import BaseGANVocoderConfig + + +@dataclass +class ParallelWaveganConfig(BaseGANVocoderConfig): + """Defines parameters for ParallelWavegan vocoder.""" + model: str = "parallel_wavegan" + + # Model specific params + discriminator_model: str = "parallel_wavegan_discriminator" + discriminator_model_params: dict = field( + default_factory=lambda: { + "num_layers": 10 + }) + generator_model: str = "parallel_wavegan_generator" + generator_model_params: dict = field( + default_factory=lambda: { + "upsample_factors":[4, 4, 4, 4], + "stacks": 3, + "num_res_blocks": 30 + }) + + # Training - overrides + batch_size: int = 6 + seq_len: int = 25600 + pad_short: int = 2000 + use_noise_augment: bool = False + use_cache: bool = True + steps_to_start_discriminator: int = 200000 + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = True + use_subband_stft_loss: bool = False + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = False # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = False + + stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }) + + # loss weights - overrides + stft_loss_weight: float = 0.5 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 2.5 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 0 + l1_spec_loss_weight: float = 0 + + # optimizer overrides + lr_gen: float = 0.0002 # Initial learning rate. + lr_disc: float = 0.0002 # Initial learning rate. + optimizer: str = "AdamW" + optimizer_params: dict = field(default_factory=lambda: { + "betas": [0.8, 0.99], + "weight_decay": 0.0 + }) + lr_scheduler_gen: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_gen_params: dict = field(default_factory=lambda: { + "gamma": 0.999, + "last_epoch": -1 + }) + lr_scheduler_disc: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_disc_params: dict = field(default_factory=lambda: { + "gamma": 0.999, + "last_epoch": -1 + }) diff --git a/TTS/vocoder/configs/shared_configs.py b/TTS/vocoder/configs/shared_configs.py new file mode 100644 index 00000000..0d64c622 --- /dev/null +++ b/TTS/vocoder/configs/shared_configs.py @@ -0,0 +1,92 @@ +from dataclasses import dataclass, field +from typing import List + +from coqpit import MISSING + +from TTS.config import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig + + +@dataclass +class BaseVocoderConfig(BaseTrainingConfig): + """Shared parameters among all the vocoder models.""" + audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) + # dataloading + use_noise_augment: bool = False # enable/disable random noise augmentation in spectrograms. + eval_split_size: int = 10 # number of samples used for evaluation. + # dataset + data_path: str = MISSING # root data path. It finds all wav files recursively from there. + feature_path: str = None # if you use precomputed features + seq_len: int = MISSING # signal length used in training. + pad_short: int = 0 # additional padding for short wavs + conv_pad: int = 0 # additional padding against convolutions applied to spectrograms + use_noise_augment: bool = False # add noise to the audio signal for augmentation + use_cache: bool = False # use in memory cache to keep the computed features. This might cause OOM. + # OPTIMIZER + epochs: int = 10000 # total number of epochs to train. + wd: float = 0.0 # Weight decay weight. + + +@dataclass +class BaseGANVocoderConfig(BaseVocoderConfig): + """Common config interface for all the GAN based vocoder models.""" + # LOSS PARAMETERS + use_stft_loss: bool = True + use_subband_stft_loss: bool = True + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = True + use_feat_match_loss: bool = True # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = True + + # loss weights + stft_loss_weight: float = 0 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 1 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 10 + l1_spec_loss_weight: float = 45 + + stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }) + + l1_spec_loss_params: dict = field( + default_factory=lambda: { + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None + }) + + target_loss: str = "avg_G_loss" # loss value to pick the best model to save after each epoch + + # optimizer + gen_clip_grad: float = -1 # Generator gradient clipping threshold. Apply gradient clipping if > 0 + disc_clip_grad: float = -1 # Discriminator gradient clipping threshold. + lr_gen: float = 0.0002 # Initial learning rate. + lr_disc: float = 0.0002 # Initial learning rate. + optimizer: str = "AdamW" + optimizer_params: dict = field(default_factory=lambda: { + "betas": [0.8, 0.99], + "weight_decay": 0.0 + }) + lr_scheduler_gen: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_gen_params: dict = field(default_factory=lambda: { + "gamma": 0.999, + "last_epoch": -1 + }) + lr_scheduler_disc: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_disc_params: dict = field(default_factory=lambda: { + "gamma": 0.999, + "last_epoch": -1 + }) + + use_pqmf: bool = False # enable/disable using pqmf for multi-band training. (Multi-band MelGAN) + steps_to_start_discriminator = 0 # start training the discriminator after this number of steps. + diff_samples_for_G_and_D: bool = False # use different samples for G and D training steps. diff --git a/TTS/vocoder/configs/wavegrad_config.py b/TTS/vocoder/configs/wavegrad_config.py new file mode 100644 index 00000000..46ff5290 --- /dev/null +++ b/TTS/vocoder/configs/wavegrad_config.py @@ -0,0 +1,58 @@ +from dataclasses import asdict, dataclass, field + +from .shared_configs import BaseVocoderConfig + + +@dataclass +class WavegradConfig(BaseVocoderConfig): + """Defines parameters for Wavernn vocoder.""" + model: str = 'wavegrad' + # Model specific params + generator_model: str = "wavegrad" + model_params: dict = field( + default_factory=lambda: { + "use_weight_norm": + True, + "y_conv_channels": + 32, + "x_conv_channels": + 768, + "ublock_out_channels": [512, 512, 256, 128, 128], + "dblock_out_channels": [128, 128, 256, 512], + "upsample_factors": [4, 4, 4, 2, 2], + "upsample_dilations": [[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8], + [1, 2, 4, 8], [1, 2, 4, 8]] + }) + target_loss: str = 'avg_wavegrad_loss' # loss value to pick the best model to save after each epoch + + # Training - overrides + epochs: int = 10000 + batch_size: int = 96 + seq_len: int = 6144 + use_cache: bool = True + steps_to_start_discriminator: int = 200000 + mixed_precision: bool = True + eval_split_size: int = 50 + + # NOISE SCHEDULE PARAMS + train_noise_schedule: dict = field(default_factory=lambda: { + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 1000 + }) + + test_noise_schedule: dict = field(default_factory=lambda: { # inference noise schedule. Try TTS/bin/tune_wavegrad.py to find the optimal values. + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 50 + }) + + # optimizer overrides + grad_clip: float = 1.0 + lr: float = 1e-4 # Initial learning rate. + lr_scheduler: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_params: dict = field( + default_factory=lambda: { + "gamma": 0.5, + "milestones": [100000, 200000, 300000, 400000, 500000, 600000] + }) diff --git a/TTS/vocoder/configs/wavernn_config.py b/TTS/vocoder/configs/wavernn_config.py new file mode 100644 index 00000000..636b0b23 --- /dev/null +++ b/TTS/vocoder/configs/wavernn_config.py @@ -0,0 +1,52 @@ +from dataclasses import asdict, dataclass, field + +from .shared_configs import BaseVocoderConfig + + +@dataclass +class WavernnConfig(BaseVocoderConfig): + """Defines parameters for Wavernn vocoder.""" + model: str = "wavernn" + + # Model specific params + mode: str = 'mold' # mold [string], gauss [string], bits [int] + mulaw: bool = True # apply mulaw if mode is bits + generator_model: str = "WaveRNN" + wavernn_model_params: dict = field( + default_factory=lambda: { + "rnn_dims": 512, + "fc_dims": 512, + "compute_dims": 128, + "res_out_dims": 128, + "num_res_blocks": 10, + "use_aux_net": True, + "use_upsample_net": True, + "upsample_factors": + [4, 8, 8] # this needs to correctly factorise hop_length + }) + + # Inference + batched: bool = True + target_samples: int = 11000 + overlap_samples: int = 550 + + # Training - overrides + epochs: int = 10000 + batch_size: int = 256 + seq_len: int = 1280 + padding: int = 2 + use_noise_augment: bool = False + use_cache: bool = True + steps_to_start_discriminator: int = 200000 + mixed_precision: bool = True + eval_split_size: int = 50 + test_every_epochs: int = 10 # number of epochs to wait until the next test run (synthesizing a full audio clip). + + # optimizer overrides + grad_clip: float = 4.0 + lr: float = 1e-4 # Initial learning rate. + lr_scheduler: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_params: dict = field(default_factory=lambda: { + "gamma": 0.5, + "milestones": [200000, 400000, 600000] + }) From 1be45eae38d3170e2e200be1dbebc3e083dfdfbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 15:34:52 +0200 Subject: [PATCH 43/87] add new tests for vocoder trainings using coqpit --- tests/vocoder_tests/__init__.py | 0 .../test_fullband_melgan_train.py | 42 +++++++++++++++++++ tests/vocoder_tests/test_melgan_train.py | 42 +++++++++++++++++++ .../test_multiband_melgan_train.py | 42 +++++++++++++++++++ .../test_parallel_wavegan_train.py | 42 +++++++++++++++++++ tests/vocoder_tests/test_wavegrad_train.py | 42 +++++++++++++++++++ tests/vocoder_tests/test_wavernn_train.py | 42 +++++++++++++++++++ 7 files changed, 252 insertions(+) create mode 100644 tests/vocoder_tests/__init__.py create mode 100644 tests/vocoder_tests/test_fullband_melgan_train.py create mode 100644 tests/vocoder_tests/test_melgan_train.py create mode 100644 tests/vocoder_tests/test_multiband_melgan_train.py create mode 100644 tests/vocoder_tests/test_parallel_wavegan_train.py create mode 100644 tests/vocoder_tests/test_wavegrad_train.py create mode 100644 tests/vocoder_tests/test_wavernn_train.py diff --git a/tests/vocoder_tests/__init__.py b/tests/vocoder_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/vocoder_tests/test_fullband_melgan_train.py b/tests/vocoder_tests/test_fullband_melgan_train.py new file mode 100644 index 00000000..358552c4 --- /dev/null +++ b/tests/vocoder_tests/test_fullband_melgan_train.py @@ -0,0 +1,42 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.vocoder.configs import FullbandMelganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = FullbandMelganConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_melgan_train.py b/tests/vocoder_tests/test_melgan_train.py new file mode 100644 index 00000000..65b7346a --- /dev/null +++ b/tests/vocoder_tests/test_melgan_train.py @@ -0,0 +1,42 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.vocoder.configs import MelganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = MelganConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_multiband_melgan_train.py b/tests/vocoder_tests/test_multiband_melgan_train.py new file mode 100644 index 00000000..8ededcce --- /dev/null +++ b/tests/vocoder_tests/test_multiband_melgan_train.py @@ -0,0 +1,42 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.vocoder.configs import MultibandMelganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = MultibandMelganConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_parallel_wavegan_train.py b/tests/vocoder_tests/test_parallel_wavegan_train.py new file mode 100644 index 00000000..a2edd0c5 --- /dev/null +++ b/tests/vocoder_tests/test_parallel_wavegan_train.py @@ -0,0 +1,42 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.vocoder.configs import ParallelWaveganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = ParallelWaveganConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_wavegrad_train.py b/tests/vocoder_tests/test_wavegrad_train.py new file mode 100644 index 00000000..ffa450b8 --- /dev/null +++ b/tests/vocoder_tests/test_wavegrad_train.py @@ -0,0 +1,42 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.vocoder.configs import WavegradConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = WavegradConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavegrad.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavegrad.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_wavernn_train.py b/tests/vocoder_tests/test_wavernn_train.py new file mode 100644 index 00000000..33fc4e57 --- /dev/null +++ b/tests/vocoder_tests/test_wavernn_train.py @@ -0,0 +1,42 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.vocoder.configs import WavernnConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = WavernnConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavernn.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavernn.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) From 70fc7a7e717377ccb317e70cd8bdc7f3b379c1b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 15:35:35 +0200 Subject: [PATCH 44/87] remove bash running tests --- tests/bash_tests/test_vocoder_gan_train.sh | 15 --------------- tests/bash_tests/test_vocoder_wavegrad_train.sh | 15 --------------- tests/bash_tests/test_vocoder_wavernn_train.sh | 15 --------------- 3 files changed, 45 deletions(-) delete mode 100755 tests/bash_tests/test_vocoder_gan_train.sh delete mode 100755 tests/bash_tests/test_vocoder_wavegrad_train.sh delete mode 100755 tests/bash_tests/test_vocoder_wavernn_train.sh diff --git a/tests/bash_tests/test_vocoder_gan_train.sh b/tests/bash_tests/test_vocoder_gan_train.sh deleted file mode 100755 index b2f43721..00000000 --- a/tests/bash_tests/test_vocoder_gan_train.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# create run dir -mkdir $BASEDIR/../train_outputs -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_gan.py --config_path $BASEDIR/../inputs/test_vocoder_multiband_melgan_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_gan.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER diff --git a/tests/bash_tests/test_vocoder_wavegrad_train.sh b/tests/bash_tests/test_vocoder_wavegrad_train.sh deleted file mode 100755 index 9626187f..00000000 --- a/tests/bash_tests/test_vocoder_wavegrad_train.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# create run dir -mkdir -p $BASEDIR/../train_outputs -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavegrad.py --config_path $BASEDIR/../inputs/test_vocoder_wavegrad.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavegrad.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER \ No newline at end of file diff --git a/tests/bash_tests/test_vocoder_wavernn_train.sh b/tests/bash_tests/test_vocoder_wavernn_train.sh deleted file mode 100755 index 7b554fc9..00000000 --- a/tests/bash_tests/test_vocoder_wavernn_train.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# create run dir -mkdir -p $BASEDIR/../train_outputs -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavernn.py --config_path $BASEDIR/../inputs/test_vocoder_wavernn_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavernn.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER \ No newline at end of file From 10db2baa06728bcc7b5b8e43038a6cc9bb49b364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 15:36:14 +0200 Subject: [PATCH 45/87] global shared Coqpit configs --- TTS/config/shared_configs.py | 258 +++++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 TTS/config/shared_configs.py diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py new file mode 100644 index 00000000..9bd18ff0 --- /dev/null +++ b/TTS/config/shared_configs.py @@ -0,0 +1,258 @@ +from dataclasses import asdict, dataclass +from typing import List, Union + +from coqpit import MISSING, Coqpit, check_argument + + +@dataclass +class BaseAudioConfig(Coqpit): + """Base config to definge audio processing parameters. It is used to initialize + ```TTS.utils.audio.AudioProcessor.``` + + Args: + fft_size (int): + Number of STFT frequency levels aka.size of the linear spectogram frame. Defaults to 1024. + win_length (int): + Each frame of audio is windowed by window of length ```win_length``` and then padded with zeros to match + ```fft_size```. Defaults to 256. + hop_length (int): + Number of audio samples between adjacent STFT columns. Defaults to 1024. + frame_shift_ms (int): + Set ```hop_length``` based on milliseconds and sampling rate. + frame_length_ms (int): + Set ```win_length``` based on milliseconds and sampling rate. + stft_pad_mode (str): + Padding method used in STFT. 'reflect' or 'center'. + sample_rate (int): + Audio sampling rate. Defaults to 22050. + resample (bool): + Enable / Disable resampling audio to ```sample_rate```. Defaults to ```False```. + preemphasis (float): + Preemphasis coefficient. Defaults to 0.0. + ref_level_db (int): 20 + Reference Db level to rebase the audio signal and ignore the level below. 20Db is assumed the sound of air. + Defaults to 20. + do_sound_norm (bool): + Enable / Disable sound normalization to reconcile the volume differences among samples. Defaults to False. + do_trim_silence (bool): + Enable / Disable trimming silences at the beginning and the end of the audio clip. Defaults to ```True```. + trim_db (int): + Silence threshold used for silence trimming. Defaults to 45. + power (float): + Exponent used for expanding spectrogra levels before running Griffin Lim. It helps to reduce the + artifacts in the synthesized voice. Defaults to 1.5. + griffin_lim_iters (int): + Number of Griffing Lim iterations. Defaults to 60. + num_mels (int): + Number of mel-basis frames that defines the frame lengths of each mel-spectrogram frame. Defaults to 80. + mel_fmin (float): Min frequency level used for the mel-basis filters. ~50 for male and ~95 for female voices. + It needs to be adjusted for a dataset. Defaults to 0. + mel_fmax (float): + Max frequency level used for the mel-basis filters. It needs to be adjusted for a dataset. + spec_gain (int): + Gain applied when converting amplitude to DB. Defaults to 20. + signal_norm (bool): + enable/disable signal normalization. Defaults to True. + min_level_db (int): + minimum db threshold for the computed melspectrograms. Defaults to -100. + symmetric_norm (bool): + enable/disable symmetric normalization. If set True normalization is performed in the range [-k, k] else + [0, k], Defaults to True. + max_norm (float): + ```k``` defining the normalization range. Defaults to 4.0. + clip_norm (bool): + enable/disable clipping the our of range values in the normalized audio signal. Defaults to True. + stats_path (str): + Path to the computed stats file. Defaults to None. + """ + + # stft parameters + fft_size: int = 1024 + win_length: int = 1024 + hop_length: int = 256 + frame_shift_ms: int = None + frame_length_ms: int = None + stft_pad_mode: str = "reflect" + # audio processing parameters + sample_rate: int = 22050 + resample: bool = False + preemphasis: float = 0.0 + ref_level_db: int = 20 + do_sound_norm: bool = False + log_func = "np.log10" + # silence trimming + do_trim_silence: bool = True + trim_db: int = 45 + # griffin-lim params + power: float = 1.5 + griffin_lim_iters: int = 60 + # mel-spec params + num_mels: int = 80 + mel_fmin: float = 0.0 + mel_fmax: float = None + spec_gain: int = 20 + # normalization params + signal_norm: bool = True + min_level_db: int = -100 + symmetric_norm: bool = True + max_norm: float = 4.0 + clip_norm: bool = True + stats_path: str = None + + def check_values( + self, + ): + """Check config fields""" + c = asdict(self) + check_argument("num_mels", c, restricted=True, min_val=10, max_val=2056) + check_argument("fft_size", c, restricted=True, min_val=128, max_val=4058) + check_argument("sample_rate", c, restricted=True, min_val=512, max_val=100000) + check_argument( + "frame_length_ms", + c, + restricted=True, + min_val=10, + max_val=1000, + alternative="win_length", + ) + check_argument("frame_shift_ms", c, restricted=True, min_val=1, max_val=1000, alternative="hop_length") + check_argument("preemphasis", c, restricted=True, min_val=0, max_val=1) + check_argument("min_level_db", c, restricted=True, min_val=-1000, max_val=10) + check_argument("ref_level_db", c, restricted=True, min_val=0, max_val=1000) + check_argument("power", c, restricted=True, min_val=1, max_val=5) + check_argument("griffin_lim_iters", c, restricted=True, min_val=10, max_val=1000) + + # normalization parameters + check_argument("signal_norm", c, restricted=True) + check_argument("symmetric_norm", c, restricted=True) + check_argument("max_norm", c, restricted=True, min_val=0.1, max_val=1000) + check_argument("clip_norm", c, restricted=True) + check_argument("mel_fmin", c, restricted=True, min_val=0.0, max_val=1000) + check_argument("mel_fmax", c, restricted=True, min_val=500.0, allow_none=True) + check_argument("spec_gain", c, restricted=True, min_val=1, max_val=100) + check_argument("do_trim_silence", c, restricted=True) + check_argument("trim_db", c, restricted=True) + + +@dataclass +class BaseDatasetConfig(Coqpit): + name: str = None + path: str = None + meta_file_train: str = None + meta_file_val: str = None + meta_file_attn_mask: str = None + + def check_values( + self, + ): + """Check config fields""" + c = asdict(self) + check_argument("name", c, restricted=True) + check_argument("path", c, restricted=True) + check_argument("meta_file_train", c, restricted=True) + check_argument("meta_file_val", c, restricted=False) + check_argument("meta_file_attn_mask", c, restricted=False) + + +@dataclass +class BaseTrainingConfig(Coqpit): + """Base config to define the basic training parameters that are shared + among all the models. + + Args: + batch_size (int): + Training batch size. + batch_group_size (int): + Number of batches to shuffle after bucketing. + eval_batch_size (int): + Validation batch size. + loss_masking (bool): + Enable / Disable masking padding segments of sequences. + mixed_precision (bool): + Enable / Disable mixed precision training. It reduces the VRAM use and allows larger batch sizes, however + it may also cause numerical unstability in some cases. + run_eval (bool): + Enable / Disable evaluation (validation) run. Defaults to True. + test_delay_epochs (int): + Number of epochs before starting to use evaluation runs. Initially, models do not generate meaningful + results, hence waiting for a couple of epochs might save some time. + print_eval (bool): + Enable / Disable console logging for evalutaion steps. If disabled then it only shows the final values at + the end of the evaluation. Default to ```False```. + print_step (int): + Number of steps required to print the next training log. + tb_plot_step (int): + Number of steps required to log training on Tensorboard. + tb_model_param_stats (bool): + Enable / Disable logging internal model stats for model diagnostic. It might be useful for model debugging. + Defaults to ```False```. + save_step (int):ipt + Number of steps required to save the next checkpoint. + checkpoint (bool): + Enable / Disable checkpointing. + keep_all_best (bool): + Enable / Disable keeping all the saved best models instead of overwriting the previous one. Defaults + to ```False```. + keep_after (int): + Number of steps to wait before saving all the best models. In use if ```keep_all_best == True```. Defaults + to 10000. + text_cleaner (str): + Text cleaner to be used at model training. It is set to be one of the cleaners in + ```TTS.tts.utils.text.cleaners```. + enable_eos_bos_chars (bool): + Enable / Disable using special characters indicating end-of-sentence and begining-of-sentence. + num_loader_workers (int): + Number of workers for training time dataloader. + num_val_loader_workers (int): + Number of workers for evaluation time dataloader. + min_seq_len (int): + Minimum sequence length to be used at training. + max_seq_len (int): + Maximum sequence length to be used at training. VRAM use at training depends on this parameter. Consider to + decrease it if you get OOM errors. + compute_f0 (bool): + Return F0 frames from the dataloader. Defaults to ```False```. + compute_input_seq_cache (bool): + Enable / Disable computing and caching phonemes sequences from character sequences at the begining of the + training. It allows faster data loading times and more precise max-min sequence prunning. Defaults + to ```False```. + output_path (str): + Path for training output folder. The nonexist part of the given path is created automatically. + All training outputs are saved there. + phoneme_cache_path (str): + Path to a folder to save the computed phoneme sequences. + datasets (List[BaseDatasetConfig]): + ist of DatasetConfig. + + """ + + model: str = None + run_name: str = "" + run_description: str = "" + # training params + epochs: int = 10000 + batch_size: int = MISSING + eval_batch_size: int = None + mixed_precision: bool = False + # eval params + run_eval: bool = True + test_delay_epochs: int = 0 + print_eval: bool = False + # logging + print_step: int = 25 + tb_plot_step: int = 100 + tb_model_param_stats: bool = False + # checkpointing + save_step: int = 10000 + checkpoint: bool = True + keep_all_best: bool = False + keep_after: int = 10000 + # dataloading + num_loader_workers: int = None + num_val_loader_workers: int = None + use_noise_augment: bool = False + # paths + output_path: str = None + # distributed + distributed_backend: str = "nccl" + distributed_url: str = "tcp://localhost:54321" From 9ee70af9bb1684f07eb9190c053159e0dc3a5563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 15:39:48 +0200 Subject: [PATCH 46/87] code styling --- TTS/bin/compute_statistics.py | 5 +- TTS/bin/train_align_tts.py | 170 ++++++------------ TTS/config/__init__.py | 3 +- TTS/tts/configs/align_tts_config.py | 16 +- TTS/tts/utils/synthesis.py | 2 +- TTS/utils/arguments.py | 2 +- TTS/vocoder/configs/fullband_melgan_config.py | 19 +- TTS/vocoder/configs/hifigan_config.py | 12 +- TTS/vocoder/configs/melgan_config.py | 19 +- .../configs/multiband_melgan_config.py | 40 ++--- .../configs/parallel_wavegan_config.py | 33 ++-- TTS/vocoder/configs/shared_configs.py | 27 ++- TTS/vocoder/configs/wavegrad_config.py | 44 ++--- TTS/vocoder/configs/wavernn_config.py | 14 +- .../test_fullband_melgan_train.py | 6 +- tests/vocoder_tests/test_hifigan_train.py | 6 +- tests/vocoder_tests/test_melgan_train.py | 6 +- .../test_multiband_melgan_train.py | 6 +- .../test_parallel_wavegan_train.py | 6 +- tests/vocoder_tests/test_wavegrad_train.py | 6 +- tests/vocoder_tests/test_wavernn_train.py | 6 +- 21 files changed, 161 insertions(+), 287 deletions(-) diff --git a/TTS/bin/compute_statistics.py b/TTS/bin/compute_statistics.py index b4ee6df7..2c13a960 100755 --- a/TTS/bin/compute_statistics.py +++ b/TTS/bin/compute_statistics.py @@ -8,11 +8,10 @@ import os import numpy as np from tqdm import tqdm -from TTS.tts.datasets.preprocess import load_meta_data -from TTS.utils.audio import AudioProcessor - # from TTS.utils.io import load_config from TTS.config import load_config +from TTS.tts.datasets.preprocess import load_meta_data +from TTS.utils.audio import AudioProcessor def main(): diff --git a/TTS/bin/train_align_tts.py b/TTS/bin/train_align_tts.py index 206d8b03..7e3921b0 100644 --- a/TTS/bin/train_align_tts.py +++ b/TTS/bin/train_align_tts.py @@ -46,8 +46,7 @@ def setup_loader(ap, r, is_val=False, verbose=False): ap=ap, tp=config.characters, add_blank=config["add_blank"], - batch_group_size=0 if is_val else config.batch_group_size * - config.batch_size, + batch_group_size=0 if is_val else config.batch_group_size * config.batch_size, min_seq_len=config.min_seq_len, max_seq_len=config.max_seq_len, phoneme_cache_path=config.phoneme_cache_path, @@ -56,8 +55,9 @@ def setup_loader(ap, r, is_val=False, verbose=False): enable_eos_bos=config.enable_eos_bos_chars, use_noise_augment=not is_val, verbose=verbose, - speaker_mapping=speaker_mapping if config.use_speaker_embedding - and config.use_external_speaker_embedding_file else None, + speaker_mapping=speaker_mapping + if config.use_speaker_embedding and config.use_external_speaker_embedding_file + else None, ) if config.use_phonemes and config.compute_input_seq_cache: @@ -73,8 +73,7 @@ def setup_loader(ap, r, is_val=False, verbose=False): collate_fn=dataset.collate_fn, drop_last=False, sampler=sampler, - num_workers=config.num_val_loader_workers - if is_val else config.num_loader_workers, + num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers, pin_memory=False, ) return loader @@ -97,9 +96,7 @@ def format_data(data): speaker_c = data[8] else: # return speaker_id to be used by an embedding layer - speaker_c = [ - speaker_mapping[speaker_name] for speaker_name in speaker_names - ] + speaker_c = [speaker_mapping[speaker_name] for speaker_name in speaker_names] speaker_c = torch.LongTensor(speaker_c) else: speaker_c = None @@ -114,15 +111,13 @@ def format_data(data): return text_input, text_lengths, mel_input, mel_lengths, speaker_c, avg_text_length, avg_spec_length, item_idx -def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, - epoch, training_phase): +def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, training_phase): model.train() epoch_time = 0 keep_avg = KeepAverage() if use_cuda: - batch_n_iter = int( - len(data_loader.dataset) / (config.batch_size * num_gpus)) + batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus)) else: batch_n_iter = int(len(data_loader.dataset) / config.batch_size) end_time = time.time() @@ -151,12 +146,8 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, # forward pass model with torch.cuda.amp.autocast(enabled=config.mixed_precision): decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward( - text_input, - text_lengths, - mel_targets, - mel_lengths, - g=speaker_c, - phase=training_phase) + text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase + ) # compute loss loss_dict = criterion( @@ -175,14 +166,12 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, if config.mixed_precision: scaler.scale(loss_dict["loss"]).backward() scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), - config.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) scaler.step(optimizer) scaler.update() else: loss_dict["loss"].backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), - config.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) optimizer.step() # setup lr @@ -201,12 +190,9 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, # aggregate losses from processes if num_gpus > 1: - loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, - num_gpus) - loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, - num_gpus) - loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, - num_gpus) + loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus) + loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, num_gpus) + loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, num_gpus) loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) # detach loss values @@ -235,18 +221,13 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, "loader_time": [loader_time, 2], "current_lr": current_lr, } - c_logger.print_train_step(batch_n_iter, num_iter, global_step, - log_dict, loss_dict, keep_avg.avg_values) + c_logger.print_train_step(batch_n_iter, num_iter, global_step, log_dict, loss_dict, keep_avg.avg_values) if args.rank == 0: # Plot Training Iter Stats # reduce TB load if global_step % config.tb_plot_step == 0: - iter_stats = { - "lr": current_lr, - "grad_norm": grad_norm, - "step_time": step_time - } + iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time} iter_stats.update(loss_dict) tb_logger.tb_train_iter_stats(global_step, iter_stats) @@ -270,8 +251,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, # Diagnostic visualizations if decoder_output is not None: idx = np.random.randint(mel_targets.shape[0]) - pred_spec = decoder_output[idx].detach().data.cpu().numpy( - ).T + pred_spec = decoder_output[idx].detach().data.cpu().numpy().T gt_spec = mel_targets[idx].data.cpu().numpy().T align_img = alignments[idx].data.cpu() @@ -285,9 +265,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, # Sample audio train_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_train_audios(global_step, - {"TrainAudio": train_audio}, - config.audio["sample_rate"]) + tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() # print epoch stats @@ -304,8 +282,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, @torch.no_grad() -def evaluate(data_loader, model, criterion, ap, global_step, epoch, - training_phase): +def evaluate(data_loader, model, criterion, ap, global_step, epoch, training_phase): model.eval() epoch_time = 0 keep_avg = KeepAverage() @@ -315,18 +292,13 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch, start_time = time.time() # format data - text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _ = format_data( - data) + text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _ = format_data(data) # forward pass model with torch.cuda.amp.autocast(enabled=config.mixed_precision): decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward( - text_input, - text_lengths, - mel_targets, - mel_lengths, - g=speaker_c, - phase=training_phase) + text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase + ) # compute loss loss_dict = criterion( @@ -351,14 +323,10 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch, # aggregate losses from processes if num_gpus > 1: - loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, - num_gpus) - loss_dict["loss_ssim"] = reduce_tensor( - loss_dict["loss_ssim"].data, num_gpus) - loss_dict["loss_dur"] = reduce_tensor( - loss_dict["loss_dur"].data, num_gpus) - loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, - num_gpus) + loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus) + loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, num_gpus) + loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, num_gpus) + loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) # detach loss values loss_dict_new = dict() @@ -376,8 +344,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch, keep_avg.update_values(update_train_values) if config.print_eval: - c_logger.print_eval_step(num_iter, loss_dict, - keep_avg.avg_values) + c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) if args.rank == 0: # Diagnostic visualizations @@ -387,17 +354,14 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch, align_img = alignments[idx].data.cpu() eval_figures = { - "prediction": plot_spectrogram(pred_spec, ap, - output_fig=False), - "ground_truth": plot_spectrogram(gt_spec, ap, - output_fig=False), + "prediction": plot_spectrogram(pred_spec, ap, output_fig=False), + "ground_truth": plot_spectrogram(gt_spec, ap, output_fig=False), "alignment": plot_alignment(align_img, output_fig=False), } # Sample audio eval_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, - config.audio["sample_rate"]) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) # Plot Validation Stats tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) @@ -422,9 +386,9 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch, print(" | > Synthesizing test sentences") if config.use_speaker_embedding: if config.use_external_speaker_embedding_file: - speaker_embedding = speaker_mapping[list( - speaker_mapping.keys())[randrange( - len(speaker_mapping) - 1)]]["embedding"] + speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][ + "embedding" + ] speaker_id = None else: speaker_id = 0 @@ -452,19 +416,15 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch, file_path = os.path.join(AUDIO_PATH, str(global_step)) os.makedirs(file_path, exist_ok=True) - file_path = os.path.join(file_path, - "TestSentence_{}.wav".format(idx)) + file_path = os.path.join(file_path, "TestSentence_{}.wav".format(idx)) ap.save_wav(wav, file_path) test_audios["{}-audio".format(idx)] = wav - test_figures["{}-prediction".format(idx)] = plot_spectrogram( - postnet_output, ap) - test_figures["{}-alignment".format(idx)] = plot_alignment( - alignment) + test_figures["{}-prediction".format(idx)] = plot_spectrogram(postnet_output, ap) + test_figures["{}-alignment".format(idx)] = plot_alignment(alignment) except: # pylint: disable=bare-except print(" !! Error creating Test Sentence -", idx) traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, - config.audio["sample_rate"]) + tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"]) tb_logger.tb_test_figures(global_step, test_figures) return keep_avg.avg_values @@ -479,32 +439,21 @@ def main(args): # pylint: disable=redefined-outer-name # DISTRUBUTED if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, - config.distributed["backend"], - config.distributed["url"]) + init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"]) # set model characters model_characters = phonemes if config.use_phonemes else symbols num_chars = len(model_characters) # load data instances - meta_data_train, meta_data_eval = load_meta_data(config.datasets, - eval_split=True) + meta_data_train, meta_data_eval = load_meta_data(config.datasets, eval_split=True) # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers( - config, args, meta_data_train, OUT_PATH) + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) # setup model - model = setup_model(num_chars, - num_speakers, - config, - speaker_embedding_dim=speaker_embedding_dim) - optimizer = RAdam(model.parameters(), - lr=config.lr, - weight_decay=0, - betas=(0.9, 0.98), - eps=1e-9) + model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim) + optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) criterion = AlignTTSLoss(config) if args.restore_path: @@ -526,8 +475,7 @@ def main(args): # pylint: disable=redefined-outer-name for group in optimizer.param_groups: group["initial_lr"] = config.lr - print(" > Model restored from step %d" % checkpoint["step"], - flush=True) + print(" > Model restored from step %d" % checkpoint["step"], flush=True) args.restore_step = checkpoint["step"] else: args.restore_step = 0 @@ -541,9 +489,7 @@ def main(args): # pylint: disable=redefined-outer-name model = DDP_th(model, device_ids=[args.rank]) if config.noam_schedule: - scheduler = NoamLR(optimizer, - warmup_steps=config.warmup_steps, - last_epoch=args.restore_step - 1) + scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1) else: scheduler = None @@ -554,10 +500,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = float("inf") print(" > Starting with inf best loss.") else: - print(" > Restoring best loss from " - f"{os.path.basename(args.best_path)} ...") - best_loss = torch.load(args.best_path, - map_location="cpu")["model_loss"] + print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") + best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] print(f" > Starting with loaded last best loss {best_loss}.") keep_all_best = config.keep_all_best keep_after = config.keep_after # void if keep_all_best False @@ -576,9 +520,10 @@ def main(args): # pylint: disable=redefined-outer-name phase = 0 else: phase = ( - len(config.phase_start_steps) - - [i < global_step - for i in config.phase_start_steps][::-1].index(True) - 1) + len(config.phase_start_steps) + - [i < global_step for i in config.phase_start_steps][::-1].index(True) + - 1 + ) else: phase = None return phase @@ -587,12 +532,10 @@ def main(args): # pylint: disable=redefined-outer-name cur_phase = set_phase() print(f"\n > Current AlignTTS phase: {cur_phase}") c_logger.print_epoch_start(epoch, config.epochs) - train_avg_loss_dict, global_step = train(train_loader, model, - criterion, optimizer, - scheduler, ap, global_step, - epoch, cur_phase) - eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, - global_step, epoch, cur_phase) + train_avg_loss_dict, global_step = train( + train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, cur_phase + ) + eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch, cur_phase) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) target_loss = train_avg_loss_dict["avg_loss"] if config.run_eval: @@ -613,8 +556,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training( - sys.argv) + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) diff --git a/TTS/config/__init__.py b/TTS/config/__init__.py index 85e7d9b9..29ba1190 100644 --- a/TTS/config/__init__.py +++ b/TTS/config/__init__.py @@ -1,10 +1,9 @@ -from TTS.config.shared_configs import * - import json import os import yaml +from TTS.config.shared_configs import * from TTS.utils.generic_utils import find_module diff --git a/TTS/tts/configs/align_tts_config.py b/TTS/tts/configs/align_tts_config.py index fae4c608..6e09e398 100644 --- a/TTS/tts/configs/align_tts_config.py +++ b/TTS/tts/configs/align_tts_config.py @@ -14,20 +14,12 @@ class AlignTTSConfig(BaseTTSConfig): hidden_channels: int = 256 encoder_type: str = "fftransformer" encoder_params: dict = field( - default_factory=lambda: { - "hidden_channels_ffn": 1024, - "num_heads": 2, - "num_layers": 6, - "dropout_p": 0.1 - }) + default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1} + ) decoder_type: str = "fftransformer" decoder_params: dict = field( - default_factory=lambda: { - "hidden_channels_ffn": 1024, - "num_heads": 2, - "num_layers": 6, - "dropout_p": 0.1 - }) + default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1} + ) phase_start_steps: list = None ssim_alpha: float = 1.0 diff --git a/TTS/tts/utils/synthesis.py b/TTS/tts/utils/synthesis.py index 405cf2dc..9f417a1d 100644 --- a/TTS/tts/utils/synthesis.py +++ b/TTS/tts/utils/synthesis.py @@ -256,7 +256,7 @@ def synthesis( """ # GST processing style_mel = None - if CONFIG.has('gst') and CONFIG.gst and style_wav is not None: + if CONFIG.has("gst") and CONFIG.gst and style_wav is not None: if isinstance(style_wav, dict): style_mel = style_wav else: diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index cf64edae..fc969593 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -9,11 +9,11 @@ import re import torch +from TTS.config import load_config from TTS.tts.utils.text.symbols import parse_symbols from TTS.utils.console_logger import ConsoleLogger from TTS.utils.generic_utils import create_experiment_folder, get_git_branch from TTS.utils.io import copy_model_files -from TTS.config import load_config from TTS.utils.tensorboard_logger import TensorboardLogger diff --git a/TTS/vocoder/configs/fullband_melgan_config.py b/TTS/vocoder/configs/fullband_melgan_config.py index 9698d36d..d206451f 100644 --- a/TTS/vocoder/configs/fullband_melgan_config.py +++ b/TTS/vocoder/configs/fullband_melgan_config.py @@ -6,22 +6,18 @@ from .shared_configs import BaseGANVocoderConfig @dataclass class FullbandMelganConfig(BaseGANVocoderConfig): """Defines parameters for FullbandMelGAN vocoder.""" + model: str = "melgan" # Model specific params discriminator_model: str = "melgan_multiscale_discriminator" discriminator_model_params: dict = field( - default_factory=lambda: { - "base_channels": 16, - "max_channels": 512, - "downsample_factors": [4, 4, 4] - }) + default_factory=lambda: {"base_channels": 16, "max_channels": 512, "downsample_factors": [4, 4, 4]} + ) generator_model: str = "melgan_generator" generator_model_params: dict = field( - default_factory=lambda: { - "upsample_factors": [8, 8, 2, 2], - "num_res_blocks": 4 - }) + default_factory=lambda: {"upsample_factors": [8, 8, 2, 2], "num_res_blocks": 4} + ) # Training - overrides batch_size: int = 16 @@ -42,8 +38,9 @@ class FullbandMelganConfig(BaseGANVocoderConfig): default_factory=lambda: { "n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }) + "win_lengths": [600, 1200, 240], + } + ) # loss weights - overrides stft_loss_weight: float = 0.5 diff --git a/TTS/vocoder/configs/hifigan_config.py b/TTS/vocoder/configs/hifigan_config.py index 072bd27f..40b5fc26 100644 --- a/TTS/vocoder/configs/hifigan_config.py +++ b/TTS/vocoder/configs/hifigan_config.py @@ -18,8 +18,9 @@ class HifiganConfig(BaseGANVocoderConfig): "upsample_initial_channel": 512, "resblock_kernel_sizes": [3, 7, 11], "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]], - "resblock_type": "1" - }) + "resblock_type": "1", + } + ) # LOSS PARAMETERS - overrides use_stft_loss: bool = False @@ -45,9 +46,10 @@ class HifiganConfig(BaseGANVocoderConfig): "win_length": 1024, "n_mels": 80, "mel_fmin": 0.0, - "mel_fmax": None - }) + "mel_fmax": None, + } + ) # optimizer parameters lr: float = 1e-4 - wd: float = 1e-6 \ No newline at end of file + wd: float = 1e-6 diff --git a/TTS/vocoder/configs/melgan_config.py b/TTS/vocoder/configs/melgan_config.py index f000be6a..f67c7d1e 100644 --- a/TTS/vocoder/configs/melgan_config.py +++ b/TTS/vocoder/configs/melgan_config.py @@ -6,22 +6,18 @@ from .shared_configs import BaseGANVocoderConfig @dataclass class MelganConfig(BaseGANVocoderConfig): """Defines parameters for MelGAN vocoder.""" + model: str = "melgan" # Model specific params discriminator_model: str = "melgan_multiscale_discriminator" discriminator_model_params: dict = field( - default_factory=lambda: { - "base_channels": 16, - "max_channels": 1024, - "downsample_factors": [4, 4, 4, 4] - }) + default_factory=lambda: {"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]} + ) generator_model: str = "melgan_generator" generator_model_params: dict = field( - default_factory=lambda: { - "upsample_factors": [8, 8, 2, 2], - "num_res_blocks": 3 - }) + default_factory=lambda: {"upsample_factors": [8, 8, 2, 2], "num_res_blocks": 3} + ) # Training - overrides batch_size: int = 16 @@ -42,8 +38,9 @@ class MelganConfig(BaseGANVocoderConfig): default_factory=lambda: { "n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }) + "win_lengths": [600, 1200, 240], + } + ) # loss weights - overrides stft_loss_weight: float = 0.5 diff --git a/TTS/vocoder/configs/multiband_melgan_config.py b/TTS/vocoder/configs/multiband_melgan_config.py index 70745b5c..f8a99152 100644 --- a/TTS/vocoder/configs/multiband_melgan_config.py +++ b/TTS/vocoder/configs/multiband_melgan_config.py @@ -6,42 +6,31 @@ from .shared_configs import BaseGANVocoderConfig @dataclass class MultibandMelganConfig(BaseGANVocoderConfig): """Defines parameters for MultiBandMelGAN vocoder.""" + model: str = "multiband_melgan" # Model specific params discriminator_model: str = "melgan_multiscale_discriminator" discriminator_model_params: dict = field( - default_factory=lambda: { - "base_channels": 16, - "max_channels": 512, - "downsample_factors": [4, 4, 4] - }) + default_factory=lambda: {"base_channels": 16, "max_channels": 512, "downsample_factors": [4, 4, 4]} + ) generator_model: str = "multiband_melgan_generator" - generator_model_params: dict = field( - default_factory=lambda: { - "upsample_factors": [8, 4, 2], - "num_res_blocks": 4 - }) + generator_model_params: dict = field(default_factory=lambda: {"upsample_factors": [8, 4, 2], "num_res_blocks": 4}) use_pqmf: bool = True # optimizer - overrides lr_gen: float = 0.0001 # Initial learning rate. lr_disc: float = 0.0001 # Initial learning rate. optimizer: str = "AdamW" - optimizer_params: dict = field(default_factory=lambda: { - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }) + optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0}) lr_scheduler_gen: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html - lr_scheduler_gen_params: dict = field(default_factory=lambda: { - "gamma": 0.5, - "milestones": [100000, 200000, 300000, 400000, 500000, 600000] - }) + lr_scheduler_gen_params: dict = field( + default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]} + ) lr_scheduler_disc: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html - lr_scheduler_disc_params: dict = field(default_factory=lambda: { - "gamma": 0.5, - "milestones": [100000, 200000, 300000, 400000, 500000, 600000] - }) + lr_scheduler_disc_params: dict = field( + default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]} + ) # Training - overrides batch_size: int = 64 @@ -60,11 +49,8 @@ class MultibandMelganConfig(BaseGANVocoderConfig): use_l1_spec_loss: bool = False subband_stft_loss_params: dict = field( - default_factory=lambda: { - "n_ffts": [384, 683, 171], - "hop_lengths": [30, 60, 10], - "win_lengths": [150, 300, 60] - }) + default_factory=lambda: {"n_ffts": [384, 683, 171], "hop_lengths": [30, 60, 10], "win_lengths": [150, 300, 60]} + ) # loss weights - overrides stft_loss_weight: float = 0.5 diff --git a/TTS/vocoder/configs/parallel_wavegan_config.py b/TTS/vocoder/configs/parallel_wavegan_config.py index 28d8107f..79afa228 100644 --- a/TTS/vocoder/configs/parallel_wavegan_config.py +++ b/TTS/vocoder/configs/parallel_wavegan_config.py @@ -6,21 +6,16 @@ from .shared_configs import BaseGANVocoderConfig @dataclass class ParallelWaveganConfig(BaseGANVocoderConfig): """Defines parameters for ParallelWavegan vocoder.""" + model: str = "parallel_wavegan" # Model specific params discriminator_model: str = "parallel_wavegan_discriminator" - discriminator_model_params: dict = field( - default_factory=lambda: { - "num_layers": 10 - }) + discriminator_model_params: dict = field(default_factory=lambda: {"num_layers": 10}) generator_model: str = "parallel_wavegan_generator" generator_model_params: dict = field( - default_factory=lambda: { - "upsample_factors":[4, 4, 4, 4], - "stacks": 3, - "num_res_blocks": 30 - }) + default_factory=lambda: {"upsample_factors": [4, 4, 4, 4], "stacks": 3, "num_res_blocks": 30} + ) # Training - overrides batch_size: int = 6 @@ -42,8 +37,9 @@ class ParallelWaveganConfig(BaseGANVocoderConfig): default_factory=lambda: { "n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }) + "win_lengths": [600, 1200, 240], + } + ) # loss weights - overrides stft_loss_weight: float = 0.5 @@ -57,17 +53,8 @@ class ParallelWaveganConfig(BaseGANVocoderConfig): lr_gen: float = 0.0002 # Initial learning rate. lr_disc: float = 0.0002 # Initial learning rate. optimizer: str = "AdamW" - optimizer_params: dict = field(default_factory=lambda: { - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }) + optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0}) lr_scheduler_gen: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html - lr_scheduler_gen_params: dict = field(default_factory=lambda: { - "gamma": 0.999, - "last_epoch": -1 - }) + lr_scheduler_gen_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1}) lr_scheduler_disc: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html - lr_scheduler_disc_params: dict = field(default_factory=lambda: { - "gamma": 0.999, - "last_epoch": -1 - }) + lr_scheduler_disc_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1}) diff --git a/TTS/vocoder/configs/shared_configs.py b/TTS/vocoder/configs/shared_configs.py index 0d64c622..d403f84c 100644 --- a/TTS/vocoder/configs/shared_configs.py +++ b/TTS/vocoder/configs/shared_configs.py @@ -9,6 +9,7 @@ from TTS.config import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig @dataclass class BaseVocoderConfig(BaseTrainingConfig): """Shared parameters among all the vocoder models.""" + audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) # dataloading use_noise_augment: bool = False # enable/disable random noise augmentation in spectrograms. @@ -29,6 +30,7 @@ class BaseVocoderConfig(BaseTrainingConfig): @dataclass class BaseGANVocoderConfig(BaseVocoderConfig): """Common config interface for all the GAN based vocoder models.""" + # LOSS PARAMETERS use_stft_loss: bool = True use_subband_stft_loss: bool = True @@ -49,8 +51,9 @@ class BaseGANVocoderConfig(BaseVocoderConfig): default_factory=lambda: { "n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }) + "win_lengths": [600, 1200, 240], + } + ) l1_spec_loss_params: dict = field( default_factory=lambda: { @@ -61,8 +64,9 @@ class BaseGANVocoderConfig(BaseVocoderConfig): "win_length": 1024, "n_mels": 80, "mel_fmin": 0.0, - "mel_fmax": None - }) + "mel_fmax": None, + } + ) target_loss: str = "avg_G_loss" # loss value to pick the best model to save after each epoch @@ -72,20 +76,11 @@ class BaseGANVocoderConfig(BaseVocoderConfig): lr_gen: float = 0.0002 # Initial learning rate. lr_disc: float = 0.0002 # Initial learning rate. optimizer: str = "AdamW" - optimizer_params: dict = field(default_factory=lambda: { - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }) + optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0}) lr_scheduler_gen: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html - lr_scheduler_gen_params: dict = field(default_factory=lambda: { - "gamma": 0.999, - "last_epoch": -1 - }) + lr_scheduler_gen_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1}) lr_scheduler_disc: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html - lr_scheduler_disc_params: dict = field(default_factory=lambda: { - "gamma": 0.999, - "last_epoch": -1 - }) + lr_scheduler_disc_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1}) use_pqmf: bool = False # enable/disable using pqmf for multi-band training. (Multi-band MelGAN) steps_to_start_discriminator = 0 # start training the discriminator after this number of steps. diff --git a/TTS/vocoder/configs/wavegrad_config.py b/TTS/vocoder/configs/wavegrad_config.py index 46ff5290..7638988f 100644 --- a/TTS/vocoder/configs/wavegrad_config.py +++ b/TTS/vocoder/configs/wavegrad_config.py @@ -6,24 +6,22 @@ from .shared_configs import BaseVocoderConfig @dataclass class WavegradConfig(BaseVocoderConfig): """Defines parameters for Wavernn vocoder.""" - model: str = 'wavegrad' + + model: str = "wavegrad" # Model specific params generator_model: str = "wavegrad" model_params: dict = field( default_factory=lambda: { - "use_weight_norm": - True, - "y_conv_channels": - 32, - "x_conv_channels": - 768, + "use_weight_norm": True, + "y_conv_channels": 32, + "x_conv_channels": 768, "ublock_out_channels": [512, 512, 256, 128, 128], "dblock_out_channels": [128, 128, 256, 512], "upsample_factors": [4, 4, 4, 2, 2], - "upsample_dilations": [[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8], - [1, 2, 4, 8], [1, 2, 4, 8]] - }) - target_loss: str = 'avg_wavegrad_loss' # loss value to pick the best model to save after each epoch + "upsample_dilations": [[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8], [1, 2, 4, 8], [1, 2, 4, 8]], + } + ) + target_loss: str = "avg_wavegrad_loss" # loss value to pick the best model to save after each epoch # Training - overrides epochs: int = 10000 @@ -35,24 +33,20 @@ class WavegradConfig(BaseVocoderConfig): eval_split_size: int = 50 # NOISE SCHEDULE PARAMS - train_noise_schedule: dict = field(default_factory=lambda: { - "min_val": 1e-6, - "max_val": 1e-2, - "num_steps": 1000 - }) + train_noise_schedule: dict = field(default_factory=lambda: {"min_val": 1e-6, "max_val": 1e-2, "num_steps": 1000}) - test_noise_schedule: dict = field(default_factory=lambda: { # inference noise schedule. Try TTS/bin/tune_wavegrad.py to find the optimal values. - "min_val": 1e-6, - "max_val": 1e-2, - "num_steps": 50 - }) + test_noise_schedule: dict = field( + default_factory=lambda: { # inference noise schedule. Try TTS/bin/tune_wavegrad.py to find the optimal values. + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 50, + } + ) # optimizer overrides grad_clip: float = 1.0 lr: float = 1e-4 # Initial learning rate. lr_scheduler: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html lr_scheduler_params: dict = field( - default_factory=lambda: { - "gamma": 0.5, - "milestones": [100000, 200000, 300000, 400000, 500000, 600000] - }) + default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]} + ) diff --git a/TTS/vocoder/configs/wavernn_config.py b/TTS/vocoder/configs/wavernn_config.py index 636b0b23..daa586f6 100644 --- a/TTS/vocoder/configs/wavernn_config.py +++ b/TTS/vocoder/configs/wavernn_config.py @@ -6,10 +6,11 @@ from .shared_configs import BaseVocoderConfig @dataclass class WavernnConfig(BaseVocoderConfig): """Defines parameters for Wavernn vocoder.""" + model: str = "wavernn" # Model specific params - mode: str = 'mold' # mold [string], gauss [string], bits [int] + mode: str = "mold" # mold [string], gauss [string], bits [int] mulaw: bool = True # apply mulaw if mode is bits generator_model: str = "WaveRNN" wavernn_model_params: dict = field( @@ -21,9 +22,9 @@ class WavernnConfig(BaseVocoderConfig): "num_res_blocks": 10, "use_aux_net": True, "use_upsample_net": True, - "upsample_factors": - [4, 8, 8] # this needs to correctly factorise hop_length - }) + "upsample_factors": [4, 8, 8], # this needs to correctly factorise hop_length + } + ) # Inference batched: bool = True @@ -46,7 +47,4 @@ class WavernnConfig(BaseVocoderConfig): grad_clip: float = 4.0 lr: float = 1e-4 # Initial learning rate. lr_scheduler: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html - lr_scheduler_params: dict = field(default_factory=lambda: { - "gamma": 0.5, - "milestones": [200000, 400000, 600000] - }) + lr_scheduler_params: dict = field(default_factory=lambda: {"gamma": 0.5, "milestones": [200000, 400000, 600000]}) diff --git a/tests/vocoder_tests/test_fullband_melgan_train.py b/tests/vocoder_tests/test_fullband_melgan_train.py index 358552c4..64355af9 100644 --- a/tests/vocoder_tests/test_fullband_melgan_train.py +++ b/tests/vocoder_tests/test_fullband_melgan_train.py @@ -21,16 +21,14 @@ config = FullbandMelganConfig( print_step=1, print_eval=True, data_path="tests/data/ljspeech", - output_path=output_path + output_path=output_path, ) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " -) +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder diff --git a/tests/vocoder_tests/test_hifigan_train.py b/tests/vocoder_tests/test_hifigan_train.py index 83a3f4b8..fa431eb3 100644 --- a/tests/vocoder_tests/test_hifigan_train.py +++ b/tests/vocoder_tests/test_hifigan_train.py @@ -22,16 +22,14 @@ config = HifiganConfig( print_step=1, print_eval=True, data_path="tests/data/ljspeech", - output_path=output_path + output_path=output_path, ) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " -) +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder diff --git a/tests/vocoder_tests/test_melgan_train.py b/tests/vocoder_tests/test_melgan_train.py index 65b7346a..b362ce86 100644 --- a/tests/vocoder_tests/test_melgan_train.py +++ b/tests/vocoder_tests/test_melgan_train.py @@ -21,16 +21,14 @@ config = MelganConfig( print_step=1, print_eval=True, data_path="tests/data/ljspeech", - output_path=output_path + output_path=output_path, ) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " -) +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder diff --git a/tests/vocoder_tests/test_multiband_melgan_train.py b/tests/vocoder_tests/test_multiband_melgan_train.py index 8ededcce..bd2ae86f 100644 --- a/tests/vocoder_tests/test_multiband_melgan_train.py +++ b/tests/vocoder_tests/test_multiband_melgan_train.py @@ -21,16 +21,14 @@ config = MultibandMelganConfig( print_step=1, print_eval=True, data_path="tests/data/ljspeech", - output_path=output_path + output_path=output_path, ) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " -) +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder diff --git a/tests/vocoder_tests/test_parallel_wavegan_train.py b/tests/vocoder_tests/test_parallel_wavegan_train.py index a2edd0c5..5d89d069 100644 --- a/tests/vocoder_tests/test_parallel_wavegan_train.py +++ b/tests/vocoder_tests/test_parallel_wavegan_train.py @@ -21,16 +21,14 @@ config = ParallelWaveganConfig( print_step=1, print_eval=True, data_path="tests/data/ljspeech", - output_path=output_path + output_path=output_path, ) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " -) +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder diff --git a/tests/vocoder_tests/test_wavegrad_train.py b/tests/vocoder_tests/test_wavegrad_train.py index ffa450b8..c2269bbd 100644 --- a/tests/vocoder_tests/test_wavegrad_train.py +++ b/tests/vocoder_tests/test_wavegrad_train.py @@ -21,16 +21,14 @@ config = WavegradConfig( print_step=1, print_eval=True, data_path="tests/data/ljspeech", - output_path=output_path + output_path=output_path, ) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavegrad.py --config_path {config_path} " -) +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavegrad.py --config_path {config_path} " run_cli(command_train) # Find latest folder diff --git a/tests/vocoder_tests/test_wavernn_train.py b/tests/vocoder_tests/test_wavernn_train.py index 33fc4e57..1ac9d9eb 100644 --- a/tests/vocoder_tests/test_wavernn_train.py +++ b/tests/vocoder_tests/test_wavernn_train.py @@ -21,16 +21,14 @@ config = WavernnConfig( print_step=1, print_eval=True, data_path="tests/data/ljspeech", - output_path=output_path + output_path=output_path, ) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavernn.py --config_path {config_path} " -) +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavernn.py --config_path {config_path} " run_cli(command_train) # Find latest folder From 3fde2001b1b9ecdbc49683022f2a04a1c8b0e6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 17:25:40 +0200 Subject: [PATCH 47/87] train_encoder refactoring for coqpit --- TTS/bin/train_encoder.py | 71 ++++++++++------------------------------ TTS/config/__init__.py | 2 +- 2 files changed, 18 insertions(+), 55 deletions(-) diff --git a/TTS/bin/train_encoder.py b/TTS/bin/train_encoder.py index 3a3f876e..3e985125 100644 --- a/TTS/bin/train_encoder.py +++ b/TTS/bin/train_encoder.py @@ -13,7 +13,7 @@ from torch.utils.data import DataLoader from TTS.speaker_encoder.dataset import MyDataset from TTS.speaker_encoder.losses import AngleProtoLoss, GE2ELoss from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.speaker_encoder.utils.generic_utils import check_config_speaker_encoder, save_best_model +from TTS.speaker_encoder.utils.io import save_best_model, save_checkpoint from TTS.speaker_encoder.utils.visual import plot_embeddings from TTS.tts.datasets.preprocess import load_meta_data from TTS.utils.audio import AudioProcessor @@ -28,6 +28,8 @@ from TTS.utils.io import copy_model_files, load_config from TTS.utils.radam import RAdam from TTS.utils.tensorboard_logger import TensorboardLogger from TTS.utils.training import NoamLR, check_update +from TTS.utils.arguments import init_training + torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True @@ -105,8 +107,9 @@ def train(model, criterion, optimizer, scheduler, ap, global_step): # Averaged Loss and Averaged Loader Time avg_loss = 0.01 * loss.item() + 0.99 * avg_loss if avg_loss != 0 else loss.item() + num_loader_workers = c.num_loader_workers if c.num_loader_workers > 0 else 1 avg_loader_time = ( - 1 / c.num_loader_workers * loader_time + (c.num_loader_workers - 1) / c.num_loader_workers * avg_loader_time + 1 / num_loader_workers * loader_time + (num_loader_workers - 1) / num_loader_workers * avg_loader_time if avg_loader_time != 0 else loader_time ) @@ -139,8 +142,13 @@ def train(model, criterion, optimizer, scheduler, ap, global_step): # save best model best_loss = save_best_model(model, optimizer, avg_loss, best_loss, OUT_PATH, global_step) - end_time = time.time() + + # checkpoint and check stop train cond. + if global_step >= c.max_train_step or global_step % c.save_step == 0: + save_checkpoint(model, optimizer, avg_loss, OUT_PATH, global_step) + if global_step >= c.max_train_step: + break return avg_loss, global_step @@ -149,12 +157,12 @@ def main(args): # pylint: disable=redefined-outer-name global meta_data_train global meta_data_eval - ap = AudioProcessor(**c.audio) + ap = AudioProcessor(**c.audio.to_dict()) model = SpeakerEncoder( - input_dim=c.model["input_dim"], - proj_dim=c.model["proj_dim"], - lstm_dim=c.model["lstm_dim"], - num_lstm_layers=c.model["num_lstm_layers"], + input_dim=c.model_params["input_dim"], + proj_dim=c.model_params["proj_dim"], + lstm_dim=c.model_params["lstm_dim"], + num_lstm_layers=c.model_params["num_lstm_layers"], ) optimizer = RAdam(model.parameters(), lr=c.lr) @@ -168,11 +176,6 @@ def main(args): # pylint: disable=redefined-outer-name if args.restore_path: checkpoint = torch.load(args.restore_path) try: - # TODO: fix optimizer init, model.cuda() needs to be called before - # optimizer restore - # optimizer.load_state_dict(checkpoint['optimizer']) - if c.reinit_layers: - raise RuntimeError model.load_state_dict(checkpoint["model"]) except KeyError: print(" > Partial model initialization.") @@ -207,47 +210,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--restore_path", type=str, help="Path to model outputs (checkpoint, tensorboard etc.).", default=0 - ) - parser.add_argument( - "--config_path", - type=str, - required=True, - help="Path to config file for training.", - ) - parser.add_argument("--debug", type=bool, default=True, help="Do not verify commit integrity to run training.") - parser.add_argument("--data_path", type=str, default="", help="Defines the data path. It overwrites config.json.") - parser.add_argument("--output_path", type=str, help="path for training outputs.", default="") - parser.add_argument("--output_folder", type=str, default="", help="folder name for training outputs.") - args = parser.parse_args() - - # setup output paths and read configs - c = load_config(args.config_path) - check_config_speaker_encoder(c) - _ = os.path.dirname(os.path.realpath(__file__)) - if args.data_path != "": - c.data_path = args.data_path - - if args.output_path == "": - OUT_PATH = os.path.join(_, c.output_path) - else: - OUT_PATH = args.output_path - - if args.output_folder == "": - OUT_PATH = create_experiment_folder(OUT_PATH, c.run_name, args.debug) - else: - OUT_PATH = os.path.join(OUT_PATH, args.output_folder) - - new_fields = {} - if args.restore_path: - new_fields["restore_path"] = args.restore_path - new_fields["github_branch"] = get_git_branch() - copy_model_files(c, args.config_path, OUT_PATH, new_fields) - - LOG_DIR = OUT_PATH - tb_logger = TensorboardLogger(LOG_DIR, model_name="Speaker_Encoder") + args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) diff --git a/TTS/config/__init__.py b/TTS/config/__init__.py index 29ba1190..e16ee6d3 100644 --- a/TTS/config/__init__.py +++ b/TTS/config/__init__.py @@ -9,7 +9,7 @@ from TTS.utils.generic_utils import find_module def _search_configs(model_name): config_class = None - paths = ["TTS.tts.configs", "TTS.vocoder.configs"] + paths = ["TTS.tts.configs", "TTS.vocoder.configs", "TTS.speaker_encoder"] for path in paths: try: config_class = find_module(path, model_name + "_config") From 812dbc2b067bf2f7eacfdd8ec7a9eb00ea345cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 17:26:02 +0200 Subject: [PATCH 48/87] rm config.json --- TTS/speaker_encoder/config.json | 103 -------------------------------- 1 file changed, 103 deletions(-) delete mode 100644 TTS/speaker_encoder/config.json diff --git a/TTS/speaker_encoder/config.json b/TTS/speaker_encoder/config.json deleted file mode 100644 index 4fbd84cc..00000000 --- a/TTS/speaker_encoder/config.json +++ /dev/null @@ -1,103 +0,0 @@ - -{ - "run_name": "mueller91", - "run_description": "train speaker encoder with voxceleb1, voxceleb2 and libriSpeech ", - "audio":{ - // Audio processing parameters - "num_mels": 40, // size of the mel spec frame. - "fft_size": 400, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 16000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "win_length": 400, // stft window length in ms. - "hop_length": 160, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "min_level_db": -100, // normalization range - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - // Normalization parameters - "signal_norm": true, // normalize the spec values in range [0, 1] - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "do_trim_silence": true, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60 // threshold for timming silence. Set this according to your dataset. - }, - "reinit_layers": [], - "loss": "angleproto", // "ge2e" to use Generalized End-to-End loss and "angleproto" to use Angular Prototypical loss (new SOTA) - "grad_clip": 3.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_decay": false, // if true, Noam learning rate decaying is applied through training. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "steps_plot_stats": 10, // number of steps to plot embeddings. - "num_speakers_in_batch": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "num_utters_per_speaker": 10, // - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "wd": 0.000001, // Weight decay weight. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. - "print_step": 20, // Number of steps to log traning on console. - "output_path": "../../MozillaTTSOutput/checkpoints/voxceleb_librispeech/speaker_encoder/", // DATASET-RELATED: output path for all training outputs. - "model": { - "input_dim": 40, - "proj_dim": 256, - "lstm_dim": 768, - "num_lstm_layers": 3, - "use_lstm_with_projection": true - }, - "storage": { - "sample_from_storage_p": 0.66, // the probability with which we'll sample from the DataSet in-memory storage - "storage_size": 15, // the size of the in-memory storage with respect to a single batch - "additive_noise": 1e-5 // add very small gaussian noise to the data in order to increase robustness - }, - "datasets": - [ - { - "name": "vctk_slim", - "path": "../../../audio-datasets/en/VCTK-Corpus/", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "libri_tts", - "path": "../../../audio-datasets/en/LibriTTS/train-clean-100", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "libri_tts", - "path": "../../../audio-datasets/en/LibriTTS/train-clean-360", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "libri_tts", - "path": "../../../audio-datasets/en/LibriTTS/train-other-500", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "voxceleb1", - "path": "../../../audio-datasets/en/voxceleb1/", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "voxceleb2", - "path": "../../../audio-datasets/en/voxceleb2/", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "common_voice", - "path": "../../../audio-datasets/en/MozillaCommonVoice", - "meta_file_train": "train.tsv", - "meta_file_val": "test.tsv" - } - ] -} \ No newline at end of file From ce2bba543e3cba4146e325be973e0dfb95fab146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 17:26:15 +0200 Subject: [PATCH 49/87] remove extra from utils and move funcs to io.py --- TTS/speaker_encoder/utils/generic_utils.py | 115 +-------------------- TTS/speaker_encoder/utils/io.py | 39 +++++++ 2 files changed, 42 insertions(+), 112 deletions(-) diff --git a/TTS/speaker_encoder/utils/generic_utils.py b/TTS/speaker_encoder/utils/generic_utils.py index 69ff25b7..5f470429 100644 --- a/TTS/speaker_encoder/utils/generic_utils.py +++ b/TTS/speaker_encoder/utils/generic_utils.py @@ -1,9 +1,5 @@ -import datetime -import os import re -import torch - from TTS.speaker_encoder.model import SpeakerEncoder @@ -13,111 +9,6 @@ def to_camel(text): def setup_model(c): - model = SpeakerEncoder(c.model["input_dim"], c.model["proj_dim"], c.model["lstm_dim"], c.model["num_lstm_layers"]) - return model - - -def save_checkpoint(model, optimizer, model_loss, out_path, current_step, epoch): - checkpoint_path = "checkpoint_{}.pth.tar".format(current_step) - checkpoint_path = os.path.join(out_path, checkpoint_path) - print(" | | > Checkpoint saving : {}".format(checkpoint_path)) - - new_state_dict = model.state_dict() - state = { - "model": new_state_dict, - "optimizer": optimizer.state_dict() if optimizer is not None else None, - "step": current_step, - "epoch": epoch, - "loss": model_loss, - "date": datetime.date.today().strftime("%B %d, %Y"), - } - torch.save(state, checkpoint_path) - - -def save_best_model(model, optimizer, model_loss, best_loss, out_path, current_step): - if model_loss < best_loss: - new_state_dict = model.state_dict() - state = { - "model": new_state_dict, - "optimizer": optimizer.state_dict(), - "step": current_step, - "loss": model_loss, - "date": datetime.date.today().strftime("%B %d, %Y"), - } - best_loss = model_loss - bestmodel_path = "best_model.pth.tar" - bestmodel_path = os.path.join(out_path, bestmodel_path) - print("\n > BEST MODEL ({0:.5f}) : {1:}".format(model_loss, bestmodel_path)) - torch.save(state, bestmodel_path) - return best_loss - - -def check_config_speaker_encoder(c): - ... - # """Check the config.json file of the speaker encoder""" - # check_argument("run_name", c, restricted=True, val_type=str) - # check_argument("run_description", c, val_type=str) - - # # audio processing parameters - # check_argument("audio", c, restricted=True, val_type=dict) - # check_argument("num_mels", c["audio"], restricted=True, val_type=int, min_val=10, max_val=2056) - # check_argument("fft_size", c["audio"], restricted=True, val_type=int, min_val=128, max_val=4058) - # check_argument("sample_rate", c["audio"], restricted=True, val_type=int, min_val=512, max_val=100000) - # check_argument( - # "frame_length_ms", - # c["audio"], - # restricted=True, - # val_type=float, - # min_val=10, - # max_val=1000, - # alternative="win_length", - # ) - # check_argument( - # "frame_shift_ms", c["audio"], restricted=True, val_type=float, min_val=1, max_val=1000, alternative="hop_length" - # ) - # check_argument("preemphasis", c["audio"], restricted=True, val_type=float, min_val=0, max_val=1) - # check_argument("min_level_db", c["audio"], restricted=True, val_type=int, min_val=-1000, max_val=10) - # check_argument("ref_level_db", c["audio"], restricted=True, val_type=int, min_val=0, max_val=1000) - # check_argument("power", c["audio"], restricted=True, val_type=float, min_val=1, max_val=5) - # check_argument("griffin_lim_iters", c["audio"], restricted=True, val_type=int, min_val=10, max_val=1000) - - # # training parameters - # check_argument("loss", c, enum_list=["ge2e", "angleproto"], restricted=True, val_type=str) - # check_argument("grad_clip", c, restricted=True, val_type=float) - # check_argument("epochs", c, restricted=True, val_type=int, min_val=1) - # check_argument("lr", c, restricted=True, val_type=float, min_val=0) - # check_argument("lr_decay", c, restricted=True, val_type=bool) - # check_argument("warmup_steps", c, restricted=True, val_type=int, min_val=0) - # check_argument("tb_model_param_stats", c, restricted=True, val_type=bool) - # check_argument("num_speakers_in_batch", c, restricted=True, val_type=int) - # check_argument("num_loader_workers", c, restricted=True, val_type=int) - # check_argument("wd", c, restricted=True, val_type=float, min_val=0.0, max_val=1.0) - - # # checkpoint and output parameters - # check_argument("steps_plot_stats", c, restricted=True, val_type=int) - # check_argument("checkpoint", c, restricted=True, val_type=bool) - # check_argument("save_step", c, restricted=True, val_type=int) - # check_argument("print_step", c, restricted=True, val_type=int) - # check_argument("output_path", c, restricted=True, val_type=str) - - # # model parameters - # check_argument("model", c, restricted=True, val_type=dict) - # check_argument("input_dim", c["model"], restricted=True, val_type=int) - # check_argument("proj_dim", c["model"], restricted=True, val_type=int) - # check_argument("lstm_dim", c["model"], restricted=True, val_type=int) - # check_argument("num_lstm_layers", c["model"], restricted=True, val_type=int) - # check_argument("use_lstm_with_projection", c["model"], restricted=True, val_type=bool) - - # # in-memory storage parameters - # check_argument("storage", c, restricted=True, val_type=dict) - # check_argument("sample_from_storage_p", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0) - # check_argument("storage_size", c["storage"], restricted=True, val_type=int, min_val=1, max_val=100) - # check_argument("additive_noise", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0) - - # # datasets - checking only the first entry - # check_argument("datasets", c, restricted=True, val_type=list) - # for dataset_entry in c["datasets"]: - # check_argument("name", dataset_entry, restricted=True, val_type=str) - # check_argument("path", dataset_entry, restricted=True, val_type=str) - # check_argument("meta_file_train", dataset_entry, restricted=True, val_type=[str, list]) - # check_argument("meta_file_val", dataset_entry, restricted=True, val_type=str) + model = SpeakerEncoder(c.model["input_dim"], c.model["proj_dim"], + c.model["lstm_dim"], c.model["num_lstm_layers"]) + return model \ No newline at end of file diff --git a/TTS/speaker_encoder/utils/io.py b/TTS/speaker_encoder/utils/io.py index e69de29b..8ba33c45 100644 --- a/TTS/speaker_encoder/utils/io.py +++ b/TTS/speaker_encoder/utils/io.py @@ -0,0 +1,39 @@ +import os +import datetime +import torch + + +def save_checkpoint(model, optimizer, model_loss, out_path, current_step): + checkpoint_path = "checkpoint_{}.pth.tar".format(current_step) + checkpoint_path = os.path.join(out_path, checkpoint_path) + print(" | | > Checkpoint saving : {}".format(checkpoint_path)) + + new_state_dict = model.state_dict() + state = { + "model": new_state_dict, + "optimizer": optimizer.state_dict() if optimizer is not None else None, + "step": current_step, + "loss": model_loss, + "date": datetime.date.today().strftime("%B %d, %Y"), + } + torch.save(state, checkpoint_path) + + +def save_best_model(model, optimizer, model_loss, best_loss, out_path, + current_step): + if model_loss < best_loss: + new_state_dict = model.state_dict() + state = { + "model": new_state_dict, + "optimizer": optimizer.state_dict(), + "step": current_step, + "loss": model_loss, + "date": datetime.date.today().strftime("%B %d, %Y"), + } + best_loss = model_loss + bestmodel_path = "best_model.pth.tar" + bestmodel_path = os.path.join(out_path, bestmodel_path) + print("\n > BEST MODEL ({0:.5f}) : {1:}".format( + model_loss, bestmodel_path)) + torch.save(state, bestmodel_path) + return best_loss From 9f2d2d2081efb6b6aa3c023e4f7b4ef2beb84736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 17:27:05 +0200 Subject: [PATCH 50/87] add speaker encoder train test --- tests/test_speaker_encoder_train.py | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/test_speaker_encoder_train.py diff --git a/tests/test_speaker_encoder_train.py b/tests/test_speaker_encoder_train.py new file mode 100644 index 00000000..0bf04966 --- /dev/null +++ b/tests/test_speaker_encoder_train.py @@ -0,0 +1,46 @@ +import glob +import os +import shutil + +from tests import get_tests_output_path, run_cli +from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig +from TTS.config.shared_configs import BaseAudioConfig + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = SpeakerEncoderConfig( + batch_size=4, + num_speakers_in_batch=1, + num_utters_per_speaker=10, + num_loader_workers=0, + max_train_step=10, + print_step=1, + save_step=10, + print_eval=True, + audio=BaseAudioConfig(num_mels=40) +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_encoder.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " + "--coqpit.datasets.0.meta_file_attn_mask tests/data/ljspeech/metadata_attn_mask.txt" +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_encoder.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) From f8e52965dd4f9f90d70271ff2d832542c96adfbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Fri, 7 May 2021 17:27:33 +0200 Subject: [PATCH 51/87] add speaker encoder coqpit --- TTS/speaker_encoder/speaker_encoder_config.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 TTS/speaker_encoder/speaker_encoder_config.py diff --git a/TTS/speaker_encoder/speaker_encoder_config.py b/TTS/speaker_encoder/speaker_encoder_config.py new file mode 100644 index 00000000..402ac892 --- /dev/null +++ b/TTS/speaker_encoder/speaker_encoder_config.py @@ -0,0 +1,54 @@ +from coqpit import MISSING +from dataclasses import dataclass, field, asdict +from typing import List +from TTS.config.shared_configs import BaseTrainingConfig, BaseAudioConfig, BaseDatasetConfig + + +@dataclass +class SpeakerEncoderConfig(BaseTrainingConfig): + """Defines parameters for Speaker Encoder model.""" + + model: str = "speaker_encoder" + audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) + datasets: List[BaseDatasetConfig] = field(default_factory=lambda: [BaseDatasetConfig()]) + + # model params + model_params: dict = field(default_factory=lambda: { + "input_dim": 40, + "proj_dim": 256, + "lstm_dim": 768, + "num_lstm_layers": 3, + "use_lstm_with_projection": True + }) + + storage: dict = field(default_factory=lambda:{ + "sample_from_storage_p": 0.66, # the probability with which we'll sample from the DataSet in-memory storage + "storage_size": 15, # the size of the in-memory storage with respect to a single batch + "additive_noise": 1e-5 # add very small gaussian noise to the data in order to increase robustness + }) + + # training params + max_train_step: int = 1000 # end training when number of training steps reaches this value. + loss: str = 'angleproto' + grad_clip: float = 3.0 + lr: float = 0.0001 + lr_decay: bool = False + warmup_steps: int = 4000 + wd: float = 1e-6 + + # logging params + tb_model_param_stats: bool = False + steps_plot_stats: int = 10 + checkpoint: bool = True + save_step: int = 1000 + print_step: int = 20 + + # data loader + num_speakers_in_batch: int = MISSING + num_utters_per_speaker: int = MISSING + num_loader_workers: int = MISSING + + def check_values(self): + super().check_values() + c = asdict(self) + assert c['model_params']['input_dim'] == self.audio.num_mels, " [!] model input dimendion must be equal to melspectrogram dimension." From e5d757a9d346df6aa14ecec8a1d22262e286065a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:07:42 +0200 Subject: [PATCH 52/87] run nosetests with --with-id and add a command to run only the failes tests --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 650958ae..2210a682 100644 --- a/Makefile +++ b/Makefile @@ -18,9 +18,12 @@ deps: ## install 🐸 requirements. pip install -r requirements.txt test: ## run tests. - nosetests -x --with-cov -cov --cover-erase --cover-package TTS tests --nologcapture + nosetests -x --with-cov -cov --cover-erase --cover-package TTS tests --nologcapture --with-id ./run_bash_tests.sh +test_failed: ## only run tests failed the last time. + nosetests -x --with-cov -cov --cover-erase --cover-package TTS tests --nologcapture --failed + style: ## update code style. black ${target_dirs} isort ${target_dirs} From 9f7599e3c362f33ce85a3e3ca4466748bf9ba680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:09:36 +0200 Subject: [PATCH 53/87] fix train_encoder for coqpit --- TTS/bin/train_encoder.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/TTS/bin/train_encoder.py b/TTS/bin/train_encoder.py index 3e985125..6c1a03e8 100644 --- a/TTS/bin/train_encoder.py +++ b/TTS/bin/train_encoder.py @@ -19,14 +19,10 @@ from TTS.tts.datasets.preprocess import load_meta_data from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import ( count_parameters, - create_experiment_folder, - get_git_branch, remove_experiment_folder, set_init_dict, ) -from TTS.utils.io import copy_model_files, load_config from TTS.utils.radam import RAdam -from TTS.utils.tensorboard_logger import TensorboardLogger from TTS.utils.training import NoamLR, check_update from TTS.utils.arguments import init_training From df1ddd3539ecb3bd5a8b3906df89aac7ee7b0687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:09:55 +0200 Subject: [PATCH 54/87] allow read_json_with_comments for backward compat --- TTS/config/__init__.py | 47 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/TTS/config/__init__.py b/TTS/config/__init__.py index e16ee6d3..3ef37820 100644 --- a/TTS/config/__init__.py +++ b/TTS/config/__init__.py @@ -1,12 +1,24 @@ import json import os - +import re import yaml from TTS.config.shared_configs import * from TTS.utils.generic_utils import find_module +def read_json_with_comments(json_path): + """for backward compat.""" + # fallback to json + with open(json_path, "r", encoding="utf-8") as f: + input_str = f.read() + # handle comments + input_str = re.sub(r"\\\n", "", input_str) + input_str = re.sub(r"//.*\n", "\n", input_str) + data = json.loads(input_str) + return data + + def _search_configs(model_name): config_class = None paths = ["TTS.tts.configs", "TTS.vocoder.configs", "TTS.speaker_encoder"] @@ -16,24 +28,47 @@ def _search_configs(model_name): except ModuleNotFoundError: pass if config_class is None: - raise ModuleNotFoundError() + raise ModuleNotFoundError(f" [!] Config for {model_name} cannot be found.") return config_class +def _process_model_name(config_dict): + model_name = config_dict["model"] if "model" in config_dict else config_dict["generator_model"] + model_name = model_name.replace('_generator', '').replace('_discriminator', '') + return model_name + + def load_config(config_path: str) -> None: + """Import `json` or `yaml` files as TTS configs. First, load the input file as a `dict` and check the model name + to find the corresponding Config class. Then initialize the Config. + + Args: + config_path (str): path to the config file. + + Raises: + TypeError: given config file has an unknown type. + + Returns: + Coqpit: TTS config object. + """ config_dict = {} ext = os.path.splitext(config_path)[1] if ext in (".yml", ".yaml"): with open(config_path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) elif ext == ".json": - with open(config_path, "r", encoding="utf-8") as f: - input_str = f.read() - data = json.loads(input_str) + try: + with open(config_path, "r", encoding="utf-8") as f: + input_str = f.read() + data = json.loads(input_str) + except json.decoder.JSONDecodeError: + # backwards compat. + data = read_json_with_comments(config_path) else: raise TypeError(f" [!] Unknown config file type {ext}") config_dict.update(data) - config_class = _search_configs(config_dict["model"].lower()) + model_name = _process_model_name(config_dict) + config_class = _search_configs(model_name.lower()) config = config_class() config.from_dict(config_dict) return config From 10de40bba174a5d91d07abf21f218e094ab81e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:10:39 +0200 Subject: [PATCH 55/87] make num_workers mandatory config field --- TTS/config/shared_configs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py index 9bd18ff0..d9a4e06a 100644 --- a/TTS/config/shared_configs.py +++ b/TTS/config/shared_configs.py @@ -248,8 +248,8 @@ class BaseTrainingConfig(Coqpit): keep_all_best: bool = False keep_after: int = 10000 # dataloading - num_loader_workers: int = None - num_val_loader_workers: int = None + num_loader_workers: int = MISSING + num_val_loader_workers: int = 0 use_noise_augment: bool = False # paths output_path: str = None From 18e76a2309bdbca742824b320d4465c5a1493823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:11:28 +0200 Subject: [PATCH 56/87] fix speaker encoder model initialization --- TTS/speaker_encoder/utils/generic_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TTS/speaker_encoder/utils/generic_utils.py b/TTS/speaker_encoder/utils/generic_utils.py index 5f470429..13296d64 100644 --- a/TTS/speaker_encoder/utils/generic_utils.py +++ b/TTS/speaker_encoder/utils/generic_utils.py @@ -9,6 +9,6 @@ def to_camel(text): def setup_model(c): - model = SpeakerEncoder(c.model["input_dim"], c.model["proj_dim"], - c.model["lstm_dim"], c.model["num_lstm_layers"]) + model = SpeakerEncoder(c.model_params["input_dim"], c.model_params["proj_dim"], + c.model_params["lstm_dim"], c.model_params["num_lstm_layers"]) return model \ No newline at end of file From c57f0b46bba042b30ff4c37aa625f59ddeed787b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:12:18 +0200 Subject: [PATCH 57/87] reintro use_gst for backwars compat --- TTS/tts/configs/shared_configs.py | 3 ++- TTS/tts/configs/tacotron_config.py | 8 +------- TTS/tts/models/tacotron.py | 9 ++++++--- TTS/tts/models/tacotron2.py | 9 ++++++--- TTS/tts/models/tacotron_abstract.py | 10 ++++++---- TTS/tts/utils/generic_utils.py | 2 ++ 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/TTS/tts/configs/shared_configs.py b/TTS/tts/configs/shared_configs.py index c82b821e..9f4b2b68 100644 --- a/TTS/tts/configs/shared_configs.py +++ b/TTS/tts/configs/shared_configs.py @@ -32,7 +32,7 @@ class GSTConfig(Coqpit): @dataclass -class CharactersConfig: +class CharactersConfig(Coqpit): """Defines character or phoneme set used by the model""" pad: str = None @@ -41,6 +41,7 @@ class CharactersConfig: characters: str = None punctuations: str = None phonemes: str = None + unique: bool = True # for backwards compatibility of models trained with char sets with duplicates def check_values( self, diff --git a/TTS/tts/configs/tacotron_config.py b/TTS/tts/configs/tacotron_config.py index 8b1ed20c..5c86f500 100644 --- a/TTS/tts/configs/tacotron_config.py +++ b/TTS/tts/configs/tacotron_config.py @@ -11,6 +11,7 @@ class TacotronConfig(BaseTTSConfig): """Defines parameters for Tacotron based models.""" model: str = "tacotron" + use_gst: bool = False gst: GSTConfig = None gst_style_input: str = None # model specific params @@ -61,10 +62,3 @@ class TacotronConfig(BaseTTSConfig): decoder_ssim_alpha: float = 0.25 postnet_ssim_alpha: float = 0.25 ga_alpha: float = 5.0 - - -@dataclass -class Tacotron2Config(TacotronConfig): - """Defines parameters for Tacotron2 based models.""" - - model: str = "tacotron2" diff --git a/TTS/tts/models/tacotron.py b/TTS/tts/models/tacotron.py index 1ffe9786..89d98e9f 100644 --- a/TTS/tts/models/tacotron.py +++ b/TTS/tts/models/tacotron.py @@ -41,6 +41,7 @@ class Tacotron(TacotronAbstract): encoder_in_features (int, optional): input channels for the encoder. Defaults to 512. decoder_in_features (int, optional): input channels for the decoder. Defaults to 512. speaker_embedding_dim (int, optional): external speaker conditioning vector channels. Defaults to None. + use_gst (bool, optional): enable/disable Global style token module. gst (Coqpit, optional): Coqpit to initialize the GST module. If `None`, GST is disabled. Defaults to None. memory_size (int, optional): size of the history queue fed to the prenet. Model feeds the last ```memory_size``` output frames to the prenet. @@ -71,6 +72,7 @@ class Tacotron(TacotronAbstract): encoder_in_features=256, decoder_in_features=256, speaker_embedding_dim=None, + use_gst=False, gst=None, memory_size=5, ): @@ -98,6 +100,7 @@ class Tacotron(TacotronAbstract): encoder_in_features, decoder_in_features, speaker_embedding_dim, + use_gst, gst, ) @@ -142,7 +145,7 @@ class Tacotron(TacotronAbstract): self.decoder.prenet.dropout_at_inference = prenet_dropout_at_inference # global style token layers - if self.gst: + if self.gst and self.use_gst: self.gst_layer = GST( num_mel=decoder_output_dim, speaker_embedding_dim=speaker_embedding_dim, @@ -191,7 +194,7 @@ class Tacotron(TacotronAbstract): # sequence masking encoder_outputs = encoder_outputs * input_mask.unsqueeze(2).expand_as(encoder_outputs) # global style token - if self.gst: + if self.gst and self.use_gst: # B x gst_dim encoder_outputs = self.compute_gst(encoder_outputs, mel_specs, speaker_embeddings) # speaker embedding @@ -247,7 +250,7 @@ class Tacotron(TacotronAbstract): def inference(self, characters, speaker_ids=None, style_mel=None, speaker_embeddings=None): inputs = self.embedding(characters) encoder_outputs = self.encoder(inputs) - if self.gst: + if self.gst and self.use_gst: # B x gst_dim encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings) if self.num_speakers > 1: diff --git a/TTS/tts/models/tacotron2.py b/TTS/tts/models/tacotron2.py index 1945a6f7..fded8f87 100644 --- a/TTS/tts/models/tacotron2.py +++ b/TTS/tts/models/tacotron2.py @@ -41,6 +41,7 @@ class Tacotron2(TacotronAbstract): encoder_in_features (int, optional): input channels for the encoder. Defaults to 512. decoder_in_features (int, optional): input channels for the decoder. Defaults to 512. speaker_embedding_dim (int, optional): external speaker conditioning vector channels. Defaults to None. + use_gst (bool, optional): enable/disable Global style token module. gst (Coqpit, optional): Coqpit to initialize the GST module. If `None`, GST is disabled. Defaults to None. """ @@ -69,6 +70,7 @@ class Tacotron2(TacotronAbstract): encoder_in_features=512, decoder_in_features=512, speaker_embedding_dim=None, + use_gst=False, gst=None, ): super().__init__( @@ -95,6 +97,7 @@ class Tacotron2(TacotronAbstract): encoder_in_features, decoder_in_features, speaker_embedding_dim, + use_gst, gst, ) @@ -136,7 +139,7 @@ class Tacotron2(TacotronAbstract): self.decoder.prenet.dropout_at_inference = prenet_dropout_at_inference # global style token layers - if self.gst: + if self.gst and use_gst: self.gst_layer = GST( num_mel=decoder_output_dim, speaker_embedding_dim=speaker_embedding_dim, @@ -190,7 +193,7 @@ class Tacotron2(TacotronAbstract): embedded_inputs = self.embedding(text).transpose(1, 2) # B x T_in_max x D_en encoder_outputs = self.encoder(embedded_inputs, text_lengths) - if self.gst: + if self.gst and self.use_gst: # B x gst_dim encoder_outputs = self.compute_gst(encoder_outputs, mel_specs, speaker_embeddings) if self.num_speakers > 1: @@ -246,7 +249,7 @@ class Tacotron2(TacotronAbstract): embedded_inputs = self.embedding(text).transpose(1, 2) encoder_outputs = self.encoder.inference(embedded_inputs) - if self.gst: + if self.gst and self.use_gst: # B x gst_dim encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings) if self.num_speakers > 1: diff --git a/TTS/tts/models/tacotron_abstract.py b/TTS/tts/models/tacotron_abstract.py index 42411656..e684ce7c 100644 --- a/TTS/tts/models/tacotron_abstract.py +++ b/TTS/tts/models/tacotron_abstract.py @@ -33,6 +33,7 @@ class TacotronAbstract(ABC, nn.Module): encoder_in_features=512, decoder_in_features=512, speaker_embedding_dim=None, + use_gst=False, gst=None, ): """Abstract Tacotron class""" @@ -41,6 +42,7 @@ class TacotronAbstract(ABC, nn.Module): self.r = r self.decoder_output_dim = decoder_output_dim self.postnet_output_dim = postnet_output_dim + self.use_gst = use_gst self.gst = gst self.num_speakers = num_speakers self.bidirectional_decoder = bidirectional_decoder @@ -77,7 +79,7 @@ class TacotronAbstract(ABC, nn.Module): self.embeddings_per_sample = True # global style token - if self.gst: + if self.gst and use_gst: self.decoder_in_features += self.gst.gst_embedding_dim # add gst embedding dim self.gst_layer = None @@ -186,18 +188,18 @@ class TacotronAbstract(ABC, nn.Module): """Compute global style token""" device = inputs.device if isinstance(style_input, dict): - query = torch.zeros(1, 1, self.gst_embedding_dim // 2).to(device) + query = torch.zeros(1, 1, self.gst.gst_embedding_dim // 2).to(device) if speaker_embedding is not None: query = torch.cat([query, speaker_embedding.reshape(1, 1, -1)], dim=-1) _GST = torch.tanh(self.gst_layer.style_token_layer.style_tokens) - gst_outputs = torch.zeros(1, 1, self.gst_embedding_dim).to(device) + gst_outputs = torch.zeros(1, 1, self.gst.gst_embedding_dim).to(device) for k_token, v_amplifier in style_input.items(): key = _GST[int(k_token)].unsqueeze(0).expand(1, -1, -1) gst_outputs_att = self.gst_layer.style_token_layer.attention(query, key) gst_outputs = gst_outputs + gst_outputs_att * v_amplifier elif style_input is None: - gst_outputs = torch.zeros(1, 1, self.gst_embedding_dim).to(device) + gst_outputs = torch.zeros(1, 1, self.gst.gst_embedding_dim).to(device) else: gst_outputs = self.gst_layer(style_input, speaker_embedding) # pylint: disable=not-callable inputs = self._concat_speaker_embedding(inputs, gst_outputs) diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index b81a75ff..b0e53f33 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -22,6 +22,7 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): r=c.r, postnet_output_dim=int(c.audio["fft_size"] / 2 + 1), decoder_output_dim=c.audio["num_mels"], + use_gst=c.use_gst, gst=c.gst, memory_size=c.memory_size, attn_type=c.attention_type, @@ -48,6 +49,7 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): r=c.r, postnet_output_dim=c.audio["num_mels"], decoder_output_dim=c.audio["num_mels"], + use_gst=c.use_gst, gst=c.gst, attn_type=c.attention_type, attn_win=c.windowing, From 21dd4d79608602fbadadc12bd6566745160c93a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:15:22 +0200 Subject: [PATCH 58/87] fix load_config imports for Coqpit --- TTS/tts/utils/speakers.py | 2 +- TTS/utils/manage.py | 2 +- TTS/utils/synthesizer.py | 2 +- tests/test_glow_tts.py | 2 +- tests/test_speaker_encoder.py | 2 -- tests/test_speaker_manager.py | 5 +++-- tests/test_synthesizer.py | 2 +- tests/test_tacotron2_model.py | 2 +- tests/test_tacotron2_tf_model.py | 4 ++-- tests/test_tacotron_model.py | 3 ++- tests/test_text_processing.py | 5 +++-- tests/vocoder_tests/test_vocoder_losses.py | 2 +- tests/vocoder_tests/test_vocoder_wavernn_datasets.py | 2 +- 13 files changed, 18 insertions(+), 17 deletions(-) diff --git a/TTS/tts/utils/speakers.py b/TTS/tts/utils/speakers.py index b80e8ee3..f92d3af5 100755 --- a/TTS/tts/utils/speakers.py +++ b/TTS/tts/utils/speakers.py @@ -8,7 +8,7 @@ import torch from TTS.speaker_encoder.utils.generic_utils import setup_model from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.config import load_config def make_speakers_json_path(out_path): diff --git a/TTS/utils/manage.py b/TTS/utils/manage.py index 0cf69706..87e499e7 100644 --- a/TTS/utils/manage.py +++ b/TTS/utils/manage.py @@ -9,7 +9,7 @@ import gdown import requests from TTS.utils.generic_utils import get_user_data_dir -from TTS.utils.io import load_config +from TTS.config import load_config class ModelManager(object): diff --git a/TTS/utils/synthesizer.py b/TTS/utils/synthesizer.py index 323231bf..7d418cb1 100644 --- a/TTS/utils/synthesizer.py +++ b/TTS/utils/synthesizer.py @@ -13,7 +13,7 @@ from TTS.tts.utils.speakers import SpeakerManager from TTS.tts.utils.synthesis import synthesis, trim_silence from TTS.tts.utils.text import make_symbols, phonemes, symbols from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.config import load_config from TTS.vocoder.utils.generic_utils import interpolate_vocoder_input, setup_generator diff --git a/tests/test_glow_tts.py b/tests/test_glow_tts.py index 7e17ed45..77801b29 100644 --- a/tests/test_glow_tts.py +++ b/tests/test_glow_tts.py @@ -9,7 +9,7 @@ from tests import get_tests_input_path from TTS.tts.layers.losses import GlowTTSLoss from TTS.tts.models.glow_tts import GlowTTS from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.tts.configs import GlowTTSConfig # pylint: disable=unused-variable diff --git a/tests/test_speaker_encoder.py b/tests/test_speaker_encoder.py index 32ba2924..3e8dd947 100644 --- a/tests/test_speaker_encoder.py +++ b/tests/test_speaker_encoder.py @@ -6,10 +6,8 @@ import torch as T from tests import get_tests_input_path from TTS.speaker_encoder.losses import AngleProtoLoss, GE2ELoss from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.utils.io import load_config file_path = get_tests_input_path() -c = load_config(os.path.join(file_path, "test_config.json")) class SpeakerEncoderTests(unittest.TestCase): diff --git a/tests/test_speaker_manager.py b/tests/test_speaker_manager.py index b176e353..f8c742d9 100644 --- a/tests/test_speaker_manager.py +++ b/tests/test_speaker_manager.py @@ -6,10 +6,11 @@ import torch from tests import get_tests_input_path from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.speaker_encoder.utils.generic_utils import save_checkpoint +from TTS.speaker_encoder.utils.io import save_checkpoint +from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig +from TTS.config import load_config from TTS.tts.utils.speakers import SpeakerManager from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config encoder_config_path = os.path.join(get_tests_input_path(), "test_speaker_encoder_config.json") encoder_model_path = os.path.join(get_tests_input_path(), "checkpoint_0.pth.tar") diff --git a/tests/test_synthesizer.py b/tests/test_synthesizer.py index aa06a6f0..46997dbb 100644 --- a/tests/test_synthesizer.py +++ b/tests/test_synthesizer.py @@ -5,7 +5,7 @@ from tests import get_tests_input_path, get_tests_output_path from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.io import save_checkpoint from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols -from TTS.utils.io import load_config +from TTS.config import load_config from TTS.utils.synthesizer import Synthesizer diff --git a/tests/test_tacotron2_model.py b/tests/test_tacotron2_model.py index 0e35605f..3a08e4d0 100644 --- a/tests/test_tacotron2_model.py +++ b/tests/test_tacotron2_model.py @@ -9,7 +9,7 @@ from tests import get_tests_input_path from TTS.tts.layers.losses import MSELossMasked from TTS.tts.models.tacotron2 import Tacotron2 from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.tts.configs import Tacotron2Config # pylint: disable=unused-variable diff --git a/tests/test_tacotron2_tf_model.py b/tests/test_tacotron2_tf_model.py index 2c4b4884..aa9c1846 100644 --- a/tests/test_tacotron2_tf_model.py +++ b/tests/test_tacotron2_tf_model.py @@ -5,10 +5,10 @@ import numpy as np import tensorflow as tf import torch -from tests import get_tests_input_path from TTS.tts.tf.models.tacotron2 import Tacotron2 from TTS.tts.tf.utils.tflite import convert_tacotron2_to_tflite, load_tflite_model -from TTS.utils.io import load_config +from TTS.tts.configs import Tacotron2Config + tf.get_logger().setLevel("INFO") diff --git a/tests/test_tacotron_model.py b/tests/test_tacotron_model.py index 72b47d23..d6f5189e 100644 --- a/tests/test_tacotron_model.py +++ b/tests/test_tacotron_model.py @@ -9,7 +9,8 @@ from tests import get_tests_input_path from TTS.tts.layers.losses import L1LossMasked from TTS.tts.models.tacotron import Tacotron from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.tts.configs import TacotronConfig + # pylint: disable=unused-variable diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py index f70056b1..b7b8c501 100644 --- a/tests/test_text_processing.py +++ b/tests/test_text_processing.py @@ -5,9 +5,10 @@ import os # pylint: disable=unused-import from tests import get_tests_input_path, get_tests_path from TTS.tts.utils.text import * -from TTS.utils.io import load_config +from TTS.tts.configs import TacotronConfig -conf = load_config(os.path.join(get_tests_input_path(), "test_config.json")) + +conf = TacotronConfig() def test_phoneme_to_sequence(): diff --git a/tests/vocoder_tests/test_vocoder_losses.py b/tests/vocoder_tests/test_vocoder_losses.py index 87151a05..915c5947 100644 --- a/tests/vocoder_tests/test_vocoder_losses.py +++ b/tests/vocoder_tests/test_vocoder_losses.py @@ -4,7 +4,7 @@ import torch from tests import get_tests_input_path, get_tests_output_path, get_tests_path from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.config import BaseAudioConfig from TTS.vocoder.layers.losses import MelganFeatureLoss, MultiScaleSTFTLoss, STFTLoss, TorchSTFT TESTS_PATH = get_tests_path() diff --git a/tests/vocoder_tests/test_vocoder_wavernn_datasets.py b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py index 7adc1dea..755d4772 100644 --- a/tests/vocoder_tests/test_vocoder_wavernn_datasets.py +++ b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py @@ -6,7 +6,7 @@ from torch.utils.data import DataLoader from tests import get_tests_input_path, get_tests_output_path, get_tests_path from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.vocoder.configs import WavernnConfig from TTS.vocoder.datasets.preprocess import load_wav_feat_data, preprocess_wav_files from TTS.vocoder.datasets.wavernn_dataset import WaveRNNDataset From a21ac883dd698835adc2d31fe8fe69469570c71d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:18:58 +0200 Subject: [PATCH 59/87] add get_cuda() --- TTS/utils/generic_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index e8beff88..5473d32d 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- import datetime import glob @@ -8,10 +7,17 @@ import re import shutil import subprocess import sys +import torch from pathlib import Path from typing import List +def get_cuda(): + use_cuda = torch.cuda.is_available() + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + return use_cuda, device + + def get_git_branch(): try: out = subprocess.check_output(["git", "branch"]).decode("utf8") From db14dcd95adbf839c7f463cdc37fd18e99018b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:19:27 +0200 Subject: [PATCH 60/87] remove old load_config --- TTS/utils/io.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/TTS/utils/io.py b/TTS/utils/io.py index 6d233d24..b09a71d1 100644 --- a/TTS/utils/io.py +++ b/TTS/utils/io.py @@ -27,37 +27,6 @@ class AttrDict(dict): self.__dict__ = self -def read_json_with_comments(json_path): - """DEPRECATED""" - # fallback to json - with open(json_path, "r", encoding="utf-8") as f: - input_str = f.read() - # handle comments - input_str = re.sub(r"\\\n", "", input_str) - input_str = re.sub(r"//.*\n", "\n", input_str) - data = json.loads(input_str) - return data - - -def load_config(config_path: str) -> None: - config_dict = {} - ext = os.path.splitext(config_path)[1] - if ext in (".yml", ".yaml"): - with open(config_path, "r", encoding="utf-8") as f: - data = yaml.safe_load(f) - elif ext == ".json": - with open(config_path, "r", encoding="utf-8") as f: - input_str = f.read() - data = json.loads(input_str) - else: - raise TypeError(f" [!] Unknown config file type {ext}") - config_dict.update(data) - config_class = find_module("TTS.tts.configs", config_dict["model"].lower() + "_config") - config = config_class() - config.from_dict(config_dict) - return config - - def copy_model_files(config, out_path, new_fields): """Copy config.json and other model files to training folder and add new fields. From 6e980b49c404fdda3af487aa4b7c41e6d2bc471d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:24:21 +0200 Subject: [PATCH 61/87] fix synthesizer.py for Coqpit --- TTS/utils/synthesizer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/TTS/utils/synthesizer.py b/TTS/utils/synthesizer.py index 7d418cb1..c7339d7b 100644 --- a/TTS/utils/synthesizer.py +++ b/TTS/utils/synthesizer.py @@ -113,12 +113,11 @@ class Synthesizer(object): # pylint: disable=global-statement global symbols, phonemes - self.tts_config = load_config(tts_config_path) self.use_phonemes = self.tts_config.use_phonemes self.ap = AudioProcessor(verbose=False, **self.tts_config.audio) - if "characters" in self.tts_config.keys(): + if self.tts_config.has('characters') and self.tts_config.characters: symbols, phonemes = make_symbols(**self.tts_config.characters) if self.use_phonemes: @@ -151,7 +150,7 @@ class Synthesizer(object): use_cuda (bool): enable/disable CUDA use. """ self.vocoder_config = load_config(model_config) - self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config["audio"]) + self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config.audio) self.vocoder_model = setup_generator(self.vocoder_config) self.vocoder_model.load_checkpoint(self.vocoder_config, model_file, eval=True) if use_cuda: From 87384c60085ec4e27879ac2995d0dd219e23d01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:24:34 +0200 Subject: [PATCH 62/87] get_device_id() for tests --- tests/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index f1445c92..c7930ef9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,16 @@ import os +from TTS.utils.generic_utils import get_cuda + + +def get_device_id(): + use_cuda, _ = get_cuda() + if use_cuda: + GPU_ID = "0" + else: + GPU_ID = "" + return GPU_ID + def get_tests_path(): """Returns the path to the test directory.""" From 5aee30443f1abfa117013a8c2f4e568948e68251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 15:27:23 +0200 Subject: [PATCH 63/87] refactoring tests after Coqpit --- run_bash_tests.sh | 8 +-- tests/inputs/test_config.json | 52 +++++++++---------- tests/inputs/test_speaker_encoder_config.json | 4 +- tests/outputs/dummy_model_config.json | 1 - tests/test_audio.py | 16 +++--- tests/test_glow_tts.py | 2 +- tests/test_loader.py | 10 ++-- tests/test_speaker_encoder_train.py | 11 ++-- tests/test_speaker_manager.py | 6 +-- tests/test_synthesize.py | 3 +- tests/test_synthesizer.py | 11 ++-- tests/test_tacotron2_model.py | 21 +++----- tests/test_tacotron2_tf_model.py | 2 +- tests/test_tacotron_model.py | 21 +++----- tests/tts_tests/test_align_tts_train.py | 7 +-- tests/tts_tests/test_glow_tts_train.py | 7 +-- tests/tts_tests/test_speedy_speech_train.py | 7 +-- tests/tts_tests/test_tacotron2_train.py | 7 +-- tests/tts_tests/test_tacotron_train.py | 7 +-- .../test_fullband_melgan_train.py | 7 +-- tests/vocoder_tests/test_hifigan_train.py | 7 +-- tests/vocoder_tests/test_melgan_train.py | 7 +-- .../test_multiband_melgan_train.py | 7 +-- .../test_parallel_wavegan_train.py | 7 +-- .../test_vocoder_gan_datasets.py | 11 ++-- tests/vocoder_tests/test_vocoder_losses.py | 3 +- .../test_vocoder_wavernn_datasets.py | 2 +- tests/vocoder_tests/test_wavegrad_train.py | 43 ++++++++------- tests/vocoder_tests/test_wavernn_train.py | 12 +++-- 29 files changed, 156 insertions(+), 153 deletions(-) diff --git a/run_bash_tests.sh b/run_bash_tests.sh index 16381611..feb9082b 100755 --- a/run_bash_tests.sh +++ b/run_bash_tests.sh @@ -2,13 +2,7 @@ set -e TF_CPP_MIN_LOG_LEVEL=3 # runtime bash based tests +# TODO: move these to python ./tests/bash_tests/test_demo_server.sh && \ ./tests/bash_tests/test_resample.sh && \ -./tests/bash_tests/test_tacotron_train.sh && \ -./tests/bash_tests/test_glow-tts_train.sh && \ -./tests/bash_tests/test_vocoder_gan_train.sh && \ -./tests/bash_tests/test_vocoder_wavernn_train.sh && \ -./tests/bash_tests/test_vocoder_wavegrad_train.sh && \ -./tests/bash_tests/test_speedy_speech_train.sh && \ -./tests/bash_tests/test_aligntts_train.sh && \ ./tests/bash_tests/test_compute_statistics.sh diff --git a/tests/inputs/test_config.json b/tests/inputs/test_config.json index 2fb52bb6..8f8810d1 100644 --- a/tests/inputs/test_config.json +++ b/tests/inputs/test_config.json @@ -1,24 +1,24 @@ { "audio":{ - "audio_processor": "audio", // to use dictate different audio processors, if available. - "num_mels": 80, // size of the mel spec frame. - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 22050, // wav sample-rate. If different than the original data, it is resampled. - "frame_length_ms": null, // stft window length in ms. - "frame_shift_ms": null, // stft window hop-lengh in ms. + "audio_processor": "audio", + "num_mels": 80, + "fft_size": 1024, + "sample_rate": 22050, + "frame_length_ms": null, + "frame_shift_ms": null, "hop_length": 256, "win_length": 1024, - "preemphasis": 0.97, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "min_level_db": -100, // normalization range - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 30,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - "signal_norm": true, // normalize the spec values in range [0, 1] - "symmetric_norm": true, // move normalization to range [-1, 1] - "clip_norm": true, // clip normalized values into the range. - "max_norm": 4, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "mel_fmin": 0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000, // maximum freq level for mel-spec. Tune for dataset!! + "preemphasis": 0.97, + "min_level_db": -100, + "ref_level_db": 20, + "power": 1.5, + "griffin_lim_iters": 30, + "signal_norm": true, + "symmetric_norm": true, + "clip_norm": true, + "max_norm": 4, + "mel_fmin": 0, + "mel_fmax": 8000, "do_trim_silence": false, "spec_gain": 20 }, @@ -53,15 +53,15 @@ "max_seq_len": 300, "log_dir": "tests/outputs/", - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": true, // use global style tokens - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_num_style_tokens). - "gst_use_speaker_embedding": true, // if true pass speaker embedding in attention input GST. + + "use_speaker_embedding": false, + "use_gst": true, + "gst": { + "gst_style_input": null, + + + + "gst_use_speaker_embedding": true, "gst_embedding_dim": 512, "gst_num_heads": 4, "gst_num_style_tokens": 10 diff --git a/tests/inputs/test_speaker_encoder_config.json b/tests/inputs/test_speaker_encoder_config.json index f1174e76..4f3678e1 100644 --- a/tests/inputs/test_speaker_encoder_config.json +++ b/tests/inputs/test_speaker_encoder_config.json @@ -1,5 +1,6 @@ { + "model": "speaker_encoder", "run_name": "test_speaker_encoder", "run_description": "test speaker encoder.", "audio":{ @@ -42,8 +43,9 @@ "checkpoint": true, // If true, it saves checkpoints per "save_step" "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. "print_step": 20, // Number of steps to log traning on console. + "batch_size": 32, "output_path": "", // DATASET-RELATED: output path for all training outputs. - "model": { + "model_params": { "input_dim": 40, "proj_dim": 256, "lstm_dim": 768, diff --git a/tests/outputs/dummy_model_config.json b/tests/outputs/dummy_model_config.json index 3996e09a..b51bb3a8 100644 --- a/tests/outputs/dummy_model_config.json +++ b/tests/outputs/dummy_model_config.json @@ -87,7 +87,6 @@ // MULTI-SPEAKER and GST "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": true, // use global style tokens "gst": { // gst parameter if gst is enabled "gst_style_input": null, // Condition the style input either on a // -> wave file [path to wave] or diff --git a/tests/test_audio.py b/tests/test_audio.py index 527defa8..7291a31f 100644 --- a/tests/test_audio.py +++ b/tests/test_audio.py @@ -3,21 +3,21 @@ import unittest from tests import get_tests_input_path, get_tests_output_path, get_tests_path from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.config import BaseAudioConfig TESTS_PATH = get_tests_path() OUT_PATH = os.path.join(get_tests_output_path(), "audio_tests") WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") os.makedirs(OUT_PATH, exist_ok=True) -conf = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +conf = BaseAudioConfig(mel_fmax=8000) # pylint: disable=protected-access class TestAudio(unittest.TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.ap = AudioProcessor(**conf.audio) + self.ap = AudioProcessor(**conf) def test_audio_synthesis(self): """1. load wav @@ -163,12 +163,12 @@ class TestAudio(unittest.TestCase): def test_scaler(self): scaler_stats_path = os.path.join(get_tests_input_path(), "scale_stats.npy") - conf.audio["stats_path"] = scaler_stats_path - conf.audio["preemphasis"] = 0.0 - conf.audio["do_trim_silence"] = True - conf.audio["signal_norm"] = True + conf.stats_path = scaler_stats_path + conf.preemphasis = 0.0 + conf.do_trim_silence = True + conf.signal_norm = True - ap = AudioProcessor(**conf.audio) + ap = AudioProcessor(**conf) mel_mean, mel_std, linear_mean, linear_std, _ = ap.load_stats(scaler_stats_path) ap.setup_scaler(mel_mean, mel_std, linear_mean, linear_std) diff --git a/tests/test_glow_tts.py b/tests/test_glow_tts.py index 77801b29..c1c8177b 100644 --- a/tests/test_glow_tts.py +++ b/tests/test_glow_tts.py @@ -17,7 +17,7 @@ torch.manual_seed(1) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +c = GlowTTSConfig() ap = AudioProcessor(**c.audio) WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") diff --git a/tests/test_loader.py b/tests/test_loader.py index 6174865b..ca2ac6eb 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -10,13 +10,17 @@ from tests import get_tests_input_path, get_tests_output_path from TTS.tts.datasets import TTSDataset from TTS.tts.datasets.preprocess import ljspeech from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.tts.configs import BaseTTSConfig # pylint: disable=unused-variable OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") os.makedirs(OUTPATH, exist_ok=True) -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) + +# create a dummy config for testing data loaders. +c = BaseTTSConfig(text_cleaner='english_cleaners', num_loader_workers=0, batch_size=2) +c.r = 5 +c.data_path = "tests/data/ljspeech/" ok_ljspeech = os.path.exists(c.data_path) DATA_EXIST = True @@ -40,7 +44,7 @@ class TestTTSDataset(unittest.TestCase): compute_linear_spec=True, ap=self.ap, meta_data=items, - tp=c.characters if "characters" in c.keys() else None, + tp=c.characters, batch_group_size=bgs, min_seq_len=c.min_seq_len, max_seq_len=float("inf"), diff --git a/tests/test_speaker_encoder_train.py b/tests/test_speaker_encoder_train.py index 0bf04966..1258f550 100644 --- a/tests/test_speaker_encoder_train.py +++ b/tests/test_speaker_encoder_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig from TTS.config.shared_configs import BaseAudioConfig @@ -15,9 +16,9 @@ config = SpeakerEncoderConfig( num_speakers_in_batch=1, num_utters_per_speaker=10, num_loader_workers=0, - max_train_step=10, + max_train_step=2, print_step=1, - save_step=10, + save_step=1, print_eval=True, audio=BaseAudioConfig(num_mels=40) ) @@ -27,7 +28,7 @@ config.save_json(config_path) # train the model for one epoch command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_encoder.py --config_path {config_path} " + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_encoder.py --config_path {config_path} " f"--coqpit.output_path {output_path} " "--coqpit.datasets.0.name ljspeech " "--coqpit.datasets.0.meta_file_train metadata.csv " @@ -41,6 +42,6 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_encoder.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_encoder.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/test_speaker_manager.py b/tests/test_speaker_manager.py index f8c742d9..31082f86 100644 --- a/tests/test_speaker_manager.py +++ b/tests/test_speaker_manager.py @@ -26,11 +26,11 @@ class SpeakerManagerTest(unittest.TestCase): def test_speaker_embedding(): # load config config = load_config(encoder_config_path) - config["audio"]["resample"] = True + config.audio.resample = True # create a dummy speaker encoder - model = SpeakerEncoder(**config.model) - save_checkpoint(model, None, None, get_tests_input_path(), 0, 0) + model = SpeakerEncoder(**config.model_params) + save_checkpoint(model, None, None, get_tests_input_path(), 0) # load audio processor and speaker encoder ap = AudioProcessor(**config.audio) diff --git a/tests/test_synthesize.py b/tests/test_synthesize.py index 526f7dc8..a8d5c31c 100644 --- a/tests/test_synthesize.py +++ b/tests/test_synthesize.py @@ -1,6 +1,7 @@ import os -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + def test_synthesize(): diff --git a/tests/test_synthesizer.py b/tests/test_synthesizer.py index 46997dbb..f29509c7 100644 --- a/tests/test_synthesizer.py +++ b/tests/test_synthesizer.py @@ -15,8 +15,8 @@ class SynthesizerTest(unittest.TestCase): # pylint: disable=global-statement global symbols, phonemes config = load_config(os.path.join(get_tests_output_path(), "dummy_model_config.json")) - if "characters" in config.keys(): - symbols, phonemes = make_symbols(**config.characters) + if config.has('characters') and config.characters: + symbols, phonemes = make_symbols(**config.characters.to_dict()) num_chars = len(phonemes) if config.use_phonemes else len(symbols) model = setup_model(num_chars, 0, config) @@ -25,11 +25,10 @@ class SynthesizerTest(unittest.TestCase): def test_in_out(self): self._create_random_model() - config = load_config(os.path.join(get_tests_input_path(), "server_config.json")) tts_root_path = get_tests_output_path() - config["tts_checkpoint"] = os.path.join(tts_root_path, config["tts_checkpoint"]) - config["tts_config"] = os.path.join(tts_root_path, config["tts_config"]) - synthesizer = Synthesizer(config["tts_checkpoint"], config["tts_config"], None, None) + tts_checkpoint = os.path.join(tts_root_path, 'checkpoint_10.pth.tar') + tts_config = os.path.join(tts_root_path, 'dummy_model_config.json') + synthesizer = Synthesizer(tts_checkpoint, tts_config, None, None) synthesizer.tts("Better this test works!!") def test_split_into_sentences(self): diff --git a/tests/test_tacotron2_model.py b/tests/test_tacotron2_model.py index 3a08e4d0..22af3384 100644 --- a/tests/test_tacotron2_model.py +++ b/tests/test_tacotron2_model.py @@ -17,7 +17,7 @@ torch.manual_seed(1) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +c = Tacotron2Config() ap = AudioProcessor(**c.audio) WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") @@ -152,10 +152,8 @@ class TacotronGSTTrainTest(unittest.TestCase): num_chars=24, r=c.r, num_speakers=5, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], + use_gst=True, + gst=c.gst ).to(device) model.train() model_ref = copy.deepcopy(model) @@ -216,10 +214,8 @@ class TacotronGSTTrainTest(unittest.TestCase): num_chars=24, r=c.r, num_speakers=5, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], + use_gst=True, + gst =c.gst ).to(device) model.train() model_ref = copy.deepcopy(model) @@ -280,11 +276,8 @@ class SCGSTMultiSpeakeTacotronTrainTest(unittest.TestCase): r=c.r, num_speakers=5, speaker_embedding_dim=55, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"], + use_gst=True, + gst=c.gst ).to(device) model.train() model_ref = copy.deepcopy(model) diff --git a/tests/test_tacotron2_tf_model.py b/tests/test_tacotron2_tf_model.py index aa9c1846..d8f88571 100644 --- a/tests/test_tacotron2_tf_model.py +++ b/tests/test_tacotron2_tf_model.py @@ -19,7 +19,7 @@ torch.manual_seed(1) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +c = Tacotron2Config() class TacotronTFTrainTest(unittest.TestCase): diff --git a/tests/test_tacotron_model.py b/tests/test_tacotron_model.py index d6f5189e..8142e23a 100644 --- a/tests/test_tacotron_model.py +++ b/tests/test_tacotron_model.py @@ -18,7 +18,7 @@ torch.manual_seed(1) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +c = TacotronConfig() ap = AudioProcessor(**c.audio) WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") @@ -175,10 +175,8 @@ class TacotronGSTTrainTest(unittest.TestCase): model = Tacotron( num_chars=32, num_speakers=5, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], + use_gst=True, + gst=c.gst, postnet_output_dim=c.audio["fft_size"], decoder_output_dim=c.audio["num_mels"], r=c.r, @@ -240,10 +238,8 @@ class TacotronGSTTrainTest(unittest.TestCase): model = Tacotron( num_chars=32, num_speakers=5, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], + use_gst=True, + gst=c.gst, postnet_output_dim=c.audio["fft_size"], decoder_output_dim=c.audio["num_mels"], r=c.r, @@ -306,11 +302,8 @@ class SCGSTMultiSpeakeTacotronTrainTest(unittest.TestCase): num_speakers=5, postnet_output_dim=c.audio["fft_size"], decoder_output_dim=c.audio["num_mels"], - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"], + use_gst=True, + gst=c.gst, r=c.r, memory_size=c.memory_size, speaker_embedding_dim=55, diff --git a/tests/tts_tests/test_align_tts_train.py b/tests/tts_tests/test_align_tts_train.py index aefc7dc3..c5fd098c 100644 --- a/tests/tts_tests/test_align_tts_train.py +++ b/tests/tts_tests/test_align_tts_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.tts.configs import AlignTTSConfig config_path = os.path.join(get_tests_output_path(), "test_model_config.json") @@ -30,7 +31,7 @@ config.save_json(config_path) # train the model for one epoch command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_align_tts.py --config_path {config_path} " + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_align_tts.py --config_path {config_path} " f"--coqpit.output_path {output_path} " "--coqpit.datasets.0.name ljspeech " "--coqpit.datasets.0.meta_file_train metadata.csv " @@ -43,6 +44,6 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_align_tts.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_align_tts.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_glow_tts_train.py b/tests/tts_tests/test_glow_tts_train.py index bb630aef..014fc5c4 100644 --- a/tests/tts_tests/test_glow_tts_train.py +++ b/tests/tts_tests/test_glow_tts_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.tts.configs import GlowTTSConfig config_path = os.path.join(get_tests_output_path(), "test_model_config.json") @@ -30,7 +31,7 @@ config.save_json(config_path) # train the model for one epoch command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_glow_tts.py --config_path {config_path} " + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_glow_tts.py --config_path {config_path} " f"--coqpit.output_path {output_path} " "--coqpit.datasets.0.name ljspeech " "--coqpit.datasets.0.meta_file_train metadata.csv " @@ -44,6 +45,6 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_glow_tts.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_glow_tts.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_speedy_speech_train.py b/tests/tts_tests/test_speedy_speech_train.py index 1b356985..a2384cb2 100644 --- a/tests/tts_tests/test_speedy_speech_train.py +++ b/tests/tts_tests/test_speedy_speech_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.tts.configs import SpeedySpeechConfig config_path = os.path.join(get_tests_output_path(), "test_speedy_speech_config.json") @@ -30,7 +31,7 @@ config.save_json(config_path) # train the model for one epoch command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_speedy_speech.py --config_path {config_path} " + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_speedy_speech.py --config_path {config_path} " f"--coqpit.output_path {output_path} " "--coqpit.datasets.0.name ljspeech " "--coqpit.datasets.0.meta_file_train metadata.csv " @@ -44,6 +45,6 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_speedy_speech.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_speedy_speech.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_tacotron2_train.py b/tests/tts_tests/test_tacotron2_train.py index 2ac17502..5743d581 100644 --- a/tests/tts_tests/test_tacotron2_train.py +++ b/tests/tts_tests/test_tacotron2_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.tts.configs import Tacotron2Config config_path = os.path.join(get_tests_output_path(), "test_model_config.json") @@ -31,7 +32,7 @@ config.save_json(config_path) # train the model for one epoch command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_tacotron.py --config_path {config_path} " + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --config_path {config_path} " f"--coqpit.output_path {output_path} " "--coqpit.datasets.0.name ljspeech " "--coqpit.datasets.0.meta_file_train metadata.csv " @@ -44,6 +45,6 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_tacotron_train.py b/tests/tts_tests/test_tacotron_train.py index b45e4a64..cd00a6f4 100644 --- a/tests/tts_tests/test_tacotron_train.py +++ b/tests/tts_tests/test_tacotron_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.tts.configs import TacotronConfig config_path = os.path.join(get_tests_output_path(), "test_model_config.json") @@ -30,7 +31,7 @@ config.save_json(config_path) # train the model for one epoch command_train = ( - f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_tacotron.py --config_path {config_path} " + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --config_path {config_path} " f"--coqpit.output_path {output_path} " "--coqpit.datasets.0.name ljspeech " "--coqpit.datasets.0.meta_file_train metadata.csv " @@ -43,6 +44,6 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_fullband_melgan_train.py b/tests/vocoder_tests/test_fullband_melgan_train.py index 64355af9..d052cc76 100644 --- a/tests/vocoder_tests/test_fullband_melgan_train.py +++ b/tests/vocoder_tests/test_fullband_melgan_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.vocoder.configs import FullbandMelganConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -28,13 +29,13 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_hifigan_train.py b/tests/vocoder_tests/test_hifigan_train.py index fa431eb3..1e0e303c 100644 --- a/tests/vocoder_tests/test_hifigan_train.py +++ b/tests/vocoder_tests/test_hifigan_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.vocoder.configs import HifiganConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -29,13 +30,13 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_melgan_train.py b/tests/vocoder_tests/test_melgan_train.py index b362ce86..bec7d5f5 100644 --- a/tests/vocoder_tests/test_melgan_train.py +++ b/tests/vocoder_tests/test_melgan_train.py @@ -2,9 +2,10 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id from TTS.vocoder.configs import MelganConfig + config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") output_path = os.path.join(get_tests_output_path(), "train_outputs") @@ -28,13 +29,13 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_multiband_melgan_train.py b/tests/vocoder_tests/test_multiband_melgan_train.py index bd2ae86f..583be8da 100644 --- a/tests/vocoder_tests/test_multiband_melgan_train.py +++ b/tests/vocoder_tests/test_multiband_melgan_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.vocoder.configs import MultibandMelganConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -28,13 +29,13 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_parallel_wavegan_train.py b/tests/vocoder_tests/test_parallel_wavegan_train.py index 5d89d069..73cfa39c 100644 --- a/tests/vocoder_tests/test_parallel_wavegan_train.py +++ b/tests/vocoder_tests/test_parallel_wavegan_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.vocoder.configs import ParallelWaveganConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -28,13 +29,13 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_vocoder_gan_datasets.py b/tests/vocoder_tests/test_vocoder_gan_datasets.py index 7202d06e..cbf6da77 100644 --- a/tests/vocoder_tests/test_vocoder_gan_datasets.py +++ b/tests/vocoder_tests/test_vocoder_gan_datasets.py @@ -3,17 +3,17 @@ import os import numpy as np from torch.utils.data import DataLoader -from tests import get_tests_input_path, get_tests_output_path, get_tests_path +from tests import get_tests_output_path, get_tests_path from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config from TTS.vocoder.datasets.gan_dataset import GANDataset from TTS.vocoder.datasets.preprocess import load_wav_data +from TTS.vocoder.configs import BaseGANVocoderConfig file_path = os.path.dirname(os.path.realpath(__file__)) OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") os.makedirs(OUTPATH, exist_ok=True) -C = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +C = BaseGANVocoderConfig() test_data_path = os.path.join(get_tests_path(), "data/ljspeech/") ok_ljspeech = os.path.exists(test_data_path) @@ -46,6 +46,8 @@ def gan_dataset_case( def check_item(feat, wav): """Pass a single pair of features and waveform""" + feat = feat.numpy() + wav = wav.numpy() expected_feat_shape = (batch_size, ap.num_mels, seq_len // hop_len + conv_pad * 2) # check shapes @@ -61,7 +63,7 @@ def gan_dataset_case( # the first 2 and the last 2 frames are skipped due to the padding # differences in stft max_diff = abs((feat - mel[:, : feat.shape[-1]])[:, 2:-2]).max() - assert max_diff <= 0, f" [!] {max_diff}" + assert max_diff <= 1e-6, f" [!] {max_diff}" # return random segments or return the whole audio if return_segments: @@ -69,7 +71,6 @@ def gan_dataset_case( for item1, item2 in loader: feat1, wav1 = item1 feat2, wav2 = item2 - check_item(feat1, wav1) check_item(feat2, wav2) count_iter += 1 diff --git a/tests/vocoder_tests/test_vocoder_losses.py b/tests/vocoder_tests/test_vocoder_losses.py index 915c5947..65b1fa86 100644 --- a/tests/vocoder_tests/test_vocoder_losses.py +++ b/tests/vocoder_tests/test_vocoder_losses.py @@ -14,8 +14,7 @@ os.makedirs(OUT_PATH, exist_ok=True) WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") -C = load_config(os.path.join(get_tests_input_path(), "test_config.json")) -ap = AudioProcessor(**C.audio) +ap = AudioProcessor(**BaseAudioConfig().to_dict()) def test_torch_stft(): diff --git a/tests/vocoder_tests/test_vocoder_wavernn_datasets.py b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py index 755d4772..588f529f 100644 --- a/tests/vocoder_tests/test_vocoder_wavernn_datasets.py +++ b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py @@ -14,7 +14,7 @@ file_path = os.path.dirname(os.path.realpath(__file__)) OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") os.makedirs(OUTPATH, exist_ok=True) -C = load_config(os.path.join(get_tests_input_path(), "test_vocoder_wavernn_config.json")) +C = WavernnConfig() test_data_path = os.path.join(get_tests_path(), "data/ljspeech/") test_mel_feat_path = os.path.join(test_data_path, "mel") diff --git a/tests/vocoder_tests/test_wavegrad_train.py b/tests/vocoder_tests/test_wavegrad_train.py index c2269bbd..b52715c7 100644 --- a/tests/vocoder_tests/test_wavegrad_train.py +++ b/tests/vocoder_tests/test_wavegrad_train.py @@ -2,39 +2,44 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.vocoder.configs import WavegradConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") output_path = os.path.join(get_tests_output_path(), "train_outputs") -config = WavegradConfig( - batch_size=8, - eval_batch_size=8, - num_loader_workers=0, - num_val_loader_workers=0, - run_eval=True, - test_delay_epochs=-1, - epochs=1, - seq_len=8192, - eval_split_size=1, - print_step=1, - print_eval=True, - data_path="tests/data/ljspeech", - output_path=output_path, -) +config = WavegradConfig(batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, + test_noise_schedule={ + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 2 + }) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavegrad.py --config_path {config_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavegrad.py --config_path {config_path} " run_cli(command_train) # Find latest folder -continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) +continue_path = max(glob.glob(os.path.join(output_path, "*/")), + key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavegrad.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavegrad.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_wavernn_train.py b/tests/vocoder_tests/test_wavernn_train.py index 1ac9d9eb..4597bb8f 100644 --- a/tests/vocoder_tests/test_wavernn_train.py +++ b/tests/vocoder_tests/test_wavernn_train.py @@ -2,7 +2,8 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli, get_device_id + from TTS.vocoder.configs import WavernnConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -16,7 +17,7 @@ config = WavernnConfig( run_eval=True, test_delay_epochs=-1, epochs=1, - seq_len=8192, + seq_len=256, # for shorter test time eval_split_size=1, print_step=1, print_eval=True, @@ -28,13 +29,14 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavernn.py --config_path {config_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavernn.py --config_path {config_path} " run_cli(command_train) # Find latest folder -continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) +continue_path = max(glob.glob(os.path.join(output_path, "*/")), + key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='' python TTS/bin/train_vocoder_wavernn.py --continue_path {continue_path} " +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavernn.py --continue_path {continue_path} " run_cli(command_train) shutil.rmtree(continue_path) From 19fb1d743d5fbd5e7d8df806bd86c5568f6b7d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 23:03:21 +0200 Subject: [PATCH 64/87] style update --- TTS/bin/train_encoder.py | 9 +--- TTS/config/__init__.py | 3 +- TTS/speaker_encoder/speaker_encoder_config.py | 42 +++++++++++------- TTS/speaker_encoder/utils/generic_utils.py | 10 +++-- TTS/speaker_encoder/utils/io.py | 9 ++-- TTS/tts/configs/shared_configs.py | 2 +- TTS/tts/utils/speakers.py | 2 +- TTS/utils/generic_utils.py | 3 +- TTS/utils/manage.py | 2 +- TTS/utils/synthesizer.py | 4 +- tests/test_audio.py | 2 +- tests/test_glow_tts.py | 2 +- tests/test_loader.py | 4 +- tests/test_speaker_encoder_train.py | 11 ++--- tests/test_speaker_manager.py | 6 +-- tests/test_synthesize.py | 3 +- tests/test_synthesizer.py | 8 ++-- tests/test_tacotron2_model.py | 29 +++--------- tests/test_tacotron2_tf_model.py | 3 +- tests/test_tacotron_model.py | 3 +- tests/test_text_processing.py | 3 +- tests/tts_tests/test_align_tts_train.py | 7 +-- tests/tts_tests/test_glow_tts_train.py | 7 +-- tests/tts_tests/test_speedy_speech_train.py | 7 +-- tests/tts_tests/test_tacotron2_train.py | 7 +-- tests/tts_tests/test_tacotron_train.py | 7 +-- .../test_fullband_melgan_train.py | 11 +++-- tests/vocoder_tests/test_hifigan_train.py | 11 +++-- tests/vocoder_tests/test_melgan_train.py | 11 +++-- .../test_multiband_melgan_train.py | 11 +++-- .../test_parallel_wavegan_train.py | 11 +++-- .../test_vocoder_gan_datasets.py | 2 +- tests/vocoder_tests/test_vocoder_losses.py | 2 +- tests/vocoder_tests/test_wavegrad_train.py | 44 +++++++++---------- tests/vocoder_tests/test_wavernn_train.py | 14 +++--- 35 files changed, 160 insertions(+), 152 deletions(-) diff --git a/TTS/bin/train_encoder.py b/TTS/bin/train_encoder.py index 6c1a03e8..70c3033b 100644 --- a/TTS/bin/train_encoder.py +++ b/TTS/bin/train_encoder.py @@ -16,16 +16,11 @@ from TTS.speaker_encoder.model import SpeakerEncoder from TTS.speaker_encoder.utils.io import save_best_model, save_checkpoint from TTS.speaker_encoder.utils.visual import plot_embeddings from TTS.tts.datasets.preprocess import load_meta_data +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor -from TTS.utils.generic_utils import ( - count_parameters, - remove_experiment_folder, - set_init_dict, -) +from TTS.utils.generic_utils import count_parameters, remove_experiment_folder, set_init_dict from TTS.utils.radam import RAdam from TTS.utils.training import NoamLR, check_update -from TTS.utils.arguments import init_training - torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True diff --git a/TTS/config/__init__.py b/TTS/config/__init__.py index 3ef37820..b4f1cbea 100644 --- a/TTS/config/__init__.py +++ b/TTS/config/__init__.py @@ -1,6 +1,7 @@ import json import os import re + import yaml from TTS.config.shared_configs import * @@ -34,7 +35,7 @@ def _search_configs(model_name): def _process_model_name(config_dict): model_name = config_dict["model"] if "model" in config_dict else config_dict["generator_model"] - model_name = model_name.replace('_generator', '').replace('_discriminator', '') + model_name = model_name.replace("_generator", "").replace("_discriminator", "") return model_name diff --git a/TTS/speaker_encoder/speaker_encoder_config.py b/TTS/speaker_encoder/speaker_encoder_config.py index 402ac892..dcba3b6c 100644 --- a/TTS/speaker_encoder/speaker_encoder_config.py +++ b/TTS/speaker_encoder/speaker_encoder_config.py @@ -1,7 +1,9 @@ -from coqpit import MISSING -from dataclasses import dataclass, field, asdict +from dataclasses import asdict, dataclass, field from typing import List -from TTS.config.shared_configs import BaseTrainingConfig, BaseAudioConfig, BaseDatasetConfig + +from coqpit import MISSING + +from TTS.config.shared_configs import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig @dataclass @@ -13,23 +15,27 @@ class SpeakerEncoderConfig(BaseTrainingConfig): datasets: List[BaseDatasetConfig] = field(default_factory=lambda: [BaseDatasetConfig()]) # model params - model_params: dict = field(default_factory=lambda: { - "input_dim": 40, - "proj_dim": 256, - "lstm_dim": 768, - "num_lstm_layers": 3, - "use_lstm_with_projection": True - }) + model_params: dict = field( + default_factory=lambda: { + "input_dim": 40, + "proj_dim": 256, + "lstm_dim": 768, + "num_lstm_layers": 3, + "use_lstm_with_projection": True, + } + ) - storage: dict = field(default_factory=lambda:{ - "sample_from_storage_p": 0.66, # the probability with which we'll sample from the DataSet in-memory storage - "storage_size": 15, # the size of the in-memory storage with respect to a single batch - "additive_noise": 1e-5 # add very small gaussian noise to the data in order to increase robustness - }) + storage: dict = field( + default_factory=lambda: { + "sample_from_storage_p": 0.66, # the probability with which we'll sample from the DataSet in-memory storage + "storage_size": 15, # the size of the in-memory storage with respect to a single batch + "additive_noise": 1e-5, # add very small gaussian noise to the data in order to increase robustness + } + ) # training params max_train_step: int = 1000 # end training when number of training steps reaches this value. - loss: str = 'angleproto' + loss: str = "angleproto" grad_clip: float = 3.0 lr: float = 0.0001 lr_decay: bool = False @@ -51,4 +57,6 @@ class SpeakerEncoderConfig(BaseTrainingConfig): def check_values(self): super().check_values() c = asdict(self) - assert c['model_params']['input_dim'] == self.audio.num_mels, " [!] model input dimendion must be equal to melspectrogram dimension." + assert ( + c["model_params"]["input_dim"] == self.audio.num_mels + ), " [!] model input dimendion must be equal to melspectrogram dimension." diff --git a/TTS/speaker_encoder/utils/generic_utils.py b/TTS/speaker_encoder/utils/generic_utils.py index 13296d64..eecf9086 100644 --- a/TTS/speaker_encoder/utils/generic_utils.py +++ b/TTS/speaker_encoder/utils/generic_utils.py @@ -9,6 +9,10 @@ def to_camel(text): def setup_model(c): - model = SpeakerEncoder(c.model_params["input_dim"], c.model_params["proj_dim"], - c.model_params["lstm_dim"], c.model_params["num_lstm_layers"]) - return model \ No newline at end of file + model = SpeakerEncoder( + c.model_params["input_dim"], + c.model_params["proj_dim"], + c.model_params["lstm_dim"], + c.model_params["num_lstm_layers"], + ) + return model diff --git a/TTS/speaker_encoder/utils/io.py b/TTS/speaker_encoder/utils/io.py index 8ba33c45..0479f1af 100644 --- a/TTS/speaker_encoder/utils/io.py +++ b/TTS/speaker_encoder/utils/io.py @@ -1,5 +1,6 @@ -import os import datetime +import os + import torch @@ -19,8 +20,7 @@ def save_checkpoint(model, optimizer, model_loss, out_path, current_step): torch.save(state, checkpoint_path) -def save_best_model(model, optimizer, model_loss, best_loss, out_path, - current_step): +def save_best_model(model, optimizer, model_loss, best_loss, out_path, current_step): if model_loss < best_loss: new_state_dict = model.state_dict() state = { @@ -33,7 +33,6 @@ def save_best_model(model, optimizer, model_loss, best_loss, out_path, best_loss = model_loss bestmodel_path = "best_model.pth.tar" bestmodel_path = os.path.join(out_path, bestmodel_path) - print("\n > BEST MODEL ({0:.5f}) : {1:}".format( - model_loss, bestmodel_path)) + print("\n > BEST MODEL ({0:.5f}) : {1:}".format(model_loss, bestmodel_path)) torch.save(state, bestmodel_path) return best_loss diff --git a/TTS/tts/configs/shared_configs.py b/TTS/tts/configs/shared_configs.py index 9f4b2b68..f3d0a528 100644 --- a/TTS/tts/configs/shared_configs.py +++ b/TTS/tts/configs/shared_configs.py @@ -41,7 +41,7 @@ class CharactersConfig(Coqpit): characters: str = None punctuations: str = None phonemes: str = None - unique: bool = True # for backwards compatibility of models trained with char sets with duplicates + unique: bool = True # for backwards compatibility of models trained with char sets with duplicates def check_values( self, diff --git a/TTS/tts/utils/speakers.py b/TTS/tts/utils/speakers.py index f92d3af5..84da1f72 100755 --- a/TTS/tts/utils/speakers.py +++ b/TTS/tts/utils/speakers.py @@ -6,9 +6,9 @@ from typing import Union import numpy as np import torch +from TTS.config import load_config from TTS.speaker_encoder.utils.generic_utils import setup_model from TTS.utils.audio import AudioProcessor -from TTS.config import load_config def make_speakers_json_path(out_path): diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index 5473d32d..709b2340 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -7,10 +7,11 @@ import re import shutil import subprocess import sys -import torch from pathlib import Path from typing import List +import torch + def get_cuda(): use_cuda = torch.cuda.is_available() diff --git a/TTS/utils/manage.py b/TTS/utils/manage.py index 87e499e7..790d6944 100644 --- a/TTS/utils/manage.py +++ b/TTS/utils/manage.py @@ -8,8 +8,8 @@ from shutil import copyfile import gdown import requests -from TTS.utils.generic_utils import get_user_data_dir from TTS.config import load_config +from TTS.utils.generic_utils import get_user_data_dir class ModelManager(object): diff --git a/TTS/utils/synthesizer.py b/TTS/utils/synthesizer.py index c7339d7b..bca3df31 100644 --- a/TTS/utils/synthesizer.py +++ b/TTS/utils/synthesizer.py @@ -5,6 +5,7 @@ import numpy as np import pysbd import torch +from TTS.config import load_config from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.speakers import SpeakerManager @@ -13,7 +14,6 @@ from TTS.tts.utils.speakers import SpeakerManager from TTS.tts.utils.synthesis import synthesis, trim_silence from TTS.tts.utils.text import make_symbols, phonemes, symbols from TTS.utils.audio import AudioProcessor -from TTS.config import load_config from TTS.vocoder.utils.generic_utils import interpolate_vocoder_input, setup_generator @@ -117,7 +117,7 @@ class Synthesizer(object): self.use_phonemes = self.tts_config.use_phonemes self.ap = AudioProcessor(verbose=False, **self.tts_config.audio) - if self.tts_config.has('characters') and self.tts_config.characters: + if self.tts_config.has("characters") and self.tts_config.characters: symbols, phonemes = make_symbols(**self.tts_config.characters) if self.use_phonemes: diff --git a/tests/test_audio.py b/tests/test_audio.py index 7291a31f..22e965f0 100644 --- a/tests/test_audio.py +++ b/tests/test_audio.py @@ -2,8 +2,8 @@ import os import unittest from tests import get_tests_input_path, get_tests_output_path, get_tests_path -from TTS.utils.audio import AudioProcessor from TTS.config import BaseAudioConfig +from TTS.utils.audio import AudioProcessor TESTS_PATH = get_tests_path() OUT_PATH = os.path.join(get_tests_output_path(), "audio_tests") diff --git a/tests/test_glow_tts.py b/tests/test_glow_tts.py index c1c8177b..07886e80 100644 --- a/tests/test_glow_tts.py +++ b/tests/test_glow_tts.py @@ -6,10 +6,10 @@ import torch from torch import optim from tests import get_tests_input_path +from TTS.tts.configs import GlowTTSConfig from TTS.tts.layers.losses import GlowTTSLoss from TTS.tts.models.glow_tts import GlowTTS from TTS.utils.audio import AudioProcessor -from TTS.tts.configs import GlowTTSConfig # pylint: disable=unused-variable diff --git a/tests/test_loader.py b/tests/test_loader.py index ca2ac6eb..96bf5993 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -7,10 +7,10 @@ import torch from torch.utils.data import DataLoader from tests import get_tests_input_path, get_tests_output_path +from TTS.tts.configs import BaseTTSConfig from TTS.tts.datasets import TTSDataset from TTS.tts.datasets.preprocess import ljspeech from TTS.utils.audio import AudioProcessor -from TTS.tts.configs import BaseTTSConfig # pylint: disable=unused-variable @@ -18,7 +18,7 @@ OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") os.makedirs(OUTPATH, exist_ok=True) # create a dummy config for testing data loaders. -c = BaseTTSConfig(text_cleaner='english_cleaners', num_loader_workers=0, batch_size=2) +c = BaseTTSConfig(text_cleaner="english_cleaners", num_loader_workers=0, batch_size=2) c.r = 5 c.data_path = "tests/data/ljspeech/" ok_ljspeech = os.path.exists(c.data_path) diff --git a/tests/test_speaker_encoder_train.py b/tests/test_speaker_encoder_train.py index 1258f550..ec777b6b 100644 --- a/tests/test_speaker_encoder_train.py +++ b/tests/test_speaker_encoder_train.py @@ -2,10 +2,9 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - -from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig +from tests import get_device_id, get_tests_output_path, run_cli from TTS.config.shared_configs import BaseAudioConfig +from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig config_path = os.path.join(get_tests_output_path(), "test_model_config.json") output_path = os.path.join(get_tests_output_path(), "train_outputs") @@ -20,7 +19,7 @@ config = SpeakerEncoderConfig( print_step=1, save_step=1, print_eval=True, - audio=BaseAudioConfig(num_mels=40) + audio=BaseAudioConfig(num_mels=40), ) config.audio.do_trim_silence = True config.audio.trim_db = 60 @@ -42,6 +41,8 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_encoder.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_encoder.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/test_speaker_manager.py b/tests/test_speaker_manager.py index 31082f86..9992dbc3 100644 --- a/tests/test_speaker_manager.py +++ b/tests/test_speaker_manager.py @@ -5,10 +5,10 @@ import numpy as np import torch from tests import get_tests_input_path -from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.speaker_encoder.utils.io import save_checkpoint -from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig from TTS.config import load_config +from TTS.speaker_encoder.model import SpeakerEncoder +from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig +from TTS.speaker_encoder.utils.io import save_checkpoint from TTS.tts.utils.speakers import SpeakerManager from TTS.utils.audio import AudioProcessor diff --git a/tests/test_synthesize.py b/tests/test_synthesize.py index a8d5c31c..ec15cb45 100644 --- a/tests/test_synthesize.py +++ b/tests/test_synthesize.py @@ -1,7 +1,6 @@ import os -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli def test_synthesize(): diff --git a/tests/test_synthesizer.py b/tests/test_synthesizer.py index f29509c7..9507e4f8 100644 --- a/tests/test_synthesizer.py +++ b/tests/test_synthesizer.py @@ -2,10 +2,10 @@ import os import unittest from tests import get_tests_input_path, get_tests_output_path +from TTS.config import load_config from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.io import save_checkpoint from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols -from TTS.config import load_config from TTS.utils.synthesizer import Synthesizer @@ -15,7 +15,7 @@ class SynthesizerTest(unittest.TestCase): # pylint: disable=global-statement global symbols, phonemes config = load_config(os.path.join(get_tests_output_path(), "dummy_model_config.json")) - if config.has('characters') and config.characters: + if config.has("characters") and config.characters: symbols, phonemes = make_symbols(**config.characters.to_dict()) num_chars = len(phonemes) if config.use_phonemes else len(symbols) @@ -26,8 +26,8 @@ class SynthesizerTest(unittest.TestCase): def test_in_out(self): self._create_random_model() tts_root_path = get_tests_output_path() - tts_checkpoint = os.path.join(tts_root_path, 'checkpoint_10.pth.tar') - tts_config = os.path.join(tts_root_path, 'dummy_model_config.json') + tts_checkpoint = os.path.join(tts_root_path, "checkpoint_10.pth.tar") + tts_config = os.path.join(tts_root_path, "dummy_model_config.json") synthesizer = Synthesizer(tts_checkpoint, tts_config, None, None) synthesizer.tts("Better this test works!!") diff --git a/tests/test_tacotron2_model.py b/tests/test_tacotron2_model.py index 22af3384..4d711700 100644 --- a/tests/test_tacotron2_model.py +++ b/tests/test_tacotron2_model.py @@ -6,10 +6,10 @@ import torch from torch import nn, optim from tests import get_tests_input_path +from TTS.tts.configs import Tacotron2Config from TTS.tts.layers.losses import MSELossMasked from TTS.tts.models.tacotron2 import Tacotron2 from TTS.utils.audio import AudioProcessor -from TTS.tts.configs import Tacotron2Config # pylint: disable=unused-variable @@ -148,13 +148,7 @@ class TacotronGSTTrainTest(unittest.TestCase): criterion = MSELossMasked(seq_len_norm=False).to(device) criterion_st = nn.BCEWithLogitsLoss().to(device) - model = Tacotron2( - num_chars=24, - r=c.r, - num_speakers=5, - use_gst=True, - gst=c.gst - ).to(device) + model = Tacotron2(num_chars=24, r=c.r, num_speakers=5, use_gst=True, gst=c.gst).to(device) model.train() model_ref = copy.deepcopy(model) count = 0 @@ -210,13 +204,7 @@ class TacotronGSTTrainTest(unittest.TestCase): criterion = MSELossMasked(seq_len_norm=False).to(device) criterion_st = nn.BCEWithLogitsLoss().to(device) - model = Tacotron2( - num_chars=24, - r=c.r, - num_speakers=5, - use_gst=True, - gst =c.gst - ).to(device) + model = Tacotron2(num_chars=24, r=c.r, num_speakers=5, use_gst=True, gst=c.gst).to(device) model.train() model_ref = copy.deepcopy(model) count = 0 @@ -271,14 +259,9 @@ class SCGSTMultiSpeakeTacotronTrainTest(unittest.TestCase): stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float().squeeze() criterion = MSELossMasked(seq_len_norm=False).to(device) criterion_st = nn.BCEWithLogitsLoss().to(device) - model = Tacotron2( - num_chars=24, - r=c.r, - num_speakers=5, - speaker_embedding_dim=55, - use_gst=True, - gst=c.gst - ).to(device) + model = Tacotron2(num_chars=24, r=c.r, num_speakers=5, speaker_embedding_dim=55, use_gst=True, gst=c.gst).to( + device + ) model.train() model_ref = copy.deepcopy(model) count = 0 diff --git a/tests/test_tacotron2_tf_model.py b/tests/test_tacotron2_tf_model.py index d8f88571..ee7f720b 100644 --- a/tests/test_tacotron2_tf_model.py +++ b/tests/test_tacotron2_tf_model.py @@ -5,10 +5,9 @@ import numpy as np import tensorflow as tf import torch +from TTS.tts.configs import Tacotron2Config from TTS.tts.tf.models.tacotron2 import Tacotron2 from TTS.tts.tf.utils.tflite import convert_tacotron2_to_tflite, load_tflite_model -from TTS.tts.configs import Tacotron2Config - tf.get_logger().setLevel("INFO") diff --git a/tests/test_tacotron_model.py b/tests/test_tacotron_model.py index 8142e23a..fcbac0f7 100644 --- a/tests/test_tacotron_model.py +++ b/tests/test_tacotron_model.py @@ -6,11 +6,10 @@ import torch from torch import nn, optim from tests import get_tests_input_path +from TTS.tts.configs import TacotronConfig from TTS.tts.layers.losses import L1LossMasked from TTS.tts.models.tacotron import Tacotron from TTS.utils.audio import AudioProcessor -from TTS.tts.configs import TacotronConfig - # pylint: disable=unused-variable diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py index b7b8c501..9d9bfafe 100644 --- a/tests/test_text_processing.py +++ b/tests/test_text_processing.py @@ -4,9 +4,8 @@ import os # pylint: disable=wildcard-import # pylint: disable=unused-import from tests import get_tests_input_path, get_tests_path -from TTS.tts.utils.text import * from TTS.tts.configs import TacotronConfig - +from TTS.tts.utils.text import * conf = TacotronConfig() diff --git a/tests/tts_tests/test_align_tts_train.py b/tests/tts_tests/test_align_tts_train.py index c5fd098c..97ffc7a7 100644 --- a/tests/tts_tests/test_align_tts_train.py +++ b/tests/tts_tests/test_align_tts_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.tts.configs import AlignTTSConfig config_path = os.path.join(get_tests_output_path(), "test_model_config.json") @@ -44,6 +43,8 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_align_tts.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_align_tts.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_glow_tts_train.py b/tests/tts_tests/test_glow_tts_train.py index 014fc5c4..a92d837f 100644 --- a/tests/tts_tests/test_glow_tts_train.py +++ b/tests/tts_tests/test_glow_tts_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.tts.configs import GlowTTSConfig config_path = os.path.join(get_tests_output_path(), "test_model_config.json") @@ -45,6 +44,8 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_glow_tts.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_glow_tts.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_speedy_speech_train.py b/tests/tts_tests/test_speedy_speech_train.py index a2384cb2..19d24ab3 100644 --- a/tests/tts_tests/test_speedy_speech_train.py +++ b/tests/tts_tests/test_speedy_speech_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.tts.configs import SpeedySpeechConfig config_path = os.path.join(get_tests_output_path(), "test_speedy_speech_config.json") @@ -45,6 +44,8 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_speedy_speech.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_speedy_speech.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_tacotron2_train.py b/tests/tts_tests/test_tacotron2_train.py index 5743d581..94e02646 100644 --- a/tests/tts_tests/test_tacotron2_train.py +++ b/tests/tts_tests/test_tacotron2_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.tts.configs import Tacotron2Config config_path = os.path.join(get_tests_output_path(), "test_model_config.json") @@ -45,6 +44,8 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_tacotron_train.py b/tests/tts_tests/test_tacotron_train.py index cd00a6f4..0f651f27 100644 --- a/tests/tts_tests/test_tacotron_train.py +++ b/tests/tts_tests/test_tacotron_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.tts.configs import TacotronConfig config_path = os.path.join(get_tests_output_path(), "test_model_config.json") @@ -44,6 +43,8 @@ run_cli(command_train) continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_fullband_melgan_train.py b/tests/vocoder_tests/test_fullband_melgan_train.py index d052cc76..d9bc51ac 100644 --- a/tests/vocoder_tests/test_fullband_melgan_train.py +++ b/tests/vocoder_tests/test_fullband_melgan_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.vocoder.configs import FullbandMelganConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -29,13 +28,17 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_hifigan_train.py b/tests/vocoder_tests/test_hifigan_train.py index 1e0e303c..11057570 100644 --- a/tests/vocoder_tests/test_hifigan_train.py +++ b/tests/vocoder_tests/test_hifigan_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.vocoder.configs import HifiganConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -30,13 +29,17 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_melgan_train.py b/tests/vocoder_tests/test_melgan_train.py index bec7d5f5..b9e3be7f 100644 --- a/tests/vocoder_tests/test_melgan_train.py +++ b/tests/vocoder_tests/test_melgan_train.py @@ -2,10 +2,9 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id +from tests import get_device_id, get_tests_output_path, run_cli from TTS.vocoder.configs import MelganConfig - config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") output_path = os.path.join(get_tests_output_path(), "train_outputs") @@ -29,13 +28,17 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_multiband_melgan_train.py b/tests/vocoder_tests/test_multiband_melgan_train.py index 583be8da..081fb40e 100644 --- a/tests/vocoder_tests/test_multiband_melgan_train.py +++ b/tests/vocoder_tests/test_multiband_melgan_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.vocoder.configs import MultibandMelganConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -29,13 +28,17 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_parallel_wavegan_train.py b/tests/vocoder_tests/test_parallel_wavegan_train.py index 73cfa39c..97d3c5f1 100644 --- a/tests/vocoder_tests/test_parallel_wavegan_train.py +++ b/tests/vocoder_tests/test_parallel_wavegan_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.vocoder.configs import ParallelWaveganConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -29,13 +28,17 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) run_cli(command_train) # Find latest folder continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_vocoder_gan_datasets.py b/tests/vocoder_tests/test_vocoder_gan_datasets.py index cbf6da77..c39d70e9 100644 --- a/tests/vocoder_tests/test_vocoder_gan_datasets.py +++ b/tests/vocoder_tests/test_vocoder_gan_datasets.py @@ -5,9 +5,9 @@ from torch.utils.data import DataLoader from tests import get_tests_output_path, get_tests_path from TTS.utils.audio import AudioProcessor +from TTS.vocoder.configs import BaseGANVocoderConfig from TTS.vocoder.datasets.gan_dataset import GANDataset from TTS.vocoder.datasets.preprocess import load_wav_data -from TTS.vocoder.configs import BaseGANVocoderConfig file_path = os.path.dirname(os.path.realpath(__file__)) OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") diff --git a/tests/vocoder_tests/test_vocoder_losses.py b/tests/vocoder_tests/test_vocoder_losses.py index 65b1fa86..2a35aa2e 100644 --- a/tests/vocoder_tests/test_vocoder_losses.py +++ b/tests/vocoder_tests/test_vocoder_losses.py @@ -3,8 +3,8 @@ import os import torch from tests import get_tests_input_path, get_tests_output_path, get_tests_path -from TTS.utils.audio import AudioProcessor from TTS.config import BaseAudioConfig +from TTS.utils.audio import AudioProcessor from TTS.vocoder.layers.losses import MelganFeatureLoss, MultiScaleSTFTLoss, STFTLoss, TorchSTFT TESTS_PATH = get_tests_path() diff --git a/tests/vocoder_tests/test_wavegrad_train.py b/tests/vocoder_tests/test_wavegrad_train.py index b52715c7..e222de3a 100644 --- a/tests/vocoder_tests/test_wavegrad_train.py +++ b/tests/vocoder_tests/test_wavegrad_train.py @@ -2,42 +2,40 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.vocoder.configs import WavegradConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") output_path = os.path.join(get_tests_output_path(), "train_outputs") -config = WavegradConfig(batch_size=8, - eval_batch_size=8, - num_loader_workers=0, - num_val_loader_workers=0, - run_eval=True, - test_delay_epochs=-1, - epochs=1, - seq_len=8192, - eval_split_size=1, - print_step=1, - print_eval=True, - data_path="tests/data/ljspeech", - output_path=output_path, - test_noise_schedule={ - "min_val": 1e-6, - "max_val": 1e-2, - "num_steps": 2 - }) +config = WavegradConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, + test_noise_schedule={"min_val": 1e-6, "max_val": 1e-2, "num_steps": 2}, +) config.audio.do_trim_silence = True config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavegrad.py --config_path {config_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavegrad.py --config_path {config_path} " +) run_cli(command_train) # Find latest folder -continue_path = max(glob.glob(os.path.join(output_path, "*/")), - key=os.path.getmtime) +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavegrad.py --continue_path {continue_path} " diff --git a/tests/vocoder_tests/test_wavernn_train.py b/tests/vocoder_tests/test_wavernn_train.py index 4597bb8f..414ed719 100644 --- a/tests/vocoder_tests/test_wavernn_train.py +++ b/tests/vocoder_tests/test_wavernn_train.py @@ -2,8 +2,7 @@ import glob import os import shutil -from tests import get_tests_output_path, run_cli, get_device_id - +from tests import get_device_id, get_tests_output_path, run_cli from TTS.vocoder.configs import WavernnConfig config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") @@ -29,14 +28,17 @@ config.audio.trim_db = 60 config.save_json(config_path) # train the model for one epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavernn.py --config_path {config_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavernn.py --config_path {config_path} " +) run_cli(command_train) # Find latest folder -continue_path = max(glob.glob(os.path.join(output_path, "*/")), - key=os.path.getmtime) +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) # restore the model and continue training for one more epoch -command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavernn.py --continue_path {continue_path} " +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavernn.py --continue_path {continue_path} " +) run_cli(command_train) shutil.rmtree(continue_path) From 843d1b3d98a791e7ffcfa3715a16898d899fa9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 10 May 2021 23:13:52 +0200 Subject: [PATCH 65/87] linter fixes --- .pylintrc | 2 +- TTS/bin/train_encoder.py | 1 - TTS/bin/train_tacotron.py | 4 ++-- TTS/config/shared_configs.py | 1 - TTS/server/server.py | 2 +- TTS/tts/configs/tacotron_config.py | 4 +--- TTS/utils/generic_utils.py | 1 - TTS/utils/io.py | 8 -------- TTS/vocoder/configs/fullband_melgan_config.py | 2 +- TTS/vocoder/configs/hifigan_config.py | 2 +- TTS/vocoder/configs/melgan_config.py | 2 +- TTS/vocoder/configs/multiband_melgan_config.py | 2 +- TTS/vocoder/configs/parallel_wavegan_config.py | 2 +- TTS/vocoder/configs/shared_configs.py | 3 +-- TTS/vocoder/configs/wavegrad_config.py | 2 +- TTS/vocoder/configs/wavernn_config.py | 2 +- tests/test_loader.py | 2 +- tests/test_speaker_encoder.py | 1 - tests/test_speaker_manager.py | 1 - tests/test_synthesize.py | 2 +- tests/test_synthesizer.py | 2 +- tests/test_text_processing.py | 3 --- tests/vocoder_tests/test_vocoder_wavernn_datasets.py | 2 +- 23 files changed, 17 insertions(+), 36 deletions(-) diff --git a/.pylintrc b/.pylintrc index 910a7a55..0bc0be4b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -563,7 +563,7 @@ max-branches=12 max-locals=15 # Maximum number of parents for a class (see R0901). -max-parents=7 +max-parents=15 # Maximum number of public methods for a class (see R0904). max-public-methods=20 diff --git a/TTS/bin/train_encoder.py b/TTS/bin/train_encoder.py index 70c3033b..66ecddfe 100644 --- a/TTS/bin/train_encoder.py +++ b/TTS/bin/train_encoder.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import argparse import os import sys import time diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index edf89858..9685d0d7 100755 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -641,7 +641,7 @@ def main(args): # pylint: disable=redefined-outer-name except (KeyError, RuntimeError): print(" > Partial model initialization...") model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint["model"], c) + model_dict = set_init_dict(model_dict, checkpoint["model"], config) model.load_state_dict(model_dict) del model_dict @@ -687,7 +687,7 @@ def main(args): # pylint: disable=redefined-outer-name c_logger.print_epoch_start(epoch, config.epochs) # set gradual training if config.gradual_training is not None: - r, config.batch_size = gradual_training_scheduler(global_step, c) + r, config.batch_size = gradual_training_scheduler(global_step, config) config.r = r model.decoder.set_r(r) if config.bidirectional_decoder: diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py index d9a4e06a..b10cc9bf 100644 --- a/TTS/config/shared_configs.py +++ b/TTS/config/shared_configs.py @@ -1,5 +1,4 @@ from dataclasses import asdict, dataclass -from typing import List, Union from coqpit import MISSING, Coqpit, check_argument diff --git a/TTS/server/server.py b/TTS/server/server.py index f6335c42..15a6b292 100644 --- a/TTS/server/server.py +++ b/TTS/server/server.py @@ -9,7 +9,7 @@ from typing import Union from flask import Flask, render_template, request, send_file -from TTS.utils.io import load_config +from TTS.config import load_config from TTS.utils.manage import ModelManager from TTS.utils.synthesizer import Synthesizer diff --git a/TTS/tts/configs/tacotron_config.py b/TTS/tts/configs/tacotron_config.py index 5c86f500..6f08e89f 100644 --- a/TTS/tts/configs/tacotron_config.py +++ b/TTS/tts/configs/tacotron_config.py @@ -1,8 +1,6 @@ -from dataclasses import asdict, dataclass +from dataclasses import dataclass from typing import List -from coqpit import check_argument - from .shared_configs import BaseTTSConfig, GSTConfig diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index 709b2340..a562e86f 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -8,7 +8,6 @@ import shutil import subprocess import sys from pathlib import Path -from typing import List import torch diff --git a/TTS/utils/io.py b/TTS/utils/io.py index b09a71d1..62d972f1 100644 --- a/TTS/utils/io.py +++ b/TTS/utils/io.py @@ -1,15 +1,7 @@ -import json import os import pickle as pickle_tts -import re from shutil import copyfile -import yaml - -from TTS.utils.generic_utils import find_module - -from .generic_utils import find_module - class RenamingUnpickler(pickle_tts.Unpickler): """Overload default pickler to solve module renaming problem""" diff --git a/TTS/vocoder/configs/fullband_melgan_config.py b/TTS/vocoder/configs/fullband_melgan_config.py index d206451f..7baa68ee 100644 --- a/TTS/vocoder/configs/fullband_melgan_config.py +++ b/TTS/vocoder/configs/fullband_melgan_config.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from .shared_configs import BaseGANVocoderConfig diff --git a/TTS/vocoder/configs/hifigan_config.py b/TTS/vocoder/configs/hifigan_config.py index 40b5fc26..cfbf5510 100644 --- a/TTS/vocoder/configs/hifigan_config.py +++ b/TTS/vocoder/configs/hifigan_config.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from .shared_configs import BaseGANVocoderConfig diff --git a/TTS/vocoder/configs/melgan_config.py b/TTS/vocoder/configs/melgan_config.py index f67c7d1e..cee64330 100644 --- a/TTS/vocoder/configs/melgan_config.py +++ b/TTS/vocoder/configs/melgan_config.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from .shared_configs import BaseGANVocoderConfig diff --git a/TTS/vocoder/configs/multiband_melgan_config.py b/TTS/vocoder/configs/multiband_melgan_config.py index f8a99152..98f1d353 100644 --- a/TTS/vocoder/configs/multiband_melgan_config.py +++ b/TTS/vocoder/configs/multiband_melgan_config.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from .shared_configs import BaseGANVocoderConfig diff --git a/TTS/vocoder/configs/parallel_wavegan_config.py b/TTS/vocoder/configs/parallel_wavegan_config.py index 79afa228..b8a489a3 100644 --- a/TTS/vocoder/configs/parallel_wavegan_config.py +++ b/TTS/vocoder/configs/parallel_wavegan_config.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from .shared_configs import BaseGANVocoderConfig diff --git a/TTS/vocoder/configs/shared_configs.py b/TTS/vocoder/configs/shared_configs.py index d403f84c..b335c36f 100644 --- a/TTS/vocoder/configs/shared_configs.py +++ b/TTS/vocoder/configs/shared_configs.py @@ -1,9 +1,8 @@ from dataclasses import dataclass, field -from typing import List from coqpit import MISSING -from TTS.config import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig +from TTS.config import BaseAudioConfig, BaseTrainingConfig @dataclass diff --git a/TTS/vocoder/configs/wavegrad_config.py b/TTS/vocoder/configs/wavegrad_config.py index 7638988f..48e3aba2 100644 --- a/TTS/vocoder/configs/wavegrad_config.py +++ b/TTS/vocoder/configs/wavegrad_config.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from .shared_configs import BaseVocoderConfig diff --git a/TTS/vocoder/configs/wavernn_config.py b/TTS/vocoder/configs/wavernn_config.py index daa586f6..0b546a91 100644 --- a/TTS/vocoder/configs/wavernn_config.py +++ b/TTS/vocoder/configs/wavernn_config.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from .shared_configs import BaseVocoderConfig diff --git a/tests/test_loader.py b/tests/test_loader.py index 96bf5993..e2dba37a 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -6,7 +6,7 @@ import numpy as np import torch from torch.utils.data import DataLoader -from tests import get_tests_input_path, get_tests_output_path +from tests import get_tests_output_path from TTS.tts.configs import BaseTTSConfig from TTS.tts.datasets import TTSDataset from TTS.tts.datasets.preprocess import ljspeech diff --git a/tests/test_speaker_encoder.py b/tests/test_speaker_encoder.py index 3e8dd947..8939ccf6 100644 --- a/tests/test_speaker_encoder.py +++ b/tests/test_speaker_encoder.py @@ -1,4 +1,3 @@ -import os import unittest import torch as T diff --git a/tests/test_speaker_manager.py b/tests/test_speaker_manager.py index 9992dbc3..ffb98ed7 100644 --- a/tests/test_speaker_manager.py +++ b/tests/test_speaker_manager.py @@ -7,7 +7,6 @@ import torch from tests import get_tests_input_path from TTS.config import load_config from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig from TTS.speaker_encoder.utils.io import save_checkpoint from TTS.tts.utils.speakers import SpeakerManager from TTS.utils.audio import AudioProcessor diff --git a/tests/test_synthesize.py b/tests/test_synthesize.py index ec15cb45..526f7dc8 100644 --- a/tests/test_synthesize.py +++ b/tests/test_synthesize.py @@ -1,6 +1,6 @@ import os -from tests import get_device_id, get_tests_output_path, run_cli +from tests import get_tests_output_path, run_cli def test_synthesize(): diff --git a/tests/test_synthesizer.py b/tests/test_synthesizer.py index 9507e4f8..a1cd4de5 100644 --- a/tests/test_synthesizer.py +++ b/tests/test_synthesizer.py @@ -1,7 +1,7 @@ import os import unittest -from tests import get_tests_input_path, get_tests_output_path +from tests import get_tests_output_path from TTS.config import load_config from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.io import save_checkpoint diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py index 9d9bfafe..711021ab 100644 --- a/tests/test_text_processing.py +++ b/tests/test_text_processing.py @@ -1,9 +1,6 @@ -import os - # pylint: disable=unused-wildcard-import # pylint: disable=wildcard-import # pylint: disable=unused-import -from tests import get_tests_input_path, get_tests_path from TTS.tts.configs import TacotronConfig from TTS.tts.utils.text import * diff --git a/tests/vocoder_tests/test_vocoder_wavernn_datasets.py b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py index 588f529f..503b4e24 100644 --- a/tests/vocoder_tests/test_vocoder_wavernn_datasets.py +++ b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py @@ -4,7 +4,7 @@ import shutil import numpy as np from torch.utils.data import DataLoader -from tests import get_tests_input_path, get_tests_output_path, get_tests_path +from tests import get_tests_output_path, get_tests_path from TTS.utils.audio import AudioProcessor from TTS.vocoder.configs import WavernnConfig from TTS.vocoder.datasets.preprocess import load_wav_feat_data, preprocess_wav_files From 715b0a65a091af1639742a5b71b2b2683b3f9473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 11 May 2021 02:42:22 +0200 Subject: [PATCH 66/87] update main.yml for python x64 fix test --- .github/workflows/main.yml | 1 + TTS/bin/compute_statistics.py | 2 +- TTS/bin/extract_tts_spectrograms.py | 5 ++--- TTS/utils/manage.py | 10 ++++------ tests/test_extract_tts_spectrograms.py | 2 +- tests/test_speaker_encoder_train.py | 1 - 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9fef1b84..74d5e85b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,6 +30,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + architecture: x64 - name: check OS run: cat /etc/os-release - name: Install dependencies diff --git a/TTS/bin/compute_statistics.py b/TTS/bin/compute_statistics.py index 2c13a960..37885fdd 100755 --- a/TTS/bin/compute_statistics.py +++ b/TTS/bin/compute_statistics.py @@ -41,7 +41,7 @@ def main(): if args.data_path: dataset_items = glob.glob(os.path.join(args.data_path, "**", "*.wav"), recursive=True) else: - dataset_items = load_meta_data(CONFIG.dataset_config)[0] # take only train data + dataset_items = load_meta_data(CONFIG.datasets)[0] # take only train data print(f" > There are {len(dataset_items)} files.") mel_sum = 0 diff --git a/TTS/bin/extract_tts_spectrograms.py b/TTS/bin/extract_tts_spectrograms.py index d5c23ccd..8a0a53ab 100755 --- a/TTS/bin/extract_tts_spectrograms.py +++ b/TTS/bin/extract_tts_spectrograms.py @@ -14,7 +14,7 @@ from TTS.tts.datasets.TTSDataset import MyDataset from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols -from TTS.utils.io import load_config +from TTS.config import load_config from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import count_parameters @@ -210,7 +210,7 @@ def main(args): # pylint: disable=redefined-outer-name # Audio processor ap = AudioProcessor(**c.audio) - if "characters" in c.keys(): + if "characters" in c.keys() and c['characters']: symbols, phonemes = make_symbols(**c.characters) # set model characters @@ -276,5 +276,4 @@ if __name__ == "__main__": args = parser.parse_args() c = load_config(args.config_path) - main(args) diff --git a/TTS/utils/manage.py b/TTS/utils/manage.py index 790d6944..fdc141ec 100644 --- a/TTS/utils/manage.py +++ b/TTS/utils/manage.py @@ -125,17 +125,15 @@ class ModelManager(object): # set scale stats path in config.json config_path = output_config_path config = load_config(config_path) - config["audio"]["stats_path"] = output_stats_path - with open(config_path, "w") as jf: - json.dump(config, jf) + config.audio.stats_path = output_stats_path + config.save_json(config_path) # update the speakers.json file path in the model config.json to the current path if os.path.exists(output_speakers_path): # set scale stats path in config.json config_path = output_config_path config = load_config(config_path) - config["external_speaker_embedding_file"] = output_speakers_path - with open(config_path, "w") as jf: - json.dump(config, jf) + config.external_speaker_embedding_file = output_speakers_path + config.save_json(config_path) return output_model_path, output_config_path, model_item def _download_gdrive_file(self, gdrive_idx, output): diff --git a/tests/test_extract_tts_spectrograms.py b/tests/test_extract_tts_spectrograms.py index 65db9c0e..94044eeb 100644 --- a/tests/test_extract_tts_spectrograms.py +++ b/tests/test_extract_tts_spectrograms.py @@ -9,7 +9,7 @@ from tests import get_tests_output_path, run_cli from TTS.tts.utils.generic_utils import setup_model -from TTS.utils.io import load_config +from TTS.config import load_config from TTS.tts.utils.text.symbols import phonemes, symbols torch.manual_seed(1) diff --git a/tests/test_speaker_encoder_train.py b/tests/test_speaker_encoder_train.py index ec777b6b..525730f2 100644 --- a/tests/test_speaker_encoder_train.py +++ b/tests/test_speaker_encoder_train.py @@ -33,7 +33,6 @@ command_train = ( "--coqpit.datasets.0.meta_file_train metadata.csv " "--coqpit.datasets.0.meta_file_val metadata.csv " "--coqpit.datasets.0.path tests/data/ljspeech " - "--coqpit.datasets.0.meta_file_attn_mask tests/data/ljspeech/metadata_attn_mask.txt" ) run_cli(command_train) From 8058aaa304190721ac08e538b83e8a416cd3089d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 11 May 2021 10:17:15 +0200 Subject: [PATCH 67/87] pin numba==0.52 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index da04b6b6..fafd5112 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ soundfile tensorboardX torch>=1.7 tqdm +numba==0.52 umap-learn==0.4.6 unidecode==0.4.20 coqpit From 0213e1cbf424f1b5ed1299a47c9d24062e1410df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 12 May 2021 00:56:25 +0200 Subject: [PATCH 68/87] update configs for tts models to match the field typed with the expected values --- TTS/config/shared_configs.py | 4 ++-- TTS/tts/configs/tacotron_config.py | 2 +- tests/inputs/test_tacotron2_config.json | 2 +- tests/inputs/test_tacotron_config.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py index b10cc9bf..153b3279 100644 --- a/TTS/config/shared_configs.py +++ b/TTS/config/shared_configs.py @@ -1,5 +1,5 @@ from dataclasses import asdict, dataclass - +from typing import List, Union from coqpit import MISSING, Coqpit, check_argument @@ -137,7 +137,7 @@ class BaseAudioConfig(Coqpit): class BaseDatasetConfig(Coqpit): name: str = None path: str = None - meta_file_train: str = None + meta_file_train: Union[str, List] = None # TODO: don't take ignored speakers for multi-speaker datasets over this. This is Union for SC-Glow compat. meta_file_val: str = None meta_file_attn_mask: str = None diff --git a/TTS/tts/configs/tacotron_config.py b/TTS/tts/configs/tacotron_config.py index 6f08e89f..5c509927 100644 --- a/TTS/tts/configs/tacotron_config.py +++ b/TTS/tts/configs/tacotron_config.py @@ -14,7 +14,7 @@ class TacotronConfig(BaseTTSConfig): gst_style_input: str = None # model specific params r: int = 2 - gradual_training: List = None + gradual_training: List[List] = None memory_size: int = -1 prenet_type: str = "original" prenet_dropout: bool = True diff --git a/tests/inputs/test_tacotron2_config.json b/tests/inputs/test_tacotron2_config.json index 779f925d..2bf1f840 100644 --- a/tests/inputs/test_tacotron2_config.json +++ b/tests/inputs/test_tacotron2_config.json @@ -64,7 +64,7 @@ "batch_size": 1, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. "eval_batch_size":1, "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 4]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. + "gradual_training": [[0, 7, 4], [1, 5, 2]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. "loss_masking": true, // enable / disable loss masking against the sequence padding. "ga_alpha": 10.0, // weight for guided attention loss. If > 0, guided attention is enabled. "mixed_precision": false, diff --git a/tests/inputs/test_tacotron_config.json b/tests/inputs/test_tacotron_config.json index a2fdd690..12da4762 100644 --- a/tests/inputs/test_tacotron_config.json +++ b/tests/inputs/test_tacotron_config.json @@ -64,7 +64,7 @@ "batch_size": 1, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. "eval_batch_size":1, "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 4]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. + "gradual_training": [[0, 7, 4], [1, 5, 2]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. "loss_masking": true, // enable / disable loss masking against the sequence padding. "ga_alpha": 10.0, // weight for guided attention loss. If > 0, guided attention is enabled. "mixed_precision": false, From da49089a7246bf75c82b05afa8038d672268997e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 12 May 2021 10:12:11 +0200 Subject: [PATCH 69/87] update melgan training test batch size --- TTS/speaker_encoder/losses.py | 4 ++-- tests/vocoder_tests/test_melgan_train.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/TTS/speaker_encoder/losses.py b/TTS/speaker_encoder/losses.py index 69264ab4..1b6b6e43 100644 --- a/TTS/speaker_encoder/losses.py +++ b/TTS/speaker_encoder/losses.py @@ -23,7 +23,7 @@ class GE2ELoss(nn.Module): self.b = nn.Parameter(torch.tensor(init_b)) self.loss_method = loss_method - print(" > Initialised Generalized End-to-End loss") + print(" > Initialized Generalized End-to-End loss") assert self.loss_method in ["softmax", "contrast"] @@ -136,7 +136,7 @@ class AngleProtoLoss(nn.Module): self.b = nn.Parameter(torch.tensor(init_b)) self.criterion = torch.nn.CrossEntropyLoss() - print(" > Initialised Angular Prototypical loss") + print(" > Initialized Angular Prototypical loss") def forward(self, x): """ diff --git a/tests/vocoder_tests/test_melgan_train.py b/tests/vocoder_tests/test_melgan_train.py index b9e3be7f..de48ca24 100644 --- a/tests/vocoder_tests/test_melgan_train.py +++ b/tests/vocoder_tests/test_melgan_train.py @@ -9,14 +9,14 @@ config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") output_path = os.path.join(get_tests_output_path(), "train_outputs") config = MelganConfig( - batch_size=8, - eval_batch_size=8, + batch_size=4, + eval_batch_size=4, num_loader_workers=0, num_val_loader_workers=0, run_eval=True, test_delay_epochs=-1, epochs=1, - seq_len=8192, + seq_len=2048, eval_split_size=1, print_step=1, print_eval=True, From 7e02cff924ba3ebc377899555f68247d22baea49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 12 May 2021 16:10:08 +0200 Subject: [PATCH 70/87] reduce pwgan test batch size --- tests/vocoder_tests/test_parallel_wavegan_train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/vocoder_tests/test_parallel_wavegan_train.py b/tests/vocoder_tests/test_parallel_wavegan_train.py index 97d3c5f1..fb6ea87c 100644 --- a/tests/vocoder_tests/test_parallel_wavegan_train.py +++ b/tests/vocoder_tests/test_parallel_wavegan_train.py @@ -9,14 +9,14 @@ config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") output_path = os.path.join(get_tests_output_path(), "train_outputs") config = ParallelWaveganConfig( - batch_size=8, - eval_batch_size=8, + batch_size=4, + eval_batch_size=4, num_loader_workers=0, num_val_loader_workers=0, run_eval=True, test_delay_epochs=-1, epochs=1, - seq_len=8192, + seq_len=2048, eval_split_size=1, print_step=1, print_eval=True, From 8b1014d188ab2ca28670f622794c51bcbe702f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Thu, 13 May 2021 16:04:49 +0200 Subject: [PATCH 71/87] add docstrings with default value fixes --- TTS/config/shared_configs.py | 55 ++++----- TTS/tts/configs/align_tts_config.py | 62 +++++++++- TTS/tts/configs/glow_tts_config.py | 59 ++++++++- TTS/tts/configs/shared_configs.py | 78 +++++++++++- TTS/tts/configs/speedy_speech_config.py | 69 ++++++++++- TTS/tts/configs/tacotron2_config.py | 110 ++++++++++++++++- TTS/tts/configs/tacotron_config.py | 112 +++++++++++++++++- TTS/tts/datasets/preprocess.py | 10 +- TTS/vocoder/configs/fullband_melgan_config.py | 59 ++++++++- TTS/vocoder/configs/hifigan_config.py | 87 +++++++++++++- TTS/vocoder/configs/melgan_config.py | 59 ++++++++- .../configs/multiband_melgan_config.py | 94 +++++++++++++-- .../configs/parallel_wavegan_config.py | 72 ++++++++++- TTS/vocoder/configs/shared_configs.py | 105 +++++++++++++++- TTS/vocoder/configs/wavegrad_config.py | 64 +++++++++- TTS/vocoder/configs/wavernn_config.py | 71 ++++++++++- 16 files changed, 1091 insertions(+), 75 deletions(-) diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py index 153b3279..7df1582f 100644 --- a/TTS/config/shared_configs.py +++ b/TTS/config/shared_configs.py @@ -13,7 +13,7 @@ class BaseAudioConfig(Coqpit): Number of STFT frequency levels aka.size of the linear spectogram frame. Defaults to 1024. win_length (int): Each frame of audio is windowed by window of length ```win_length``` and then padded with zeros to match - ```fft_size```. Defaults to 256. + ```fft_size```. Defaults to 1024. hop_length (int): Number of audio samples between adjacent STFT columns. Defaults to 1024. frame_shift_ms (int): @@ -21,7 +21,7 @@ class BaseAudioConfig(Coqpit): frame_length_ms (int): Set ```win_length``` based on milliseconds and sampling rate. stft_pad_mode (str): - Padding method used in STFT. 'reflect' or 'center'. + Padding method used in STFT. 'reflect' or 'center'. Defaults to 'reflect'. sample_rate (int): Audio sampling rate. Defaults to 22050. resample (bool): @@ -135,11 +135,27 @@ class BaseAudioConfig(Coqpit): @dataclass class BaseDatasetConfig(Coqpit): - name: str = None - path: str = None - meta_file_train: Union[str, List] = None # TODO: don't take ignored speakers for multi-speaker datasets over this. This is Union for SC-Glow compat. - meta_file_val: str = None - meta_file_attn_mask: str = None + """Base config for TTS datasets. + + Args: + name (str): + Dataset name that defines the preprocessor in use. Defaults to None. + path (str): + Root path to the dataset files. Defaults to None. + meta_file_train (Union[str, List]): + Name of the dataset meta file. Or a list of speakers to be ignored at training for multi-speaker datasets. + Defaults to None. + meta_file_val (str): + Name of the dataset meta file that defines the instances used at validation. + meta_file_attn_mask (str): + Path to the file that lists the attention mask files used with models that require attention masks to + train the duration predictor. + """ + name: str = '' + path: str = '' + meta_file_train: Union[str, List] = '' # TODO: don't take ignored speakers for multi-speaker datasets over this. This is Union for SC-Glow compat. + meta_file_val: str = '' + meta_file_attn_mask: str = '' def check_values( self, @@ -161,12 +177,8 @@ class BaseTrainingConfig(Coqpit): Args: batch_size (int): Training batch size. - batch_group_size (int): - Number of batches to shuffle after bucketing. eval_batch_size (int): Validation batch size. - loss_masking (bool): - Enable / Disable masking padding segments of sequences. mixed_precision (bool): Enable / Disable mixed precision training. It reduces the VRAM use and allows larger batch sizes, however it may also cause numerical unstability in some cases. @@ -195,34 +207,13 @@ class BaseTrainingConfig(Coqpit): keep_after (int): Number of steps to wait before saving all the best models. In use if ```keep_all_best == True```. Defaults to 10000. - text_cleaner (str): - Text cleaner to be used at model training. It is set to be one of the cleaners in - ```TTS.tts.utils.text.cleaners```. - enable_eos_bos_chars (bool): - Enable / Disable using special characters indicating end-of-sentence and begining-of-sentence. num_loader_workers (int): Number of workers for training time dataloader. num_val_loader_workers (int): Number of workers for evaluation time dataloader. - min_seq_len (int): - Minimum sequence length to be used at training. - max_seq_len (int): - Maximum sequence length to be used at training. VRAM use at training depends on this parameter. Consider to - decrease it if you get OOM errors. - compute_f0 (bool): - Return F0 frames from the dataloader. Defaults to ```False```. - compute_input_seq_cache (bool): - Enable / Disable computing and caching phonemes sequences from character sequences at the begining of the - training. It allows faster data loading times and more precise max-min sequence prunning. Defaults - to ```False```. output_path (str): Path for training output folder. The nonexist part of the given path is created automatically. All training outputs are saved there. - phoneme_cache_path (str): - Path to a folder to save the computed phoneme sequences. - datasets (List[BaseDatasetConfig]): - ist of DatasetConfig. - """ model: str = None diff --git a/TTS/tts/configs/align_tts_config.py b/TTS/tts/configs/align_tts_config.py index 6e09e398..84e0ba13 100644 --- a/TTS/tts/configs/align_tts_config.py +++ b/TTS/tts/configs/align_tts_config.py @@ -1,11 +1,69 @@ from dataclasses import dataclass, field -from .shared_configs import BaseTTSConfig +from TTS.tts.configs.shared_configs import BaseTTSConfig @dataclass class AlignTTSConfig(BaseTTSConfig): - """Defines parameters for AlignTTS model.""" + """Defines parameters for AlignTTS model. + Example: + + >>> from TTS.tts.configs import AlignTTSConfig + >>> config = AlignTTSConfig() + + Args: + model(str): + Model name used for selecting the right model at initialization. Defaults to `align_tts`. + positional_encoding (bool): + enable / disable positional encoding applied to the encoder output. Defaults to True. + hidden_channels (int): + Base number of hidden channels. Defines all the layers expect ones defined by the specific encoder or decoder + parameters. Defaults to 256. + hidden_channels_dp (int): + Number of hidden channels of the duration predictor's layers. Defaults to 256. + encoder_type (str): + Type of the encoder used by the model. Look at `TTS.tts.layers.feed_forward.encoder` for more details. + Defaults to `fftransformer`. + encoder_params (dict): + Parameters used to define the encoder network. Look at `TTS.tts.layers.feed_forward.encoder` for more details. + Defaults to `{"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}`. + decoder_type (str): + Type of the decoder used by the model. Look at `TTS.tts.layers.feed_forward.decoder` for more details. + Defaults to `fftransformer`. + decoder_params (dict): + Parameters used to define the decoder network. Look at `TTS.tts.layers.feed_forward.decoder` for more details. + Defaults to `{"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}`. + phase_start_steps (List[int]): + A list of number of steps required to start the next training phase. AlignTTS has 4 different training + phases. Thus you need to define 4 different values to enable phase based training. If None, it + trains the whole model together. Defaults to None. + ssim_alpha (float): + Weight for the SSIM loss. If set <= 0, disables the SSIM loss. Defaults to 1.0. + duration_loss_alpha (float): + Weight for the duration predictor's loss. Defaults to 1.0. + mdn_alpha (float): + Weight for the MDN loss. Defaults to 1.0. + spec_loss_alpha (float): + Weight for the MSE spectrogram loss. If set <= 0, disables the L1 loss. Defaults to 1.0. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-3`. + wd (float): + Weight decay coefficient. Defaults to `1e-7`. + min_seq_len (int): + Minimum input sequence length to be used at training. + max_seq_len (int): + Maximum input sequence length to be used at training. Larger values result in more VRAM usage.""" model: str = "align_tts" # model specific params diff --git a/TTS/tts/configs/glow_tts_config.py b/TTS/tts/configs/glow_tts_config.py index 8474caae..c0eadb1f 100644 --- a/TTS/tts/configs/glow_tts_config.py +++ b/TTS/tts/configs/glow_tts_config.py @@ -1,11 +1,64 @@ from dataclasses import dataclass, field -from .shared_configs import BaseTTSConfig +from TTS.tts.configs.shared_configs import BaseTTSConfig @dataclass class GlowTTSConfig(BaseTTSConfig): - """Defines parameters for GlowTTS model.""" + """Defines parameters for GlowTTS model. + + Example: + + >>> from TTS.tts.configs import GlowTTSConfig + >>> config = GlowTTSConfig() + + Args: + model(str): + Model name used for selecting the right model at initialization. Defaults to `glow_tts`. + encoder_type (str): + Type of the encoder used by the model. Look at `TTS.tts.layers.glow_tts.encoder` for more details. + Defaults to `rel_pos_transformers`. + encoder_params (dict): + Parameters used to define the encoder network. Look at `TTS.tts.layers.glow_tts.encoder` for more details. + Defaults to `{"kernel_size": 3, "dropout_p": 0.1, "num_layers": 6, "num_heads": 2, "hidden_channels_ffn": 768}` + use_encoder_prenet (bool): + enable / disable the use of a prenet for the encoder. Defaults to True. + hidden_channels_encoder (int): + Number of base hidden channels used by the encoder network. It defines the input and the output channel sizes, + and for some encoder types internal hidden channels sizes too. Defaults to 192. + hidden_channels_decoder (int): + Number of base hidden channels used by the decoder WaveNet network. Defaults to 192 as in the original work. + hidden_channels_duration_predictor (int): + Number of layer channels of the duration predictor network. Defaults to 256 as in the original work. + data_dep_init_steps (int): + Number of steps used for computing normalization parameters at the beginning of the training. GlowTTS uses + Activation Normalization that pre-computes normalization stats at the beginning and use the same values + for the rest. Defaults to 10. + style_wav_for_test (str): + Path to the wav file used for changing the style of the speech. Defaults to None. + inference_noise_scale (float): + Variance used for sampling the random noise added to the decoder's input at inference. Defaults to 0.0. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-3`. + wd (float): + Weight decay coefficient. Defaults to `1e-7`. + min_seq_len (int): + Minimum input sequence length to be used at training. + max_seq_len (int): + Maximum input sequence length to be used at training. Larger values result in more VRAM usage. + """ + model: str = "glow_tts" @@ -47,4 +100,4 @@ class GlowTTSConfig(BaseTTSConfig): # overrides min_seq_len: int = 3 max_seq_len: int = 500 - r: int = 1 + r: int = 1 # DO NOT CHANGE - TODO: make this immutable once coqpit implements it. diff --git a/TTS/tts/configs/shared_configs.py b/TTS/tts/configs/shared_configs.py index f3d0a528..885896c7 100644 --- a/TTS/tts/configs/shared_configs.py +++ b/TTS/tts/configs/shared_configs.py @@ -8,8 +8,20 @@ from TTS.config import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig @dataclass class GSTConfig(Coqpit): - """Defines Global Style Toke module""" + """Defines the Global Style Token Module + Args: + gst_style_input_wav (str): + Path to the wav file used to define the style of the output speech at inference. Defaults to None. + gst_style_input_weights (dict): + Defines the weights for each style token used at inference. Defaults to None. + gst_embedding_dim (int): + Defines the size of the GST embedding vector dimensions. Defaults to 256. + gst_num_heads (int): + Number of attention heads used by the multi-head attention. Defaults to 4. + gst_num_style_tokens (int): + Number of style token vectors. Defaults to 10. + """ gst_style_input_wav: str = None gst_style_input_weights: dict = None gst_embedding_dim: int = 256 @@ -33,7 +45,26 @@ class GSTConfig(Coqpit): @dataclass class CharactersConfig(Coqpit): - """Defines character or phoneme set used by the model""" + """Defines character or phoneme set used by the model + + Args: + pad (str): + characters in place of empty padding. Defaults to None. + eos (str): + characters showing the end of a sentence. Defaults to None. + bos (str): + characters showing the beginning of a sentence. Defaults to None. + characters (str): + character set used by the model. Characters not in this list are ignored when converting input text to + a list of sequence IDs. Defaults to None. + punctuations (str): + characters considered as punctuation as parsing the input sentence. Defaults to None. + phonemes (str): + characters considered as parsing phonemes. Defaults to None. + unique (bool): + remove any duplicate characters in the character lists. It is a bandaid for compatibility with the old + models trained with character lists with duplicates. + """ pad: str = None eos: str = None @@ -58,7 +89,48 @@ class CharactersConfig(Coqpit): @dataclass class BaseTTSConfig(BaseTrainingConfig): - """Shared parameters among all the tts models.""" + """Shared parameters among all the tts models. + + Args: + audio (BaseAudioConfig): + Audio processor config object instance. + use_phonemes (bool): + enable / disable phoneme use. + compute_input_seq_cache (bool): + enable / disable precomputation of the phoneme sequences. At the expense of some delay at the beginning of + the training, It allows faster data loader time and precise limitation with `max_seq_len` and + `min_seq_len`. + text_cleaner (str): + Name of the text cleaner used for cleaning and formatting transcripts. + enable_eos_bos_chars (bool): + enable / disable the use of eos and bos characters. + test_senteces_file (str): + Path to a txt file that has sentences used at test time. The file must have a sentence per line. + phoneme_cache_path (str): + Path to the output folder caching the computed phonemes for each sample. + characters (CharactersConfig): + Instance of a CharactersConfig class. + batch_group_size (int): + Size of the batch groups used for bucketing. By default, the dataloader orders samples by the sequence + length for a more efficient and stable training. If `batch_group_size > 1` then it performs bucketing to + prevent using the same batches for each epoch. + loss_masking (bool): + enable / disable masking loss values against padded segments of samples in a batch. + min_seq_len (int): + Minimum input sequence length to be used at training. + max_seq_len (int): + Maximum input sequence length to be used at training. Larger values result in more VRAM usage. + compute_f0 (int): + (Not in use yet). + use_noise_augment (bool): + Augment the input audio with random noise. + add_blank (bool): + Add blank characters between each other two characters. It improves performance for some models at expense + of slower run-time due to the longer input sequence. + datasets (List[BaseDatasetConfig]): + List of datasets used for training. If multiple datasets are provided, they are merged and used together + for training. + """ audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) # phoneme settings diff --git a/TTS/tts/configs/speedy_speech_config.py b/TTS/tts/configs/speedy_speech_config.py index a2e90cb8..85df543d 100644 --- a/TTS/tts/configs/speedy_speech_config.py +++ b/TTS/tts/configs/speedy_speech_config.py @@ -1,11 +1,74 @@ from dataclasses import dataclass, field -from .shared_configs import BaseTTSConfig +from TTS.tts.configs.shared_configs import BaseTTSConfig @dataclass class SpeedySpeechConfig(BaseTTSConfig): - """Defines parameters for Speedy Speech (feed-forward encoder-decoder) based models.""" + """Defines parameters for Speedy Speech (feed-forward encoder-decoder) based models. + + Example: + + >>> from TTS.tts.configs import SpeedySpeechConfig + >>> config = SpeedySpeechConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `speedy_speech`. + positional_encoding (bool): + enable / disable positional encoding applied to the encoder output. Defaults to True. + hidden_channels (int): + Base number of hidden channels. Defines all the layers expect ones defined by the specific encoder or decoder + parameters. Defaults to 128. + encoder_type (str): + Type of the encoder used by the model. Look at `TTS.tts.layers.feed_forward.encoder` for more details. + Defaults to `residual_conv_bn`. + encoder_params (dict): + Parameters used to define the encoder network. Look at `TTS.tts.layers.feed_forward.encoder` for more details. + Defaults to `{"kernel_size": 4, "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1], "num_conv_blocks": 2, "num_res_blocks": 13}` + decoder_type (str): + Type of the decoder used by the model. Look at `TTS.tts.layers.feed_forward.decoder` for more details. + Defaults to `residual_conv_bn`. + decoder_params (dict): + Parameters used to define the decoder network. Look at `TTS.tts.layers.feed_forward.decoder` for more details. + Defaults to `{"kernel_size": 4, "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1], "num_conv_blocks": 2, "num_res_blocks": 17}` + hidden_channels_encoder (int): + Number of base hidden channels used by the encoder network. It defines the input and the output channel sizes, + and for some encoder types internal hidden channels sizes too. Defaults to 192. + hidden_channels_decoder (int): + Number of base hidden channels used by the decoder WaveNet network. Defaults to 192 as in the original work. + hidden_channels_duration_predictor (int): + Number of layer channels of the duration predictor network. Defaults to 256 as in the original work. + data_dep_init_steps (int): + Number of steps used for computing normalization parameters at the beginning of the training. GlowTTS uses + Activation Normalization that pre-computes normalization stats at the beginning and use the same values + for the rest. Defaults to 10. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-3`. + wd (float): + Weight decay coefficient. Defaults to `1e-7`. + ssim_alpha (float): + Weight for the SSIM loss. If set <= 0, disables the SSIM loss. Defaults to 1.0. + huber_alpha (float): + Weight for the duration predictor's loss. Defaults to 1.0. + l1_alpha (float): + Weight for the L1 spectrogram loss. If set <= 0, disables the L1 loss. Defaults to 1.0. + min_seq_len (int): + Minimum input sequence length to be used at training. + max_seq_len (int): + Maximum input sequence length to be used at training. Larger values result in more VRAM usage. + """ model: str = "speedy_speech" # model specific params @@ -50,4 +113,4 @@ class SpeedySpeechConfig(BaseTTSConfig): # overrides min_seq_len: int = 13 max_seq_len: int = 200 - r: int = 1 + r: int = 1 #DO NOT CHANGE diff --git a/TTS/tts/configs/tacotron2_config.py b/TTS/tts/configs/tacotron2_config.py index e6767d41..ea66fae8 100644 --- a/TTS/tts/configs/tacotron2_config.py +++ b/TTS/tts/configs/tacotron2_config.py @@ -5,6 +5,114 @@ from TTS.tts.configs.tacotron_config import TacotronConfig @dataclass class Tacotron2Config(TacotronConfig): - """Defines parameters for Tacotron2 based models.""" + """Defines parameters for Tacotron2 based models. + + Example: + + >>> from TTS.tts.configs import Tacotron2Config + >>> config = Tacotron2Config() + + Args: + model (str): + Model name used to select the right model class to initilize. Defaults to `Tacotron2`. + use_gst (bool): + enable / disable the use of Global Style Token modules. Defaults to False. + gst (GSTConfig): + Instance of `GSTConfig` class. + gst_style_input (str): + Path to the wav file used at inference to set the speech style through GST. If `GST` is enabled and + this is not defined, the model uses a zero vector as an input. Defaults to None. + r (int): + Number of output frames that the decoder computed per iteration. Larger values makes training and inference + faster but reduces the quality of the output frames. This needs to be tuned considering your own needs. + Defaults to 1. + gradual_trainin (List[List]): + Parameters for the gradual training schedule. It is in the form `[[a, b, c], [d ,e ,f] ..]` where `a` is + the step number to start using the rest of the values, `b` is the `r` value and `c` is the batch size. + If sets None, no gradual training is used. Defaults to None. + memory_size (int): + Defines the number of previous frames used by the Prenet. If set to < 0, then it uses only the last frame. + Defaults to -1. + prenet_type (str): + `original` or `bn`. `original` sets the default Prenet and `bn` uses Batch Normalization version of the + Prenet. Defaults to `original`. + prenet_dropout (bool): + enables / disables the use of dropout in the Prenet. Defaults to True. + prenet_dropout_at_inference (bool): + enable / disable the use of dropout in the Prenet at the inference time. Defaults to False. + stopnet (bool): + enable /disable the Stopnet that predicts the end of the decoder sequence. Defaults to True. + stopnet_pos_weight (float): + Weight that is applied to over-weight positive instances in the Stopnet loss. Use larger values with + datasets with longer sentences. Defaults to 10. + separate_stopnet (bool): + Use a distinct Stopnet which is trained separately from the rest of the model. Defaults to True. + attention_type (str): + attention type. Check ```TTS.tts.layers.attentions.init_attn```. Defaults to 'original'. + attention_heads (int): + Number of attention heads for GMM attention. Defaults to 5. + windowing (bool): + It especially useful at inference to keep attention alignment diagonal. Defaults to False. + use_forward_attn (bool): + It is only valid if ```attn_type``` is ```original```. Defaults to False. + forward_attn_mask (bool): + enable/disable extra masking over forward attention. It is useful at inference to prevent + possible attention failures. Defaults to False. + transition_agent (bool): + enable/disable transition agent in forward attention. Defaults to False. + location_attn (bool): + enable/disable location sensitive attention as in the original Tacotron2 paper. + It is only valid if ```attn_type``` is ```original```. Defaults to True. + bidirectional_decoder (bool): + enable/disable bidirectional decoding. Defaults to False. + double_decoder_consistency (bool): + enable/disable double decoder consistency. Defaults to False. + ddc_r (int): + reduction rate used by the coarse decoder when `double_decoder_consistency` is in use. Set this + as a multiple of the `r` value. Defaults to 6. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-4`. + wd (float): + Weight decay coefficient. Defaults to `1e-6`. + grad_clip (float): + Gradient clipping threshold. Defaults to `5`. + seq_len_notm (bool): + enable / disable the sequnce length normalization in the loss functions. If set True, loss of a sample + is divided by the sequence length. Defaults to False. + loss_masking (bool): + enable / disable masking the paddings of the samples in loss computation. Defaults to True. + decoder_loss_alpha (float): + Weight for the decoder loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_loss_alpha (float): + Weight for the postnet loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_diff_spec_alpha (float): + Weight for the postnet differential loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + decoder_diff_spec_alpha (float): + Weight for the decoder differential loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + decoder_ssim_alpha (float): + Weight for the decoder SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_ssim_alpha (float): + Weight for the postnet SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + ga_alpha (float): + Weight for the guided attention loss. If set less than or equal to zero, it disables the corresponding loss + function. Defaults to 5. + """ model: str = "tacotron2" diff --git a/TTS/tts/configs/tacotron_config.py b/TTS/tts/configs/tacotron_config.py index 5c509927..53f5739e 100644 --- a/TTS/tts/configs/tacotron_config.py +++ b/TTS/tts/configs/tacotron_config.py @@ -1,12 +1,120 @@ from dataclasses import dataclass from typing import List -from .shared_configs import BaseTTSConfig, GSTConfig +from TTS.tts.configs.shared_configs import BaseTTSConfig, GSTConfig @dataclass class TacotronConfig(BaseTTSConfig): - """Defines parameters for Tacotron based models.""" + """Defines parameters for Tacotron based models. + + Example: + + >>> from TTS.tts.configs import TacotronConfig + >>> config = TacotronConfig() + + Args: + model (str): + Model name used to select the right model class to initilize. Defaults to `Tacotron`. + use_gst (bool): + enable / disable the use of Global Style Token modules. Defaults to False. + gst (GSTConfig): + Instance of `GSTConfig` class. + gst_style_input (str): + Path to the wav file used at inference to set the speech style through GST. If `GST` is enabled and + this is not defined, the model uses a zero vector as an input. Defaults to None. + r (int): + Number of output frames that the decoder computed per iteration. Larger values makes training and inference + faster but reduces the quality of the output frames. This needs to be tuned considering your own needs. + Defaults to 1. + gradual_trainin (List[List]): + Parameters for the gradual training schedule. It is in the form `[[a, b, c], [d ,e ,f] ..]` where `a` is + the step number to start using the rest of the values, `b` is the `r` value and `c` is the batch size. + If sets None, no gradual training is used. Defaults to None. + memory_size (int): + Defines the number of previous frames used by the Prenet. If set to < 0, then it uses only the last frame. + Defaults to -1. + prenet_type (str): + `original` or `bn`. `original` sets the default Prenet and `bn` uses Batch Normalization version of the + Prenet. Defaults to `original`. + prenet_dropout (bool): + enables / disables the use of dropout in the Prenet. Defaults to True. + prenet_dropout_at_inference (bool): + enable / disable the use of dropout in the Prenet at the inference time. Defaults to False. + stopnet (bool): + enable /disable the Stopnet that predicts the end of the decoder sequence. Defaults to True. + stopnet_pos_weight (float): + Weight that is applied to over-weight positive instances in the Stopnet loss. Use larger values with + datasets with longer sentences. Defaults to 10. + separate_stopnet (bool): + Use a distinct Stopnet which is trained separately from the rest of the model. Defaults to True. + attention_type (str): + attention type. Check ```TTS.tts.layers.attentions.init_attn```. Defaults to 'original'. + attention_heads (int): + Number of attention heads for GMM attention. Defaults to 5. + windowing (bool): + It especially useful at inference to keep attention alignment diagonal. Defaults to False. + use_forward_attn (bool): + It is only valid if ```attn_type``` is ```original```. Defaults to False. + forward_attn_mask (bool): + enable/disable extra masking over forward attention. It is useful at inference to prevent + possible attention failures. Defaults to False. + transition_agent (bool): + enable/disable transition agent in forward attention. Defaults to False. + location_attn (bool): + enable/disable location sensitive attention as in the original Tacotron2 paper. + It is only valid if ```attn_type``` is ```original```. Defaults to True. + bidirectional_decoder (bool): + enable/disable bidirectional decoding. Defaults to False. + double_decoder_consistency (bool): + enable/disable double decoder consistency. Defaults to False. + ddc_r (int): + reduction rate used by the coarse decoder when `double_decoder_consistency` is in use. Set this + as a multiple of the `r` value. Defaults to 6. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-4`. + wd (float): + Weight decay coefficient. Defaults to `1e-6`. + grad_clip (float): + Gradient clipping threshold. Defaults to `5`. + seq_len_notm (bool): + enable / disable the sequnce length normalization in the loss functions. If set True, loss of a sample + is divided by the sequence length. Defaults to False. + loss_masking (bool): + enable / disable masking the paddings of the samples in loss computation. Defaults to True. + decoder_loss_alpha (float): + Weight for the decoder loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_loss_alpha (float): + Weight for the postnet loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_diff_spec_alpha (float): + Weight for the postnet differential loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + decoder_diff_spec_alpha (float): + Weight for the decoder differential loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + decoder_ssim_alpha (float): + Weight for the decoder SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_ssim_alpha (float): + Weight for the postnet SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + ga_alpha (float): + Weight for the guided attention loss. If set less than or equal to zero, it disables the corresponding loss + function. Defaults to 5. + """ model: str = "tacotron" use_gst: bool = False diff --git a/TTS/tts/datasets/preprocess.py b/TTS/tts/datasets/preprocess.py index 4523d70b..72ab160e 100644 --- a/TTS/tts/datasets/preprocess.py +++ b/TTS/tts/datasets/preprocess.py @@ -52,19 +52,19 @@ def load_meta_data(datasets, eval_split=True): print(f" | > Found {len(meta_data_train)} files in {Path(root_path).resolve()}") # load evaluation split if set if eval_split: - if meta_file_val is None: - meta_data_eval, meta_data_train = split_dataset(meta_data_train) - else: + if meta_file_val: meta_data_eval = preprocessor(root_path, meta_file_val) + else: + meta_data_eval, meta_data_train = split_dataset(meta_data_train) meta_data_eval_all += meta_data_eval meta_data_train_all += meta_data_train # load attention masks for duration predictor training - if dataset.meta_file_attn_mask is not None: + if dataset.meta_file_attn_mask: meta_data = dict(load_attention_mask_meta_data(dataset["meta_file_attn_mask"])) for idx, ins in enumerate(meta_data_train_all): attn_file = meta_data[ins[1]].strip() meta_data_train_all[idx].append(attn_file) - if meta_data_eval_all is not None: + if meta_data_eval_all: for idx, ins in enumerate(meta_data_eval_all): attn_file = meta_data[ins[1]].strip() meta_data_eval_all[idx].append(attn_file) diff --git a/TTS/vocoder/configs/fullband_melgan_config.py b/TTS/vocoder/configs/fullband_melgan_config.py index 7baa68ee..53444214 100644 --- a/TTS/vocoder/configs/fullband_melgan_config.py +++ b/TTS/vocoder/configs/fullband_melgan_config.py @@ -5,7 +5,62 @@ from .shared_configs import BaseGANVocoderConfig @dataclass class FullbandMelganConfig(BaseGANVocoderConfig): - """Defines parameters for FullbandMelGAN vocoder.""" + """Defines parameters for FullBand MelGAN vocoder. + + Example: + + >>> from TTS.vocoder.configs import FullbandMelganConfig + >>> config = FullbandMelganConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `melgan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'melgan_multiscale_discriminator`. + discriminator_model_params (dict): The discriminator model parameters. Defaults to + '{"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]}` + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `melgan_generator`. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + use_stft_loss (bool): + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): STFT loss parameters. Default to + `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + """ model: str = "melgan" @@ -48,4 +103,4 @@ class FullbandMelganConfig(BaseGANVocoderConfig): mse_G_loss_weight: float = 2.5 hinge_G_loss_weight: float = 0 feat_match_loss_weight: float = 108 - l1_spec_loss_weight: float = 0 + l1_spec_loss_weight: float = 0.0 diff --git a/TTS/vocoder/configs/hifigan_config.py b/TTS/vocoder/configs/hifigan_config.py index cfbf5510..f76bb14c 100644 --- a/TTS/vocoder/configs/hifigan_config.py +++ b/TTS/vocoder/configs/hifigan_config.py @@ -1,11 +1,94 @@ from dataclasses import dataclass, field -from .shared_configs import BaseGANVocoderConfig +from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig @dataclass class HifiganConfig(BaseGANVocoderConfig): - """Defines parameters for HifiGAN vocoder.""" + """Defines parameters for FullBand MelGAN vocoder. + + Example: + + >>> from TTS.vocoder.configs import HifiganConfig + >>> config = HifiganConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `hifigan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'hifigan_discriminator`. + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `hifigan_generator`. + generator_model_params (dict): Parameters of the generator model. Defaults to + ` + { + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None, + } + ` + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + use_stft_loss (bool): + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): + STFT loss parameters. Default to + `{ + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }` + l1_spec_loss_params (dict): + L1 spectrogram loss parameters. Default to + `{ + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None, + }` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + """ model: str = "hifigan" # model specific params diff --git a/TTS/vocoder/configs/melgan_config.py b/TTS/vocoder/configs/melgan_config.py index cee64330..dc35b6f8 100644 --- a/TTS/vocoder/configs/melgan_config.py +++ b/TTS/vocoder/configs/melgan_config.py @@ -1,11 +1,66 @@ from dataclasses import dataclass, field -from .shared_configs import BaseGANVocoderConfig +from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig @dataclass class MelganConfig(BaseGANVocoderConfig): - """Defines parameters for MelGAN vocoder.""" + """Defines parameters for MelGAN vocoder. + + Example: + + >>> from TTS.vocoder.configs import MelganConfig + >>> config = MelganConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `melgan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'melgan_multiscale_discriminator`. + discriminator_model_params (dict): The discriminator model parameters. Defaults to + '{"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]}` + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `melgan_generator`. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + use_stft_loss (bool): + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): STFT loss parameters. Default to + `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + """ model: str = "melgan" diff --git a/TTS/vocoder/configs/multiband_melgan_config.py b/TTS/vocoder/configs/multiband_melgan_config.py index 98f1d353..f7aabdb3 100644 --- a/TTS/vocoder/configs/multiband_melgan_config.py +++ b/TTS/vocoder/configs/multiband_melgan_config.py @@ -1,11 +1,95 @@ from dataclasses import dataclass, field -from .shared_configs import BaseGANVocoderConfig +from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig @dataclass class MultibandMelganConfig(BaseGANVocoderConfig): - """Defines parameters for MultiBandMelGAN vocoder.""" + """Defines parameters for MultiBandMelGAN vocoder. + + Example: + + >>> from TTS.vocoder.configs import MultibandMelganConfig + >>> config = MultibandMelganConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `melgan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'melgan_multiscale_discriminator`. + discriminator_model_params (dict): The discriminator model parameters. Defaults to + '{ + "base_channels": 16, + "max_channels": 512, + "downsample_factors": [4, 4, 4] + }` + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `melgan_generator`. + generator_model_param (dict): + The generator model parameters. Defaults to `{"upsample_factors": [8, 4, 2], "num_res_blocks": 4}`. + use_pqmf (bool): + enable / disable PQMF modulation for multi-band training. Defaults to True. + lr_gen (float): + Initial learning rate for the generator model. Defaults to 0.0001. + lr_disc (float): + Initial learning rate for the discriminator model. Defaults to 0.0001. + optimizer (torch.optim.Optimizer): + Optimizer used for the training. Defaults to `AdamW`. + optimizer_params (dict): + Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}` + lr_scheduler_gen (torch.optim.Scheduler): + Learning rate scheduler for the generator. Defaults to `MultiStepLR`. + lr_scheduler_gen_params (dict): + Parameters for the generator learning rate scheduler. Defaults to + `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}`. + lr_scheduler_disc (torch.optim.Scheduler): + Learning rate scheduler for the discriminator. Defaults to `MultiStepLR`. + lr_scheduler_dict_params (dict): + Parameters for the discriminator learning rate scheduler. Defaults to + `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}`. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + steps_to_start_discriminator (int): + Number of steps required to start training the discriminator. Defaults to 0. + use_stft_loss (bool):` + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): STFT loss parameters. Default to + `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + """ + model: str = "multiband_melgan" @@ -58,8 +142,4 @@ class MultibandMelganConfig(BaseGANVocoderConfig): mse_G_loss_weight: float = 2.5 hinge_G_loss_weight: float = 0 feat_match_loss_weight: float = 108 - l1_spec_loss_weight: float = 0 - - # optimizer parameters - lr: float = 1e-4 - wd: float = 1e-6 + l1_spec_loss_weight: float = 0 \ No newline at end of file diff --git a/TTS/vocoder/configs/parallel_wavegan_config.py b/TTS/vocoder/configs/parallel_wavegan_config.py index b8a489a3..d132d2e1 100644 --- a/TTS/vocoder/configs/parallel_wavegan_config.py +++ b/TTS/vocoder/configs/parallel_wavegan_config.py @@ -5,7 +5,77 @@ from .shared_configs import BaseGANVocoderConfig @dataclass class ParallelWaveganConfig(BaseGANVocoderConfig): - """Defines parameters for ParallelWavegan vocoder.""" + """Defines parameters for ParallelWavegan vocoder. + + Args: + model (str): + Model name used for selecting the right configuration at initialization. Defaults to `parallel_wavegan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'parallel_wavegan_discriminator`. + discriminator_model_params (dict): The discriminator model kwargs. Defaults to + '{"num_layers": 10}` + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `parallel_wavegan_generator`. + generator_model_param (dict): + The generator model kwargs. Defaults to `{"upsample_factors": [4, 4, 4, 4], "stacks": 3, "num_res_blocks": 30}`. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + steps_to_start_discriminator (int): + Number of steps required to start training the discriminator. Defaults to 0. + use_stft_loss (bool):` + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): STFT loss parameters. Default to + `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 0. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + lr_gen (float): + Generator model initial learning rate. Defaults to 0.0002. + lr_disc (float): + Discriminator model initial learning rate. Defaults to 0.0002. + optimizer (torch.optim.Optimizer): + Optimizer used for the training. Defaults to `AdamW`. + optimizer_params (dict): + Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}` + lr_scheduler_gen (torch.optim.Scheduler): + Learning rate scheduler for the generator. Defaults to `ExponentialLR`. + lr_scheduler_gen_params (dict): + Parameters for the generator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`. + lr_scheduler_disc (torch.optim.Scheduler): + Learning rate scheduler for the discriminator. Defaults to `ExponentialLR`. + lr_scheduler_dict_params (dict): + Parameters for the discriminator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`. + """ model: str = "parallel_wavegan" diff --git a/TTS/vocoder/configs/shared_configs.py b/TTS/vocoder/configs/shared_configs.py index b335c36f..664032d2 100644 --- a/TTS/vocoder/configs/shared_configs.py +++ b/TTS/vocoder/configs/shared_configs.py @@ -7,7 +7,34 @@ from TTS.config import BaseAudioConfig, BaseTrainingConfig @dataclass class BaseVocoderConfig(BaseTrainingConfig): - """Shared parameters among all the vocoder models.""" + """Shared parameters among all the vocoder models. + Args: + audio (BaseAudioConfig): + Audio processor config instance. Defaultsto `BaseAudioConfig()`. + use_noise_augment (bool): + Augment the input audio with random noise. Defaults to False/ + eval_split_size (int): + Number of instances used for evaluation. Defaults to 10. + data_path (str): + Root path of the training data. All the audio files found recursively from this root path are used for + training. Defaults to MISSING. + feature_path (str): + Root path to the precomputed feature files. Defaults to None. + seq_len (int): + Length of the waveform segments used for training. Defaults to MISSING. + pad_short (int): + Extra padding for the waveforms shorter than `seq_len`. Defaults to 0. + conv_path (int): + Extra padding for the feature frames against convolution of the edge frames. Defaults to MISSING. + Defaults to 0. + use_cache (bool): + enable / disable in memory caching of the computed features. If the RAM is not enough, if may cause OOM. + Defaults to False. + epochs (int): + Number of training epochs to. Defaults to 10000. + wd (float): + Weight decay. + """ audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) # dataloading @@ -19,7 +46,6 @@ class BaseVocoderConfig(BaseTrainingConfig): seq_len: int = MISSING # signal length used in training. pad_short: int = 0 # additional padding for short wavs conv_pad: int = 0 # additional padding against convolutions applied to spectrograms - use_noise_augment: bool = False # add noise to the audio signal for augmentation use_cache: bool = False # use in memory cache to keep the computed features. This might cause OOM. # OPTIMIZER epochs: int = 10000 # total number of epochs to train. @@ -28,7 +54,78 @@ class BaseVocoderConfig(BaseTrainingConfig): @dataclass class BaseGANVocoderConfig(BaseVocoderConfig): - """Common config interface for all the GAN based vocoder models.""" + """Base config class used among all the GAN based vocoders. + Args: + use_stft_loss (bool): + enable / disable the use of STFT loss. Defaults to True. + use_subband_stft_loss (bool): + enable / disable the use of Subband STFT loss. Defaults to True. + use_mse_gan_loss (bool): + enable / disable the use of Mean Squared Error based GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable the use of Hinge GAN loss. Defaults to True. + use_feat_match_loss (bool): + enable / disable feature matching loss. Defaults to True. + use_l1_spec_loss (bool): + enable / disable L1 spectrogram loss. Defaults to True. + stft_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 0. + subband_stft_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 0. + mse_G_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 1. + hinge_G_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 0. + feat_match_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 100. + l1_spec_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 45. + stft_loss_params (dict): + Parameters for the STFT loss. Defaults to `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}`. + l1_spec_loss_params (dict): + Parameters for the L1 spectrogram loss. Defaults to + `{ + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None, + }` + target_loss (str): + Target loss name that defines the quality of the model. Defaults to `avg_G_loss`. + gen_clip_grad (float): + Gradient clipping threshold for the generator model. Any value less than 0 disables clipping. + Defaults to -1. + disc_clip_grad (float): + Gradient clipping threshold for the discriminator model. Any value less than 0 disables clipping. + Defaults to -1. + lr_gen (float): + Generator model initial learning rate. Defaults to 0.0002. + lr_disc (float): + Discriminator model initial learning rate. Defaults to 0.0002. + optimizer (torch.optim.Optimizer): + Optimizer used for the training. Defaults to `AdamW`. + optimizer_params (dict): + Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}` + lr_scheduler_gen (torch.optim.Scheduler): + Learning rate scheduler for the generator. Defaults to `ExponentialLR`. + lr_scheduler_gen_params (dict): + Parameters for the generator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`. + lr_scheduler_disc (torch.optim.Scheduler): + Learning rate scheduler for the discriminator. Defaults to `ExponentialLR`. + lr_scheduler_dict_params (dict): + Parameters for the discriminator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`. + use_pqmf (bool): + enable / disable PQMF for subband approximation at training. Defaults to False. + steps_to_start_discriminator (int): + Number of steps required to start training the discriminator. Defaults to 0. + diff_samples_for_G_and_D (bool): + enable / disable use of different training samples for the generator and the discriminator iterations. + Enabling it results in slower iterations but faster convergance in some cases. Defaults to False. + """ # LOSS PARAMETERS use_stft_loss: bool = True @@ -43,7 +140,7 @@ class BaseGANVocoderConfig(BaseVocoderConfig): subband_stft_loss_weight: float = 0 mse_G_loss_weight: float = 1 hinge_G_loss_weight: float = 0 - feat_match_loss_weight: float = 10 + feat_match_loss_weight: float = 100 l1_spec_loss_weight: float = 45 stft_loss_params: dict = field( diff --git a/TTS/vocoder/configs/wavegrad_config.py b/TTS/vocoder/configs/wavegrad_config.py index 48e3aba2..9d5cc683 100644 --- a/TTS/vocoder/configs/wavegrad_config.py +++ b/TTS/vocoder/configs/wavegrad_config.py @@ -1,12 +1,71 @@ from dataclasses import dataclass, field -from .shared_configs import BaseVocoderConfig +from TTS.vocoder.configs.shared_configs import BaseVocoderConfig @dataclass class WavegradConfig(BaseVocoderConfig): - """Defines parameters for Wavernn vocoder.""" + """Defines parameters for WaveGrad vocoder. + Example: + >>> from TTS.vocoder.configs import WavegradConfig + >>> config = WavegradConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `wavegrad`. + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `wavegrad`. + model_params (dict): + WaveGrad kwargs. Defaults to + ` + { + "use_weight_norm": True, + "y_conv_channels": 32, + "x_conv_channels": 768, + "ublock_out_channels": [512, 512, 256, 128, 128], + "dblock_out_channels": [128, 128, 256, 512], + "upsample_factors": [4, 4, 4, 2, 2], + "upsample_dilations": [[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8], [1, 2, 4, 8], [1, 2, 4, 8]], + } + ` + target_loss (str): + Target loss name that defines the quality of the model. Defaults to `avg_wavegrad_loss`. + epochs (int): + Number of epochs to traing the model. Defaults to 10000. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 96. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 6144. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + mixed_precision (bool): + enable / disable mixed precision training. Default is True. + eval_split_size (int): + Number of samples used for evalutaion. Defaults to 50. + train_noise_schedule (dict): + Training noise schedule. Defaults to + `{"min_val": 1e-6, "max_val": 1e-2, "num_steps": 1000}` + test_noise_schedule (dict): + Inference noise schedule. For a better performance, you may need to use `bin/tune_wavegrad.py` to find a + better schedule. Defaults to + ` + { + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 50, + } + ` + grad_clip (float): + Gradient clipping threshold. If <= 0.0, no clipping is applied. Defaults to 1.0 + lr (float): + Initila leraning rate. Defaults to 1e-4. + lr_scheduler (str): + One of the learning rate schedulers from `torch.optim.scheduler.*`. Defaults to `MultiStepLR`. + lr_scheduler_params (dict): + kwargs for the scheduler. Defaults to `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}` + """ model: str = "wavegrad" # Model specific params generator_model: str = "wavegrad" @@ -28,7 +87,6 @@ class WavegradConfig(BaseVocoderConfig): batch_size: int = 96 seq_len: int = 6144 use_cache: bool = True - steps_to_start_discriminator: int = 200000 mixed_precision: bool = True eval_split_size: int = 50 diff --git a/TTS/vocoder/configs/wavernn_config.py b/TTS/vocoder/configs/wavernn_config.py index 0b546a91..95a3cfc4 100644 --- a/TTS/vocoder/configs/wavernn_config.py +++ b/TTS/vocoder/configs/wavernn_config.py @@ -1,11 +1,77 @@ from dataclasses import dataclass, field -from .shared_configs import BaseVocoderConfig +from TTS.vocoder.configs.shared_configs import BaseVocoderConfig @dataclass class WavernnConfig(BaseVocoderConfig): - """Defines parameters for Wavernn vocoder.""" + """Defines parameters for Wavernn vocoder. + Example: + + >>> from TTS.vocoder.configs import WavernnConfig + >>> config = WavernnConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `wavernn`. + mode (str): + Output mode of the WaveRNN vocoder. `mold` for Mixture of Logistic Distribution, `gauss` for a single + Gaussian Distribution and `bits` for quantized bits as the model's output. + mulaw (bool): + enable / disable the use of Mulaw quantization for training. Only applicable if `mode == 'bits'`. Defaults + to `True`. + generator_model (str): + One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `WaveRNN`. + wavernn_model_params (dict): + kwargs for the WaveRNN model. Defaults to + `{ + "rnn_dims": 512, + "fc_dims": 512, + "compute_dims": 128, + "res_out_dims": 128, + "num_res_blocks": 10, + "use_aux_net": True, + "use_upsample_net": True, + "upsample_factors": [4, 8, 8] + }` + batched (bool): + enable / disable the batched inference. It speeds up the inference by splitting the input into segments and + processing the segments in a batch. Then it merges the outputs with a certain overlap and smoothing. If + you set it False, without CUDA, it is too slow to be practical. Defaults to True. + target_samples (int): + Size of the segments in batched mode. Defaults to 11000. + overlap_sampels (int): + Size of the overlap between consecutive segments. Defaults to 550. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 256. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 1280. + padding (int): + Padding applied to the input feature frames against the convolution layers of the feature network. + Defaults to 2. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + mixed_precision (bool): + enable / disable mixed precision training. Default is True. + eval_split_size (int): + Number of samples used for evalutaion. Defaults to 50. + test_every_epoch (int): + Number of epochs waited to run the next evalution. Since inference takes some time, it is better to + wait some number of epochs not ot waste training time. Defaults to 10. + grad_clip (float): + Gradient clipping threshold. If <= 0.0, no clipping is applied. Defaults to 4.0 + lr (float): + Initila leraning rate. Defaults to 1e-4. + lr_scheduler (str): + One of the learning rate schedulers from `torch.optim.scheduler.*`. Defaults to `MultiStepLR`. + lr_scheduler_params (dict): + kwargs for the scheduler. Defaults to `{"gamma": 0.5, "milestones": [200000, 400000, 600000]}` + """ model: str = "wavernn" @@ -38,7 +104,6 @@ class WavernnConfig(BaseVocoderConfig): padding: int = 2 use_noise_augment: bool = False use_cache: bool = True - steps_to_start_discriminator: int = 200000 mixed_precision: bool = True eval_split_size: int = 50 test_every_epochs: int = 10 # number of epochs to wait until the next test run (synthesizing a full audio clip). From 12722501bb7b48d9e35475449b583b17e4ec9871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Sat, 15 May 2021 23:48:31 +0200 Subject: [PATCH 72/87] styling --- TTS/_version.py | 2 +- TTS/bin/extract_tts_spectrograms.py | 109 +++++++++++------- TTS/bin/train_vocoder_gan.py | 10 +- TTS/config/shared_configs.py | 14 ++- TTS/tts/configs/glow_tts_config.py | 1 - TTS/tts/configs/shared_configs.py | 3 +- TTS/tts/configs/speedy_speech_config.py | 2 +- .../configs/multiband_melgan_config.py | 3 +- TTS/vocoder/configs/wavegrad_config.py | 1 + tests/test_extract_tts_spectrograms.py | 34 +++--- tests/test_glow_tts.py | 16 +-- 11 files changed, 115 insertions(+), 80 deletions(-) diff --git a/TTS/_version.py b/TTS/_version.py index d7d14b11..f0584d70 100644 --- a/TTS/_version.py +++ b/TTS/_version.py @@ -1 +1 @@ -__version__ = '0.0.13.2' +__version__ = "0.0.13.2" diff --git a/TTS/bin/extract_tts_spectrograms.py b/TTS/bin/extract_tts_spectrograms.py index 8a0a53ab..ace7464a 100755 --- a/TTS/bin/extract_tts_spectrograms.py +++ b/TTS/bin/extract_tts_spectrograms.py @@ -1,25 +1,26 @@ #!/usr/bin/env python3 """Extract Mel spectrograms with teacher forcing.""" -import os import argparse +import os + import numpy as np -from tqdm import tqdm import torch - from torch.utils.data import DataLoader +from tqdm import tqdm +from TTS.config import load_config from TTS.tts.datasets.preprocess import load_meta_data from TTS.tts.datasets.TTSDataset import MyDataset from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols -from TTS.config import load_config from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import count_parameters use_cuda = torch.cuda.is_available() + def setup_loader(ap, r, verbose=False): dataset = MyDataset( r, @@ -38,9 +39,7 @@ def setup_loader(ap, r, verbose=False): enable_eos_bos=c.enable_eos_bos_chars, use_noise_augment=False, verbose=verbose, - speaker_mapping=speaker_mapping - if c.use_speaker_embedding and c.use_external_speaker_embedding_file - else None, + speaker_mapping=speaker_mapping if c.use_speaker_embedding and c.use_external_speaker_embedding_file else None, ) if c.use_phonemes and c.compute_input_seq_cache: @@ -60,19 +59,21 @@ def setup_loader(ap, r, verbose=False): ) return loader + def set_filename(wav_path, out_path): wav_file = os.path.basename(wav_path) - file_name = wav_file.split('.')[0] + file_name = wav_file.split(".")[0] os.makedirs(os.path.join(out_path, "quant"), exist_ok=True) os.makedirs(os.path.join(out_path, "mel"), exist_ok=True) os.makedirs(os.path.join(out_path, "wav_gl"), exist_ok=True) os.makedirs(os.path.join(out_path, "wav"), exist_ok=True) wavq_path = os.path.join(out_path, "quant", file_name) mel_path = os.path.join(out_path, "mel", file_name) - wav_gl_path = os.path.join(out_path, "wav_gl", file_name+'.wav') - wav_path = os.path.join(out_path, "wav", file_name+'.wav') + wav_gl_path = os.path.join(out_path, "wav_gl", file_name + ".wav") + wav_path = os.path.join(out_path, "wav", file_name + ".wav") return file_name, wavq_path, mel_path, wav_gl_path, wav_path + def format_data(data): # setup input data text_input = data[0] @@ -123,10 +124,22 @@ def format_data(data): item_idx, ) + @torch.no_grad() -def inference(model_name, model, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask=None, speaker_ids=None, speaker_embeddings=None): +def inference( + model_name, + model, + ap, + text_input, + text_lengths, + mel_input, + mel_lengths, + attn_mask=None, + speaker_ids=None, + speaker_embeddings=None, +): if model_name == "glow_tts": - mel_input = mel_input.permute(0, 2, 1) # B x D x T + mel_input = mel_input.permute(0, 2, 1) # B x D x T speaker_c = None if speaker_ids is not None: speaker_c = speaker_ids @@ -140,7 +153,13 @@ def inference(model_name, model, ap, text_input, text_lengths, mel_input, mel_le elif "tacotron" in model_name: _, postnet_outputs, *_ = model( - text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids, speaker_embeddings=speaker_embeddings) + text_input, + text_lengths, + mel_input, + mel_lengths, + speaker_ids=speaker_ids, + speaker_embeddings=speaker_embeddings, + ) # normalize tacotron output if model_name == "tacotron": mel_specs = [] @@ -154,7 +173,10 @@ def inference(model_name, model, ap, text_input, text_lengths, mel_input, mel_le model_output = postnet_outputs.detach().cpu().numpy() return model_output -def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=False, save_audio=False, debug=False, metada_name="metada.txt"): + +def extract_spectrograms( + data_loader, model, ap, output_path, quantized_wav=False, save_audio=False, debug=False, metada_name="metada.txt" +): model.eval() export_metadata = [] for _, data in tqdm(enumerate(data_loader), total=len(data_loader)): @@ -173,7 +195,18 @@ def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=Fals item_idx, ) = format_data(data) - model_output = inference(c.model.lower(), model, ap, text_input, text_lengths, mel_input, mel_lengths, attn_mask, speaker_ids, speaker_embeddings) + model_output = inference( + c.model.lower(), + model, + ap, + text_input, + text_lengths, + mel_input, + mel_lengths, + attn_mask, + speaker_ids, + speaker_embeddings, + ) for idx in range(text_input.shape[0]): wav_file_path = item_idx[idx] @@ -204,13 +237,14 @@ def extract_spectrograms(data_loader, model, ap, output_path, quantized_wav=Fals for data in export_metadata: f.write(f"{data[0]}|{data[1]+'.npy'}\n") + def main(args): # pylint: disable=redefined-outer-name # pylint: disable=global-variable-undefined global meta_data, symbols, phonemes, model_characters, speaker_mapping # Audio processor ap = AudioProcessor(**c.audio) - if "characters" in c.keys() and c['characters']: + if "characters" in c.keys() and c["characters"]: symbols, phonemes = make_symbols(**c.characters) # set model characters @@ -242,37 +276,26 @@ def main(args): # pylint: disable=redefined-outer-name r = 1 if c.model.lower() == "glow_tts" else model.decoder.r own_loader = setup_loader(ap, r, verbose=True) - extract_spectrograms(own_loader, model, ap, args.output_path, quantized_wav=args.quantized, save_audio=args.save_audio, debug=args.debug, metada_name="metada.txt") + extract_spectrograms( + own_loader, + model, + ap, + args.output_path, + quantized_wav=args.quantized, + save_audio=args.save_audio, + debug=args.debug, + metada_name="metada.txt", + ) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument( - '--config_path', - type=str, - help='Path to config file for training.', - required=True) - parser.add_argument( - '--checkpoint_path', - type=str, - help='Model file to be restored.', - required=True) - parser.add_argument( - '--output_path', - type=str, - help='Path to save mel specs', - required=True) - parser.add_argument('--debug', - default=False, - action='store_true', - help='Save audio files for debug') - parser.add_argument('--save_audio', - default=False, - action='store_true', - help='Save audio files') - parser.add_argument('--quantized', - action='store_true', - help='Save quantized audio files') + parser.add_argument("--config_path", type=str, help="Path to config file for training.", required=True) + parser.add_argument("--checkpoint_path", type=str, help="Model file to be restored.", required=True) + parser.add_argument("--output_path", type=str, help="Path to save mel specs", required=True) + parser.add_argument("--debug", default=False, action="store_true", help="Save audio files for debug") + parser.add_argument("--save_audio", default=False, action="store_true", help="Save audio files") + parser.add_argument("--quantized", action="store_true", help="Save quantized audio files") args = parser.parse_args() c = load_config(args.config_path) diff --git a/TTS/bin/train_vocoder_gan.py b/TTS/bin/train_vocoder_gan.py index 4159f12f..123d5a43 100755 --- a/TTS/bin/train_vocoder_gan.py +++ b/TTS/bin/train_vocoder_gan.py @@ -2,10 +2,10 @@ # TODO: mixed precision training """Trains GAN based vocoder model.""" +import itertools import os import sys import time -import itertools import traceback from inspect import signature @@ -496,8 +496,12 @@ def main(args): # pylint: disable=redefined-outer-name optimizer_gen = optimizer_gen(model_gen.parameters(), lr=c.lr_gen, **c.optimizer_params) optimizer_disc = getattr(torch.optim, c.optimizer) - if c.discriminator_model == 'hifigan_discriminator': - optimizer_disc = optimizer_disc(itertools.chain(model_disc.msd.parameters(), model_disc.mpd.parameters()), lr=c.lr_disc, **c.optimizer_params) + if c.discriminator_model == "hifigan_discriminator": + optimizer_disc = optimizer_disc( + itertools.chain(model_disc.msd.parameters(), model_disc.mpd.parameters()), + lr=c.lr_disc, + **c.optimizer_params, + ) else: optimizer_disc = optimizer_disc(model_disc.parameters(), lr=c.lr_disc, **c.optimizer_params) diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py index 7df1582f..94e1c6f3 100644 --- a/TTS/config/shared_configs.py +++ b/TTS/config/shared_configs.py @@ -1,5 +1,6 @@ from dataclasses import asdict, dataclass from typing import List, Union + from coqpit import MISSING, Coqpit, check_argument @@ -151,11 +152,14 @@ class BaseDatasetConfig(Coqpit): Path to the file that lists the attention mask files used with models that require attention masks to train the duration predictor. """ - name: str = '' - path: str = '' - meta_file_train: Union[str, List] = '' # TODO: don't take ignored speakers for multi-speaker datasets over this. This is Union for SC-Glow compat. - meta_file_val: str = '' - meta_file_attn_mask: str = '' + + name: str = "" + path: str = "" + meta_file_train: Union[ + str, List + ] = "" # TODO: don't take ignored speakers for multi-speaker datasets over this. This is Union for SC-Glow compat. + meta_file_val: str = "" + meta_file_attn_mask: str = "" def check_values( self, diff --git a/TTS/tts/configs/glow_tts_config.py b/TTS/tts/configs/glow_tts_config.py index c0eadb1f..36ccb612 100644 --- a/TTS/tts/configs/glow_tts_config.py +++ b/TTS/tts/configs/glow_tts_config.py @@ -59,7 +59,6 @@ class GlowTTSConfig(BaseTTSConfig): Maximum input sequence length to be used at training. Larger values result in more VRAM usage. """ - model: str = "glow_tts" # model params diff --git a/TTS/tts/configs/shared_configs.py b/TTS/tts/configs/shared_configs.py index 885896c7..6c710ca2 100644 --- a/TTS/tts/configs/shared_configs.py +++ b/TTS/tts/configs/shared_configs.py @@ -22,6 +22,7 @@ class GSTConfig(Coqpit): gst_num_style_tokens (int): Number of style token vectors. Defaults to 10. """ + gst_style_input_wav: str = None gst_style_input_weights: dict = None gst_embedding_dim: int = 256 @@ -130,7 +131,7 @@ class BaseTTSConfig(BaseTrainingConfig): datasets (List[BaseDatasetConfig]): List of datasets used for training. If multiple datasets are provided, they are merged and used together for training. - """ + """ audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) # phoneme settings diff --git a/TTS/tts/configs/speedy_speech_config.py b/TTS/tts/configs/speedy_speech_config.py index 85df543d..1b8f0c82 100644 --- a/TTS/tts/configs/speedy_speech_config.py +++ b/TTS/tts/configs/speedy_speech_config.py @@ -113,4 +113,4 @@ class SpeedySpeechConfig(BaseTTSConfig): # overrides min_seq_len: int = 13 max_seq_len: int = 200 - r: int = 1 #DO NOT CHANGE + r: int = 1 # DO NOT CHANGE diff --git a/TTS/vocoder/configs/multiband_melgan_config.py b/TTS/vocoder/configs/multiband_melgan_config.py index f7aabdb3..81fd7904 100644 --- a/TTS/vocoder/configs/multiband_melgan_config.py +++ b/TTS/vocoder/configs/multiband_melgan_config.py @@ -90,7 +90,6 @@ class MultibandMelganConfig(BaseGANVocoderConfig): L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. """ - model: str = "multiband_melgan" # Model specific params @@ -142,4 +141,4 @@ class MultibandMelganConfig(BaseGANVocoderConfig): mse_G_loss_weight: float = 2.5 hinge_G_loss_weight: float = 0 feat_match_loss_weight: float = 108 - l1_spec_loss_weight: float = 0 \ No newline at end of file + l1_spec_loss_weight: float = 0 diff --git a/TTS/vocoder/configs/wavegrad_config.py b/TTS/vocoder/configs/wavegrad_config.py index 9d5cc683..271422ee 100644 --- a/TTS/vocoder/configs/wavegrad_config.py +++ b/TTS/vocoder/configs/wavegrad_config.py @@ -66,6 +66,7 @@ class WavegradConfig(BaseVocoderConfig): lr_scheduler_params (dict): kwargs for the scheduler. Defaults to `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}` """ + model: str = "wavegrad" # Model specific params generator_model: str = "wavegrad" diff --git a/tests/test_extract_tts_spectrograms.py b/tests/test_extract_tts_spectrograms.py index 94044eeb..38cee473 100644 --- a/tests/test_extract_tts_spectrograms.py +++ b/tests/test_extract_tts_spectrograms.py @@ -3,13 +3,9 @@ import unittest import torch -from tests import get_tests_input_path - -from tests import get_tests_output_path, run_cli - -from TTS.tts.utils.generic_utils import setup_model - +from tests import get_tests_input_path, get_tests_output_path, run_cli from TTS.config import load_config +from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.text.symbols import phonemes, symbols torch.manual_seed(1) @@ -20,8 +16,8 @@ class TestExtractTTSSpectrograms(unittest.TestCase): def test_GlowTTS(): # set paths config_path = os.path.join(get_tests_input_path(), "test_glow_tts.json") - checkpoint_path = os.path.join(get_tests_output_path(), 'checkpoint_test.pth.tar') - output_path = os.path.join(get_tests_output_path(), 'output_extract_tts_spectrograms/') + checkpoint_path = os.path.join(get_tests_output_path(), "checkpoint_test.pth.tar") + output_path = os.path.join(get_tests_output_path(), "output_extract_tts_spectrograms/") # load config c = load_config(config_path) # create model @@ -30,14 +26,17 @@ class TestExtractTTSSpectrograms(unittest.TestCase): # save model torch.save({"model": model.state_dict()}, checkpoint_path) # run test - run_cli(f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli( + f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"' + ) run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') + @staticmethod def test_Tacotron2(): # set paths config_path = os.path.join(get_tests_input_path(), "test_tacotron2_config.json") - checkpoint_path = os.path.join(get_tests_output_path(), 'checkpoint_test.pth.tar') - output_path = os.path.join(get_tests_output_path(), 'output_extract_tts_spectrograms/') + checkpoint_path = os.path.join(get_tests_output_path(), "checkpoint_test.pth.tar") + output_path = os.path.join(get_tests_output_path(), "output_extract_tts_spectrograms/") # load config c = load_config(config_path) # create model @@ -46,14 +45,17 @@ class TestExtractTTSSpectrograms(unittest.TestCase): # save model torch.save({"model": model.state_dict()}, checkpoint_path) # run test - run_cli(f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli( + f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"' + ) run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') + @staticmethod def test_Tacotron(): # set paths config_path = os.path.join(get_tests_input_path(), "test_tacotron_config.json") - checkpoint_path = os.path.join(get_tests_output_path(), 'checkpoint_test.pth.tar') - output_path = os.path.join(get_tests_output_path(), 'output_extract_tts_spectrograms/') + checkpoint_path = os.path.join(get_tests_output_path(), "checkpoint_test.pth.tar") + output_path = os.path.join(get_tests_output_path(), "output_extract_tts_spectrograms/") # load config c = load_config(config_path) # create model @@ -62,5 +64,7 @@ class TestExtractTTSSpectrograms(unittest.TestCase): # save model torch.save({"model": model.state_dict()}, checkpoint_path) # run test - run_cli(f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"') + run_cli( + f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"' + ) run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') diff --git a/tests/test_glow_tts.py b/tests/test_glow_tts.py index 07886e80..486de274 100644 --- a/tests/test_glow_tts.py +++ b/tests/test_glow_tts.py @@ -130,6 +130,7 @@ class GlowTTSTrainTest(unittest.TestCase): ) count += 1 + class GlowTTSInferenceTest(unittest.TestCase): @staticmethod def test_inference(): @@ -174,13 +175,12 @@ class GlowTTSInferenceTest(unittest.TestCase): print(" > Num parameters for GlowTTS model:%s" % (count_parameters(model))) # inference encoder and decoder with MAS - y, *_ = model.inference_with_MAS( - input_dummy, input_lengths, mel_spec, mel_lengths, None - ) + y, *_ = model.inference_with_MAS(input_dummy, input_lengths, mel_spec, mel_lengths, None) - y_dec, _ = model.decoder_inference(mel_spec, mel_lengths - ) + y_dec, _ = model.decoder_inference(mel_spec, mel_lengths) - assert (y_dec.shape == y.shape), "Difference between the shapes of the glowTTS inference with MAS ({}) and the inference using only the decoder ({}) !!".format( - y.shape, y_dec.shape - ) + assert ( + y_dec.shape == y.shape + ), "Difference between the shapes of the glowTTS inference with MAS ({}) and the inference using only the decoder ({}) !!".format( + y.shape, y_dec.shape + ) From 34a42d379f591f9198a10544870a47200b3fb8b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 17 May 2021 11:35:30 +0200 Subject: [PATCH 73/87] update tacotron_config.py for checking `r` and the docstring --- TTS/tts/configs/tacotron_config.py | 13 ++- recipes/ljspeech/tacotron/run.sh | 21 +++++ recipes/ljspeech/tacotron/tacotron2-DDC.json | 91 ++++++++++++++++++++ 3 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 recipes/ljspeech/tacotron/run.sh create mode 100644 recipes/ljspeech/tacotron/tacotron2-DDC.json diff --git a/TTS/tts/configs/tacotron_config.py b/TTS/tts/configs/tacotron_config.py index 53f5739e..91213012 100644 --- a/TTS/tts/configs/tacotron_config.py +++ b/TTS/tts/configs/tacotron_config.py @@ -24,10 +24,10 @@ class TacotronConfig(BaseTTSConfig): Path to the wav file used at inference to set the speech style through GST. If `GST` is enabled and this is not defined, the model uses a zero vector as an input. Defaults to None. r (int): - Number of output frames that the decoder computed per iteration. Larger values makes training and inference - faster but reduces the quality of the output frames. This needs to be tuned considering your own needs. - Defaults to 1. - gradual_trainin (List[List]): + Initial number of output frames that the decoder computed per iteration. Larger values makes training and inference + faster but reduces the quality of the output frames. This must be equal to the largest `r` value used in + `gradual_training` schedule. Defaults to 1. + gradual_training (List[List]): Parameters for the gradual training schedule. It is in the form `[[a, b, c], [d ,e ,f] ..]` where `a` is the step number to start using the rest of the values, `b` is the `r` value and `c` is the batch size. If sets None, no gradual training is used. Defaults to None. @@ -168,3 +168,8 @@ class TacotronConfig(BaseTTSConfig): decoder_ssim_alpha: float = 0.25 postnet_ssim_alpha: float = 0.25 ga_alpha: float = 5.0 + + + def check_values(self): + if self.gradual_training: + assert self.gradual_training[0][1] == self.r, f"[!] the first scheduled gradual training `r` must be equal to the model's `r` value. {self.gradual_training[0][1]} vs {self.r}" \ No newline at end of file diff --git a/recipes/ljspeech/tacotron/run.sh b/recipes/ljspeech/tacotron/run.sh new file mode 100644 index 00000000..9f5435db --- /dev/null +++ b/recipes/ljspeech/tacotron/run.sh @@ -0,0 +1,21 @@ +#!/bin/bash +RUN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +echo $RUN_DIR +# download LJSpeech dataset +wget http://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2 +# extract +tar -xjf LJSpeech-1.1.tar.bz2 +# create train-val splits +shuf LJSpeech-1.1/metadata.csv > LJSpeech-1.1/metadata_shuf.csv +head -n 12000 LJSpeech-1.1/metadata_shuf.csv > LJSpeech-1.1/metadata_train.csv +tail -n 1100 LJSpeech-1.1/metadata_shuf.csv > LJSpeech-1.1/metadata_val.csv +mv LJSpeech-1.1 $RUN_DIR/ +rm LJSpeech-1.1.tar.bz2 +# compute dataset mean and variance for normalization +python TTS/bin/compute_statistics.py $RUN_DIR/tacotron2-DCA.json $RUN_DIR/scale_stats.npy --data_path $RUN_DIR/LJSpeech-1.1/wavs/ +# training .... +# change the GPU id if needed +CUDA_VISIBLE_DEVICES="0" python TTS/bin/train_tacotron.py --config_path $RUN_DIR/tacotron2-DDC.json \ + --output_path $RUN_DIR \ + --coqpit.datasets.0.path $RUN_DIR/LJSpeech-1.1/ \ + --coqpit.audio.stats_path $RUN_DIR/scale_stats.npy \ \ No newline at end of file diff --git a/recipes/ljspeech/tacotron/tacotron2-DDC.json b/recipes/ljspeech/tacotron/tacotron2-DDC.json new file mode 100644 index 00000000..0e290405 --- /dev/null +++ b/recipes/ljspeech/tacotron/tacotron2-DDC.json @@ -0,0 +1,91 @@ +{ + "datasets": [ + { + "name": "ljspeech", + "path": "DEFINE THIS", + "meta_file_train": "metadata.csv", + "meta_file_val": null + } + ], + "audio": { + "fft_size": 1024, + "win_length": 1024, + "hop_length": 256, + "frame_length_ms": null, + "frame_shift_ms": null, + "sample_rate": 22050, + "preemphasis": 0.0, + "ref_level_db": 20, + "do_trim_silence": true, + "trim_db": 60, + "power": 1.5, + "griffin_lim_iters": 60, + "num_mels": 80, + "mel_fmin": 50.0, + "mel_fmax": 7600.0, + "spec_gain": 1, + "signal_norm": true, + "min_level_db": -100, + "symmetric_norm": true, + "max_norm": 4.0, + "clip_norm": true, + "stats_path": "scale_stats.npy" + }, + "gst":{ + "gst_embedding_dim": 256, + "gst_num_heads": 4, + "gst_num_style_tokens": 10 + }, + "model": "Tacotron2", + "run_name": "ljspeech-dcattn", + "run_description": "tacotron2 with dynamic convolution attention.", + "batch_size": 64, + "eval_batch_size": 16, + "r": 2, + "mixed_precision": true, + "loss_masking": true, + "decoder_loss_alpha": 0.25, + "postnet_loss_alpha": 0.25, + "postnet_diff_spec_alpha": 0.25, + "decoder_diff_spec_alpha": 0.25, + "decoder_ssim_alpha": 0.25, + "postnet_ssim_alpha": 0.25, + "ga_alpha": 5.0, + "stopnet_pos_weight": 15.0, + "run_eval": true, + "test_delay_epochs": 10, + "test_sentences_file": null, + "noam_schedule": true, + "grad_clip": 0.05, + "epochs": 1000, + "lr": 0.001, + "wd": 1e-06, + "warmup_steps": 4000, + "memory_size": -1, + "prenet_type": "original", + "prenet_dropout": true, + "attention_type": "original", + "location_attn": true, + "double_decoder_consistency": true, + "ddc_r": 6, + "attention_norm": "sigmoid", + "gradual_training": [[0, 6, 64], [10000, 4, 32], [50000, 3, 32], [100000, 2, 32]], + "stopnet": true, + "separate_stopnet": true, + "print_step": 25, + "tb_plot_step": 100, + "print_eval": false, + "save_step": 10000, + "checkpoint": true, + "text_cleaner": "phoneme_cleaners", + "num_loader_workers": 4, + "num_val_loader_workers": 4, + "batch_group_size": 4, + "min_seq_len": 6, + "max_seq_len": 180, + "compute_input_seq_cache": true, + "output_path": "DEFINE THIS", + "phoneme_cache_path": "DEFINE THIS", + "use_phonemes": false, + "phoneme_language": "en-us" +} \ No newline at end of file From d1b469935d640d55156b8d377362e19ddaa4fa55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Mon, 17 May 2021 11:38:01 +0200 Subject: [PATCH 74/87] tacotron DDC LJSpeech recipe --- .gitignore | 1 - TTS/tts/configs/tacotron_config.py | 5 +++-- recipes/README.md | 13 +++++++++++++ recipes/ljspeech/{tacotron => tacotron2-DDC}/run.sh | 5 +++-- .../{tacotron => tacotron2-DDC}/tacotron2-DDC.json | 6 +++--- 5 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 recipes/README.md rename recipes/ljspeech/{tacotron => tacotron2-DDC}/run.sh (81%) rename recipes/ljspeech/{tacotron => tacotron2-DDC}/tacotron2-DDC.json (95%) diff --git a/.gitignore b/.gitignore index 1829dd93..1d3ab8c2 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,3 @@ notebooks/data/* TTS/tts/layers/glow_tts/monotonic_align/core.c .vscode-upload.json temp_build/* -recipes/* diff --git a/TTS/tts/configs/tacotron_config.py b/TTS/tts/configs/tacotron_config.py index 91213012..d3a54269 100644 --- a/TTS/tts/configs/tacotron_config.py +++ b/TTS/tts/configs/tacotron_config.py @@ -169,7 +169,8 @@ class TacotronConfig(BaseTTSConfig): postnet_ssim_alpha: float = 0.25 ga_alpha: float = 5.0 - def check_values(self): if self.gradual_training: - assert self.gradual_training[0][1] == self.r, f"[!] the first scheduled gradual training `r` must be equal to the model's `r` value. {self.gradual_training[0][1]} vs {self.r}" \ No newline at end of file + assert ( + self.gradual_training[0][1] == self.r + ), f"[!] the first scheduled gradual training `r` must be equal to the model's `r` value. {self.gradual_training[0][1]} vs {self.r}" diff --git a/recipes/README.md b/recipes/README.md new file mode 100644 index 00000000..041693a2 --- /dev/null +++ b/recipes/README.md @@ -0,0 +1,13 @@ +# 🐸💬 TTS Training Recipes + +TTS recipes intended to host bash scripts running all the necessary steps to train a TTS model with a particular dataset. + +Run each script from the root TTS folder as follows + +```console +$ bash ./recipes///run.sh +``` + +All the outputs are held under the recipe directory unless you change the paths in the bash script. + +If you train a new model using TTS, feel free to share your training to expand the list of recipes. \ No newline at end of file diff --git a/recipes/ljspeech/tacotron/run.sh b/recipes/ljspeech/tacotron2-DDC/run.sh similarity index 81% rename from recipes/ljspeech/tacotron/run.sh rename to recipes/ljspeech/tacotron2-DDC/run.sh index 9f5435db..eaa05b60 100644 --- a/recipes/ljspeech/tacotron/run.sh +++ b/recipes/ljspeech/tacotron2-DDC/run.sh @@ -1,4 +1,5 @@ #!/bin/bash +# take the scripts's parent's directory to prefix all the output paths. RUN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" echo $RUN_DIR # download LJSpeech dataset @@ -12,10 +13,10 @@ tail -n 1100 LJSpeech-1.1/metadata_shuf.csv > LJSpeech-1.1/metadata_val.csv mv LJSpeech-1.1 $RUN_DIR/ rm LJSpeech-1.1.tar.bz2 # compute dataset mean and variance for normalization -python TTS/bin/compute_statistics.py $RUN_DIR/tacotron2-DCA.json $RUN_DIR/scale_stats.npy --data_path $RUN_DIR/LJSpeech-1.1/wavs/ +python TTS/bin/compute_statistics.py $RUN_DIR/tacotron2-DDC.json $RUN_DIR/scale_stats.npy --data_path $RUN_DIR/LJSpeech-1.1/wavs/ # training .... # change the GPU id if needed CUDA_VISIBLE_DEVICES="0" python TTS/bin/train_tacotron.py --config_path $RUN_DIR/tacotron2-DDC.json \ - --output_path $RUN_DIR \ + --coqpit.output_path $RUN_DIR \ --coqpit.datasets.0.path $RUN_DIR/LJSpeech-1.1/ \ --coqpit.audio.stats_path $RUN_DIR/scale_stats.npy \ \ No newline at end of file diff --git a/recipes/ljspeech/tacotron/tacotron2-DDC.json b/recipes/ljspeech/tacotron2-DDC/tacotron2-DDC.json similarity index 95% rename from recipes/ljspeech/tacotron/tacotron2-DDC.json rename to recipes/ljspeech/tacotron2-DDC/tacotron2-DDC.json index 0e290405..9cdbbd3b 100644 --- a/recipes/ljspeech/tacotron/tacotron2-DDC.json +++ b/recipes/ljspeech/tacotron2-DDC/tacotron2-DDC.json @@ -37,11 +37,10 @@ "gst_num_style_tokens": 10 }, "model": "Tacotron2", - "run_name": "ljspeech-dcattn", - "run_description": "tacotron2 with dynamic convolution attention.", + "run_name": "ljspeech-ddc", + "run_description": "tacotron2 with double decoder consistency.", "batch_size": 64, "eval_batch_size": 16, - "r": 2, "mixed_precision": true, "loss_masking": true, "decoder_loss_alpha": 0.25, @@ -69,6 +68,7 @@ "double_decoder_consistency": true, "ddc_r": 6, "attention_norm": "sigmoid", + "r": 6, "gradual_training": [[0, 6, 64], [10000, 4, 32], [50000, 3, 32], [100000, 2, 32]], "stopnet": true, "separate_stopnet": true, From c2c7dff8057f30f111016cb9b85d01dfbc4c4b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 18 May 2021 14:49:47 +0200 Subject: [PATCH 75/87] use relaxted coqpit parser --- TTS/bin/compute_statistics.py | 2 +- TTS/utils/arguments.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TTS/bin/compute_statistics.py b/TTS/bin/compute_statistics.py index 37885fdd..f3234c2a 100755 --- a/TTS/bin/compute_statistics.py +++ b/TTS/bin/compute_statistics.py @@ -28,7 +28,7 @@ def main(): args, overrides = parser.parse_known_args() CONFIG = load_config(args.config_path) - CONFIG.parse_args(overrides) + CONFIG.parse_known_args(overrides, relaxed_parser=True) # load config CONFIG.audio.signal_norm = False # do not apply earlier normalization diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index fc969593..1b5a424b 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -144,7 +144,7 @@ def process_args(args): # setup output paths and read configs config = load_config(args.config_path) # override values from command-line args - config.parse_args(coqpit_overrides) + config.parse_known_args(coqpit_overrides, relaxed_parser=True) if config.mixed_precision: print(" > Mixed precision mode is ON") experiment_path = args.continue_path From 4df31f7fbd65ff186d2700cd1b343f88f32110c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 18 May 2021 14:50:03 +0200 Subject: [PATCH 76/87] unused_speakers argument for ignoring speaker ids in multi-speaker training --- TTS/config/shared_configs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py index 94e1c6f3..3aa80c98 100644 --- a/TTS/config/shared_configs.py +++ b/TTS/config/shared_configs.py @@ -143,9 +143,11 @@ class BaseDatasetConfig(Coqpit): Dataset name that defines the preprocessor in use. Defaults to None. path (str): Root path to the dataset files. Defaults to None. - meta_file_train (Union[str, List]): + meta_file_train (str): Name of the dataset meta file. Or a list of speakers to be ignored at training for multi-speaker datasets. Defaults to None. + unused_speakers (List): + List of speakers IDs that are not used at the training. Default None. meta_file_val (str): Name of the dataset meta file that defines the instances used at validation. meta_file_attn_mask (str): @@ -155,9 +157,8 @@ class BaseDatasetConfig(Coqpit): name: str = "" path: str = "" - meta_file_train: Union[ - str, List - ] = "" # TODO: don't take ignored speakers for multi-speaker datasets over this. This is Union for SC-Glow compat. + meta_file_train: str = "" + ununsed_speakers: List[str] = None meta_file_val: str = "" meta_file_attn_mask: str = "" From 8142291b3653712b6b222faba8375dfd1d35dcab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 18 May 2021 14:50:29 +0200 Subject: [PATCH 77/87] change `list` to `List` in config --- TTS/config/shared_configs.py | 2 +- TTS/tts/configs/align_tts_config.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py index 3aa80c98..69f1ee31 100644 --- a/TTS/config/shared_configs.py +++ b/TTS/config/shared_configs.py @@ -1,5 +1,5 @@ from dataclasses import asdict, dataclass -from typing import List, Union +from typing import List from coqpit import MISSING, Coqpit, check_argument diff --git a/TTS/tts/configs/align_tts_config.py b/TTS/tts/configs/align_tts_config.py index 84e0ba13..2956d935 100644 --- a/TTS/tts/configs/align_tts_config.py +++ b/TTS/tts/configs/align_tts_config.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from typing import List from TTS.tts.configs.shared_configs import BaseTTSConfig @@ -78,7 +79,7 @@ class AlignTTSConfig(BaseTTSConfig): decoder_params: dict = field( default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1} ) - phase_start_steps: list = None + phase_start_steps: List[int] = None ssim_alpha: float = 1.0 spec_loss_alpha: float = 1.0 From 218af1d9a2a4b270a09a42af7502d473a833c172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 18 May 2021 14:50:29 +0200 Subject: [PATCH 78/87] change `list` to `List` in config --- TTS/config/shared_configs.py | 2 +- TTS/tts/configs/align_tts_config.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py index 3aa80c98..69f1ee31 100644 --- a/TTS/config/shared_configs.py +++ b/TTS/config/shared_configs.py @@ -1,5 +1,5 @@ from dataclasses import asdict, dataclass -from typing import List, Union +from typing import List from coqpit import MISSING, Coqpit, check_argument diff --git a/TTS/tts/configs/align_tts_config.py b/TTS/tts/configs/align_tts_config.py index 84e0ba13..2956d935 100644 --- a/TTS/tts/configs/align_tts_config.py +++ b/TTS/tts/configs/align_tts_config.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from typing import List from TTS.tts.configs.shared_configs import BaseTTSConfig @@ -78,7 +79,7 @@ class AlignTTSConfig(BaseTTSConfig): decoder_params: dict = field( default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1} ) - phase_start_steps: list = None + phase_start_steps: List[int] = None ssim_alpha: float = 1.0 spec_loss_alpha: float = 1.0 From ced05e812a2ab15a58fc5af6285bba03d041ee98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 18 May 2021 14:56:14 +0200 Subject: [PATCH 79/87] move chinese phonemizer --- TTS/tts/utils/text/__init__.py | 2 +- TTS/tts/utils/{ => text}/chinese_mandarin/__init__.py | 0 TTS/tts/utils/{ => text}/chinese_mandarin/numbers.py | 0 TTS/tts/utils/{ => text}/chinese_mandarin/phonemizer.py | 0 TTS/tts/utils/{ => text}/chinese_mandarin/pinyinToPhonemes.py | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename TTS/tts/utils/{ => text}/chinese_mandarin/__init__.py (100%) rename TTS/tts/utils/{ => text}/chinese_mandarin/numbers.py (100%) rename TTS/tts/utils/{ => text}/chinese_mandarin/phonemizer.py (100%) rename TTS/tts/utils/{ => text}/chinese_mandarin/pinyinToPhonemes.py (100%) diff --git a/TTS/tts/utils/text/__init__.py b/TTS/tts/utils/text/__init__.py index 1000b191..6c193ff5 100644 --- a/TTS/tts/utils/text/__init__.py +++ b/TTS/tts/utils/text/__init__.py @@ -6,7 +6,7 @@ import phonemizer from packaging import version from phonemizer.phonemize import phonemize -from TTS.tts.utils.chinese_mandarin.phonemizer import chinese_text_to_phonemes +from TTS.tts.utils.text.chinese_mandarin.phonemizer import chinese_text_to_phonemes from TTS.tts.utils.text import cleaners from TTS.tts.utils.text.symbols import _bos, _eos, _punctuations, make_symbols, phonemes, symbols diff --git a/TTS/tts/utils/chinese_mandarin/__init__.py b/TTS/tts/utils/text/chinese_mandarin/__init__.py similarity index 100% rename from TTS/tts/utils/chinese_mandarin/__init__.py rename to TTS/tts/utils/text/chinese_mandarin/__init__.py diff --git a/TTS/tts/utils/chinese_mandarin/numbers.py b/TTS/tts/utils/text/chinese_mandarin/numbers.py similarity index 100% rename from TTS/tts/utils/chinese_mandarin/numbers.py rename to TTS/tts/utils/text/chinese_mandarin/numbers.py diff --git a/TTS/tts/utils/chinese_mandarin/phonemizer.py b/TTS/tts/utils/text/chinese_mandarin/phonemizer.py similarity index 100% rename from TTS/tts/utils/chinese_mandarin/phonemizer.py rename to TTS/tts/utils/text/chinese_mandarin/phonemizer.py diff --git a/TTS/tts/utils/chinese_mandarin/pinyinToPhonemes.py b/TTS/tts/utils/text/chinese_mandarin/pinyinToPhonemes.py similarity index 100% rename from TTS/tts/utils/chinese_mandarin/pinyinToPhonemes.py rename to TTS/tts/utils/text/chinese_mandarin/pinyinToPhonemes.py From d7fae3f5157c71016b08816dc7e9ca33f4d9ac20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 18 May 2021 15:07:25 +0200 Subject: [PATCH 80/87] remove all espeaker and phonemizer deps --- .compute | 1 - Makefile | 1 - TTS/server/README.md | 39 - TTS/tts/utils/text/__init__.py | 48 +- hubconf.py | 2 +- ..._and_MultiBand_MelGAN_TFLite_Example.ipynb | 387 -------- ...oqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb | 650 -------------- ...MultiSpeaker_jia_et_al_2018_With_GST.ipynb | 847 ------------------ requirements.txt | 1 - 9 files changed, 3 insertions(+), 1973 deletions(-) delete mode 100644 notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb delete mode 100644 notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb delete mode 100644 notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018_With_GST.ipynb diff --git a/.compute b/.compute index cda787d2..9786a689 100644 --- a/.compute +++ b/.compute @@ -1,7 +1,6 @@ #!/bin/bash yes | apt-get install sox yes | apt-get install ffmpeg -yes | apt-get install espeak yes | apt-get install tmux yes | apt-get install zsh sh -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)" diff --git a/Makefile b/Makefile index 2210a682..4dc2d588 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,6 @@ help: target_dirs := tests TTS notebooks system-deps: ## install linux system deps - sudo apt-get install -y espeak-ng sudo apt-get install -y libsndfile1-dev dev-deps: ## install development deps diff --git a/TTS/server/README.md b/TTS/server/README.md index 51cedc05..89ee21eb 100644 --- a/TTS/server/README.md +++ b/TTS/server/README.md @@ -22,42 +22,3 @@ Run the server with the official models on a GPU. Run the server with a custom models. ```python TTS/server/server.py --tts_checkpoint /path/to/tts/model.pth.tar --tts_config /path/to/tts/config.json --vocoder_checkpoint /path/to/vocoder/model.pth.tar --vocoder_config /path/to/vocoder/config.json``` - - - - - - diff --git a/TTS/tts/utils/text/__init__.py b/TTS/tts/utils/text/__init__.py index 6c193ff5..2b73d4e4 100644 --- a/TTS/tts/utils/text/__init__.py +++ b/TTS/tts/utils/text/__init__.py @@ -2,9 +2,7 @@ import re -import phonemizer from packaging import version -from phonemizer.phonemize import phonemize from TTS.tts.utils.text.chinese_mandarin.phonemizer import chinese_text_to_phonemes from TTS.tts.utils.text import cleaners @@ -28,9 +26,7 @@ PHONEME_PUNCTUATION_PATTERN = r"[" + _punctuations.replace(" ", "") + "]+" def text2phone(text, language): - """Convert graphemes to phonemes. For most of the languages, it calls - the phonemizer python library that calls espeak/espeak-ng. For chinese - mandarin, it calls pypinyin + custom function for phonemizing + """Convert graphemes to phonemes. Parameters: text (str): text to phonemize language (str): language of the text @@ -43,47 +39,7 @@ def text2phone(text, language): if language == "zh-CN": ph = chinese_text_to_phonemes(text) return ph - - seperator = phonemizer.separator.Separator(" |", "", "|") - # try: - punctuations = re.findall(PHONEME_PUNCTUATION_PATTERN, text) - if version.parse(phonemizer.__version__) < version.parse("2.1"): - ph = phonemize(text, separator=seperator, strip=False, njobs=1, backend="espeak", language=language) - ph = ph[:-1].strip() # skip the last empty character - # phonemizer does not tackle punctuations. Here we do. - # Replace \n with matching punctuations. - if punctuations: - # if text ends with a punctuation. - if text[-1] == punctuations[-1]: - for punct in punctuations[:-1]: - ph = ph.replace("| |\n", "|" + punct + "| |", 1) - ph = ph + punctuations[-1] - else: - for punct in punctuations: - ph = ph.replace("| |\n", "|" + punct + "| |", 1) - elif version.parse(phonemizer.__version__) >= version.parse("2.1"): - ph = phonemize( - text, - separator=seperator, - strip=False, - njobs=1, - backend="espeak", - language=language, - preserve_punctuation=True, - language_switch="remove-flags", - ) - # this is a simple fix for phonemizer. - # https://github.com/bootphon/phonemizer/issues/32 - if punctuations: - for punctuation in punctuations: - ph = ph.replace(f"| |{punctuation} ", f"|{punctuation}| |").replace( - f"| |{punctuation}", f"|{punctuation}| |" - ) - ph = ph[:-3] - else: - raise RuntimeError(" [!] Use 'phonemizer' version 2.1 or older.") - - return ph + raise ValueError(f" [!] Language {language} is nor supported for phonemization.") def intersperse(sequence, token): diff --git a/hubconf.py b/hubconf.py index 152374c8..bcbd6fce 100644 --- a/hubconf.py +++ b/hubconf.py @@ -1,6 +1,6 @@ dependencies = [ 'torch', 'gdown', 'pysbd', 'phonemizer', 'unidecode', 'pypinyin' -] # apt install espeak-ng +] import torch from TTS.utils.manage import ModelManager diff --git a/notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb b/notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb deleted file mode 100644 index c39cc53e..00000000 --- a/notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb +++ /dev/null @@ -1,387 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6LWsNd3_M3MP" - }, - "source": [ - "# Mozilla TTS on CPU Real-Time Speech Synthesis with TFLite" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FAqrSIWgLyP0" - }, - "source": [ - "**These models are converted from released [PyTorch models](https://colab.research.google.com/drive/1u_16ZzHjKYFn1HNVuA4Qf_i2MMFB9olY?usp=sharing) using our TF utilities provided in Mozilla TTS.**\n", - "\n", - "#### **Notebook Details**\n", - "These TFLite models support TF 2.3rc0 and for different versions you might need to regenerate them. \n", - "\n", - "TFLite optimizations degrades the TTS model performance and we do not apply\n", - "any optimization for the vocoder model due to the same reason. If you like to\n", - "keep the quality, consider to regenerate TFLite model accordingly.\n", - "\n", - "Models optimized with TFLite can be slow on a regular CPU since it is optimized\n", - "specifically for lower-end systems.\n", - "\n", - "---\n", - "\n", - "\n", - "\n", - "#### **Model Details** \n", - "We use Tacotron2 and MultiBand-Melgan models and LJSpeech dataset.\n", - "\n", - "Tacotron2 is trained using [Double Decoder Consistency](https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency/) (DDC) only for 130K steps (3 days) with a single GPU.\n", - "\n", - "MultiBand-Melgan is trained 1.45M steps with real spectrograms.\n", - "\n", - "Note that both model performances can be improved with more training.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ku-dA4DKoeXk" - }, - "source": [ - "### Download TF Models and configs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 162 - }, - "colab_type": "code", - "id": "jGIgnWhGsxU1", - "outputId": "57af701e-77ec-400d-fee5-64aa7603d357" - }, - "outputs": [], - "source": [ - "!gdown --id 17PYXCmTe0el_SLTwznrt3vOArNGMGo5v -O tts_model.tflite\n", - "!gdown --id 18CQ6G6tBEOfvCHlPqP8EBI4xWbrr9dBc -O config.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 235 - }, - "colab_type": "code", - "id": "4dnpE0-kvTsu", - "outputId": "6aab0622-9add-4ee4-b9f8-177d6ddc0e86" - }, - "outputs": [], - "source": [ - "!gdown --id 1aXveT-NjOM1mUr6tM4JfWjshq67GvVIO -O vocoder_model.tflite\n", - "!gdown --id 1Rd0R_nRCrbjEdpOwq6XwZAktvugiBvmu -O config_vocoder.json\n", - "!gdown --id 11oY3Tv0kQtxK_JPgxrfesa99maVXHNxU -O scale_stats.npy" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ZuDrj_ioqHE" - }, - "source": [ - "### Setup Libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 964 - }, - "colab_type": "code", - "id": "X2axt5BYq7gv", - "outputId": "aa53986f-f218-4d17-8667-0d74bb90c927" - }, - "outputs": [], - "source": [ - "# need it for char to phoneme conversion\n", - "! sudo apt-get install espeak" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 144 - }, - "colab_type": "code", - "id": "ZduAf-qYYEIT", - "outputId": "c1fcac0d-b8f8-442c-d598-4f549c42b698" - }, - "outputs": [], - "source": [ - "!git clone https://github.com/mozilla/TTS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "colab_type": "code", - "id": "ofPCvPyjZEcT", - "outputId": "f3d3ea73-eae5-473c-db19-276bd0e721cc" - }, - "outputs": [], - "source": [ - "%cd TTS\n", - "!git checkout c7296b3\n", - "!pip install -r requirements.txt\n", - "!python setup.py install\n", - "!pip install tensorflow==2.3.0rc0\n", - "%cd .." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zlgi8fPdpRF0" - }, - "source": [ - "### Define TTS function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "f-Yc42nQZG5A" - }, - "outputs": [], - "source": [ - "def run_vocoder(mel_spec):\n", - " vocoder_inputs = mel_spec[None, :, :]\n", - " # get input and output details\n", - " input_details = vocoder_model.get_input_details()\n", - " # reshape input tensor for the new input shape\n", - " vocoder_model.resize_tensor_input(input_details[0]['index'], vocoder_inputs.shape)\n", - " vocoder_model.allocate_tensors()\n", - " detail = input_details[0]\n", - " vocoder_model.set_tensor(detail['index'], vocoder_inputs)\n", - " # run the model\n", - " vocoder_model.invoke()\n", - " # collect outputs\n", - " output_details = vocoder_model.get_output_details()\n", - " waveform = vocoder_model.get_tensor(output_details[0]['index'])\n", - " return waveform \n", - "\n", - "\n", - "def tts(model, text, CONFIG, p):\n", - " t_1 = time.time()\n", - " waveform, alignment, mel_spec, mel_postnet_spec, stop_tokens, inputs = synthesis(model, text, CONFIG, use_cuda, ap, speaker_id, style_wav=None,\n", - " truncated=False, enable_eos_bos_chars=CONFIG.enable_eos_bos_chars,\n", - " backend='tflite')\n", - " waveform = run_vocoder(mel_postnet_spec.T)\n", - " waveform = waveform[0, 0]\n", - " rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n", - " tps = (time.time() - t_1) / len(waveform)\n", - " print(waveform.shape)\n", - " print(\" > Run-time: {}\".format(time.time() - t_1))\n", - " print(\" > Real-time factor: {}\".format(rtf))\n", - " print(\" > Time per step: {}\".format(tps))\n", - " IPython.display.display(IPython.display.Audio(waveform, rate=CONFIG.audio['sample_rate'])) \n", - " return alignment, mel_postnet_spec, stop_tokens, waveform" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZksegYQepkFg" - }, - "source": [ - "### Load TF Models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "oVa0kOamprgj" - }, - "outputs": [], - "source": [ - "import os\n", - "import torch\n", - "import time\n", - "import IPython\n", - "\n", - "from TTS.tf.utils.tflite import load_tflite_model\n", - "from TTS.tf.utils.io import load_checkpoint\n", - "from TTS.utils.io import load_config\n", - "from TTS.utils.text.symbols import symbols, phonemes\n", - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.tts.utils.synthesis import synthesis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EY-sHVO8IFSH" - }, - "outputs": [], - "source": [ - "# runtime settings\n", - "use_cuda = False" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_1aIUp2FpxOQ" - }, - "outputs": [], - "source": [ - "# model paths\n", - "TTS_MODEL = \"tts_model.tflite\"\n", - "TTS_CONFIG = \"config.json\"\n", - "VOCODER_MODEL = \"vocoder_model.tflite\"\n", - "VOCODER_CONFIG = \"config_vocoder.json\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "CpgmdBVQplbv" - }, - "outputs": [], - "source": [ - "# load configs\n", - "TTS_CONFIG = load_config(TTS_CONFIG)\n", - "VOCODER_CONFIG = load_config(VOCODER_CONFIG)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 471 - }, - "colab_type": "code", - "id": "zmrQxiozIUVE", - "outputId": "ca7e9016-4c28-4cef-efe7-0613d399aa4c" - }, - "outputs": [], - "source": [ - "# load the audio processor\n", - "ap = AudioProcessor(**TTS_CONFIG.audio) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8fLoI4ipqMeS" - }, - "outputs": [], - "source": [ - "# LOAD TTS MODEL\n", - "# multi speaker \n", - "speaker_id = None\n", - "speakers = []\n", - "\n", - "# load the models\n", - "model = load_tflite_model(TTS_MODEL)\n", - "vocoder_model = load_tflite_model(VOCODER_MODEL)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ws_YkPKsLgo-" - }, - "source": [ - "## Run Inference" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 134 - }, - "colab_type": "code", - "id": "FuWxZ9Ey5Puj", - "outputId": "d1888ebd-3208-42a4-aaf9-78d0e3ec987d" - }, - "outputs": [], - "source": [ - "sentence = \"Bill got in the habit of asking himself “Is that thought true?” and if he wasn’t absolutely certain it was, he just let it go.\"\n", - "align, spec, stop_tokens, wav = tts(model, sentence, TTS_CONFIG, ap)" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "DDC-TTS_and_MultiBand-MelGAN_TFLite_Example.ipynb", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb b/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb deleted file mode 100644 index 82efdc2a..00000000 --- a/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb +++ /dev/null @@ -1,650 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yZK6UdwSFnOO" - }, - "source": [ - "# **Download and install Coqui TTS**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yvb0pX3WY6MN" - }, - "outputs": [], - "source": [ - "import os \n", - "!git clone https://github.com/Edresson/TTS -b dev-gst-embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iB9nl2UEG3SY" - }, - "outputs": [], - "source": [ - "!apt-get install espeak\n", - "os.chdir('TTS')\n", - "!pip install -r requirements.txt\n", - "!python setup.py develop\n", - "os.chdir('..')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w6Krn8k1inC_" - }, - "source": [ - "\n", - "\n", - "**Download Checkpoint**\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "PiYHf3lKhi9z" - }, - "outputs": [], - "source": [ - "!wget -c -q --show-progress -O ./TTS-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/Checkpoints-TTS-MultiSpeaker-Jia-et-al-2018.zip\n", - "!unzip ./TTS-checkpoint.zip\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MpYNgqrZcJKn" - }, - "source": [ - "**Utils Functions**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4KZA4b_CbMqx" - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "import argparse\n", - "import json\n", - "# pylint: disable=redefined-outer-name, unused-argument\n", - "import os\n", - "import string\n", - "import time\n", - "import sys\n", - "import numpy as np\n", - "\n", - "TTS_PATH = \"../content/TTS\"\n", - "# add libraries into environment\n", - "sys.path.append(TTS_PATH) # set this if TTS is not installed globally\n", - "\n", - "import torch\n", - "\n", - "from TTS.tts.utils.generic_utils import setup_model\n", - "from TTS.tts.utils.synthesis import synthesis\n", - "from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols\n", - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.utils.io import load_config\n", - "from TTS.vocoder.utils.generic_utils import setup_generator\n", - "\n", - "\n", - "def tts(model, vocoder_model, text, CONFIG, use_cuda, ap, use_gl, speaker_fileid, speaker_embedding=None):\n", - " t_1 = time.time()\n", - " waveform, _, _, mel_postnet_spec, _, _ = synthesis(model, text, CONFIG, use_cuda, ap, speaker_fileid, None, False, CONFIG.enable_eos_bos_chars, use_gl, speaker_embedding=speaker_embedding)\n", - " if CONFIG.model == \"Tacotron\" and not use_gl:\n", - " mel_postnet_spec = ap.out_linear_to_mel(mel_postnet_spec.T).T\n", - " if not use_gl:\n", - " waveform = vocoder_model.inference(torch.FloatTensor(mel_postnet_spec.T).unsqueeze(0))\n", - " if use_cuda and not use_gl:\n", - " waveform = waveform.cpu()\n", - " if not use_gl:\n", - " waveform = waveform.numpy()\n", - " waveform = waveform.squeeze()\n", - " rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n", - " tps = (time.time() - t_1) / len(waveform)\n", - " print(\" > Run-time: {}\".format(time.time() - t_1))\n", - " print(\" > Real-time factor: {}\".format(rtf))\n", - " print(\" > Time per step: {}\".format(tps))\n", - " return waveform\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ENA2OumIVeMA" - }, - "source": [ - "# **Vars definitions**\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jPD0d_XpVXmY" - }, - "outputs": [], - "source": [ - "TEXT = ''\n", - "OUT_PATH = 'tests-audios/'\n", - "# create output path\n", - "os.makedirs(OUT_PATH, exist_ok=True)\n", - "\n", - "SPEAKER_FILEID = None # if None use the first embedding from speakers.json\n", - "\n", - "# model vars \n", - "MODEL_PATH = 'best_model.pth.tar'\n", - "CONFIG_PATH = 'config.json'\n", - "SPEAKER_JSON = 'speakers.json'\n", - "\n", - "# vocoder vars\n", - "VOCODER_PATH = ''\n", - "VOCODER_CONFIG_PATH = ''\n", - "\n", - "USE_CUDA = True" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dV6cXXlfi72r" - }, - "source": [ - "# **Restore TTS Model**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "x1WgLFauWUPe" - }, - "outputs": [], - "source": [ - "# load the config\n", - "C = load_config(CONFIG_PATH)\n", - "C.forward_attn_mask = True\n", - "\n", - "# load the audio processor\n", - "ap = AudioProcessor(**C.audio)\n", - "\n", - "# if the vocabulary was passed, replace the default\n", - "if 'characters' in C.keys():\n", - " symbols, phonemes = make_symbols(**C.characters)\n", - "\n", - "speaker_embedding = None\n", - "speaker_embedding_dim = None\n", - "num_speakers = 0\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " num_speakers = len(speaker_mapping)\n", - " if C.use_external_speaker_embedding_file:\n", - " if SPEAKER_FILEID is not None:\n", - " speaker_embedding = speaker_mapping[SPEAKER_FILEID]['embedding']\n", - " else: # if speaker_fileid is not specificated use the first sample in speakers.json\n", - " choise_speaker = list(speaker_mapping.keys())[0]\n", - " print(\" Speaker: \",choise_speaker.split('_')[0],'was chosen automatically', \"(this speaker seen in training)\")\n", - " speaker_embedding = speaker_mapping[choise_speaker]['embedding']\n", - " speaker_embedding_dim = len(speaker_embedding)\n", - "\n", - "# load the model\n", - "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n", - "model = setup_model(num_chars, num_speakers, C, speaker_embedding_dim)\n", - "cp = torch.load(MODEL_PATH, map_location=torch.device('cpu'))\n", - "model.load_state_dict(cp['model'])\n", - "model.eval()\n", - "\n", - "if USE_CUDA:\n", - " model.cuda()\n", - "\n", - "model.decoder.set_r(cp['r'])\n", - "\n", - "# load vocoder model\n", - "if VOCODER_PATH!= \"\":\n", - " VC = load_config(VOCODER_CONFIG_PATH)\n", - " vocoder_model = setup_generator(VC)\n", - " vocoder_model.load_state_dict(torch.load(VOCODER_PATH, map_location=\"cpu\")[\"model\"])\n", - " vocoder_model.remove_weight_norm()\n", - " if USE_CUDA:\n", - " vocoder_model.cuda()\n", - " vocoder_model.eval()\n", - "else:\n", - " vocoder_model = None\n", - " VC = None\n", - "\n", - "# synthesize voice\n", - "use_griffin_lim = VOCODER_PATH== \"\"\n", - "\n", - "if not C.use_external_speaker_embedding_file:\n", - " if SPEAKER_FILEID.isdigit():\n", - " SPEAKER_FILEID = int(SPEAKER_FILEID)\n", - " else:\n", - " SPEAKER_FILEID = None\n", - "else:\n", - " SPEAKER_FILEID = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tNvVEoE30qY6" - }, - "source": [ - "Synthesize sentence with Speaker\n", - "\n", - "> Stop running the cell to leave!\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2o8fXkVSyXOa" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",choise_speaker.split('_')[0], \"(this speaker seen in training)\")\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vnV-FigfvsS2" - }, - "source": [ - "# **Select Speaker**\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RuCGOnJ_fgDV" - }, - "outputs": [], - "source": [ - "\n", - "# VCTK speakers not seen in training (new speakers)\n", - "VCTK_test_Speakers = [\"p225\", \"p234\", \"p238\", \"p245\", \"p248\", \"p261\", \"p294\", \"p302\", \"p326\", \"p335\", \"p347\"]\n", - "\n", - "# VCTK speakers seen in training\n", - "VCTK_train_Speakers = ['p244', 'p300', 'p303', 'p273', 'p292', 'p252', 'p254', 'p269', 'p345', 'p274', 'p363', 'p285', 'p351', 'p361', 'p295', 'p266', 'p307', 'p230', 'p339', 'p253', 'p310', 'p241', 'p256', 'p323', 'p237', 'p229', 'p298', 'p336', 'p276', 'p305', 'p255', 'p278', 'p299', 'p265', 'p267', 'p280', 'p260', 'p272', 'p262', 'p334', 'p283', 'p247', 'p246', 'p374', 'p297', 'p249', 'p250', 'p304', 'p240', 'p236', 'p312', 'p286', 'p263', 'p258', 'p313', 'p376', 'p279', 'p340', 'p362', 'p284', 'p231', 'p308', 'p277', 'p275', 'p333', 'p314', 'p330', 'p264', 'p226', 'p288', 'p343', 'p239', 'p232', 'p268', 'p270', 'p329', 'p227', 'p271', 'p228', 'p311', 'p301', 'p293', 'p364', 'p251', 'p317', 'p360', 'p281', 'p243', 'p287', 'p233', 'p259', 'p316', 'p257', 'p282', 'p306', 'p341', 'p318']\n", - "\n", - "\n", - "num_samples_speaker = 2 # In theory the more samples of the speaker the more similar to the real voice it will be!\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hkvv7gRcx4WV" - }, - "source": [ - "## **Example select a VCTK seen speaker in training**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BviNMI9UyCYz" - }, - "outputs": [], - "source": [ - "# get embedding\n", - "Speaker_choise = VCTK_train_Speakers[0] # choise one of training speakers\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " if C.use_external_speaker_embedding_file:\n", - " speaker_embeddings = []\n", - " for key in list(speaker_mapping.keys()):\n", - " if Speaker_choise in key:\n", - " if len(speaker_embeddings) < num_samples_speaker:\n", - " speaker_embeddings.append(speaker_mapping[key]['embedding'])\n", - " # takes the average of the embedings samples of the announcers\n", - " speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5e5_XnLsx3jg" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker seen in training)\")\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJ6VgT2a4vHW" - }, - "source": [ - "## **Example select a VCTK not seen speaker in training (new Speakers)**\n", - "\n", - "\n", - "> Fitting new Speakers :)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SZS57ZK-4vHa" - }, - "outputs": [], - "source": [ - "# get embedding\n", - "Speaker_choise = VCTK_test_Speakers[0] # choise one of training speakers\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " if C.use_external_speaker_embedding_file:\n", - " speaker_embeddings = []\n", - " for key in list(speaker_mapping.keys()):\n", - " if Speaker_choise in key:\n", - " if len(speaker_embeddings) < num_samples_speaker:\n", - " speaker_embeddings.append(speaker_mapping[key]['embedding'])\n", - " # takes the average of the embedings samples of the announcers\n", - " speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bbs85vzz4vHo" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LEE6mQLh5Who" - }, - "source": [ - "# **Example Synthesizing with your own voice :)**\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "La70gSB65nrs" - }, - "source": [ - " Download and load GE2E Speaker Encoder " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "r0IEFZ0B5vQg" - }, - "outputs": [], - "source": [ - "!wget -c -q --show-progress -O ./SpeakerEncoder-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/GE2E-SpeakerEncoder-iter25k.zip\n", - "!unzip ./SpeakerEncoder-checkpoint.zip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jEH8HCTh5mF6" - }, - "outputs": [], - "source": [ - "SE_MODEL_RUN_PATH = \"GE2E-SpeakerEncoder/\"\n", - "SE_MODEL_PATH = os.path.join(SE_MODEL_RUN_PATH, \"best_model.pth.tar\")\n", - "SE_CONFIG_PATH =os.path.join(SE_MODEL_RUN_PATH, \"config.json\")\n", - "USE_CUDA = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tOwkfQqT6-Qo" - }, - "outputs": [], - "source": [ - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.speaker_encoder.model import SpeakerEncoder\n", - "se_config = load_config(SE_CONFIG_PATH)\n", - "se_ap = AudioProcessor(**se_config['audio'])\n", - "\n", - "se_model = SpeakerEncoder(**se_config.model)\n", - "se_model.load_state_dict(torch.load(SE_MODEL_PATH)['model'])\n", - "se_model.eval()\n", - "if USE_CUDA:\n", - " se_model.cuda()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TLlbUFG8O36" - }, - "source": [ - "Upload a wav audio file in your voice.\n", - "\n", - "\n", - "> We recommend files longer than 3 seconds, the bigger the file the closer to your voice :)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_FWwHPjJ8NXl" - }, - "outputs": [], - "source": [ - "from google.colab import files\n", - "file_list = files.upload()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WWOf6sgbBbGY" - }, - "outputs": [], - "source": [ - "# extract embedding from wav files\n", - "speaker_embeddings = []\n", - "for name in file_list.keys():\n", - " if '.wav' in name:\n", - " mel_spec = se_ap.melspectrogram(se_ap.load_wav(name, sr=se_ap.sample_rate)).T\n", - " mel_spec = torch.FloatTensor(mel_spec[None, :, :])\n", - " if USE_CUDA:\n", - " mel_spec = mel_spec.cuda()\n", - " embedd = se_model.compute_embedding(mel_spec).cpu().detach().numpy().reshape(-1)\n", - " speaker_embeddings.append(embedd)\n", - " else:\n", - " print(\" You need upload Wav files, others files is not supported !!\")\n", - "\n", - "# takes the average of the embedings samples of the announcers\n", - "speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xmItcGac5WiG" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [ - "vnV-FigfvsS2", - "hkvv7gRcx4WV", - "QJ6VgT2a4vHW" - ], - "name": "Demo-Mozilla-TTS-MultiSpeaker-jia-et-al-2018.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018_With_GST.ipynb b/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018_With_GST.ipynb deleted file mode 100644 index f65d09a6..00000000 --- a/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018_With_GST.ipynb +++ /dev/null @@ -1,847 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yZK6UdwSFnOO" - }, - "source": [ - "# **Download and install Coqui TTS**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yvb0pX3WY6MN" - }, - "outputs": [], - "source": [ - "import os \n", - "!git clone https://github.com/Edresson/TTS -b dev-gst-embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iB9nl2UEG3SY" - }, - "outputs": [], - "source": [ - "!apt-get install espeak\n", - "os.chdir('TTS')\n", - "!pip install -r requirements.txt\n", - "!python setup.py develop\n", - "os.chdir('..')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w6Krn8k1inC_" - }, - "source": [ - "\n", - "\n", - "**Download Checkpoint**\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "PiYHf3lKhi9z" - }, - "outputs": [], - "source": [ - "!wget -c -q --show-progress -O ./TTS-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/Checkpoints-TTS-MultiSpeaker-Jia-et-al-2018-with-GST.zip\n", - "!unzip ./TTS-checkpoint.zip\n", - "\n", - "# Download gst style example\n", - "!wget https://github.com/Edresson/TTS/releases/download/v1.0.0/gst-style-example.wav" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MpYNgqrZcJKn" - }, - "source": [ - "**Utils Functions**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4KZA4b_CbMqx" - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "import argparse\n", - "import json\n", - "# pylint: disable=redefined-outer-name, unused-argument\n", - "import os\n", - "import string\n", - "import time\n", - "import sys\n", - "import numpy as np\n", - "\n", - "TTS_PATH = \"../content/TTS\"\n", - "# add libraries into environment\n", - "sys.path.append(TTS_PATH) # set this if TTS is not installed globally\n", - "\n", - "import torch\n", - "\n", - "from TTS.tts.utils.generic_utils import setup_model\n", - "from TTS.tts.utils.synthesis import synthesis\n", - "from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols\n", - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.utils.io import load_config\n", - "from TTS.vocoder.utils.generic_utils import setup_generator\n", - "\n", - "\n", - "def tts(model, vocoder_model, text, CONFIG, use_cuda, ap, use_gl, speaker_fileid, speaker_embedding=None, gst_style=None):\n", - " t_1 = time.time()\n", - " waveform, _, _, mel_postnet_spec, _, _ = synthesis(model, text, CONFIG, use_cuda, ap, speaker_fileid, gst_style, False, CONFIG.enable_eos_bos_chars, use_gl, speaker_embedding=speaker_embedding)\n", - " if CONFIG.model == \"Tacotron\" and not use_gl:\n", - " mel_postnet_spec = ap.out_linear_to_mel(mel_postnet_spec.T).T\n", - " if not use_gl:\n", - " waveform = vocoder_model.inference(torch.FloatTensor(mel_postnet_spec.T).unsqueeze(0))\n", - " if use_cuda and not use_gl:\n", - " waveform = waveform.cpu()\n", - " if not use_gl:\n", - " waveform = waveform.numpy()\n", - " waveform = waveform.squeeze()\n", - " rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n", - " tps = (time.time() - t_1) / len(waveform)\n", - " print(\" > Run-time: {}\".format(time.time() - t_1))\n", - " print(\" > Real-time factor: {}\".format(rtf))\n", - " print(\" > Time per step: {}\".format(tps))\n", - " return waveform\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ENA2OumIVeMA" - }, - "source": [ - "# **Vars definitions**\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jPD0d_XpVXmY" - }, - "outputs": [], - "source": [ - "TEXT = ''\n", - "OUT_PATH = 'tests-audios/'\n", - "# create output path\n", - "os.makedirs(OUT_PATH, exist_ok=True)\n", - "\n", - "SPEAKER_FILEID = None # if None use the first embedding from speakers.json\n", - "\n", - "# model vars \n", - "MODEL_PATH = 'best_model.pth.tar'\n", - "CONFIG_PATH = 'config.json'\n", - "SPEAKER_JSON = 'speakers.json'\n", - "\n", - "# vocoder vars\n", - "VOCODER_PATH = ''\n", - "VOCODER_CONFIG_PATH = ''\n", - "\n", - "USE_CUDA = True" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dV6cXXlfi72r" - }, - "source": [ - "# **Restore TTS Model**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "x1WgLFauWUPe" - }, - "outputs": [], - "source": [ - "# load the config\n", - "C = load_config(CONFIG_PATH)\n", - "C.forward_attn_mask = True\n", - "\n", - "# load the audio processor\n", - "ap = AudioProcessor(**C.audio)\n", - "\n", - "# if the vocabulary was passed, replace the default\n", - "if 'characters' in C.keys():\n", - " symbols, phonemes = make_symbols(**C.characters)\n", - "\n", - "speaker_embedding = None\n", - "speaker_embedding_dim = None\n", - "num_speakers = 0\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " num_speakers = len(speaker_mapping)\n", - " if C.use_external_speaker_embedding_file:\n", - " if SPEAKER_FILEID is not None:\n", - " speaker_embedding = speaker_mapping[SPEAKER_FILEID]['embedding']\n", - " else: # if speaker_fileid is not specificated use the first sample in speakers.json\n", - " choise_speaker = list(speaker_mapping.keys())[0]\n", - " print(\" Speaker: \",choise_speaker.split('_')[0],'was chosen automatically', \"(this speaker seen in training)\")\n", - " speaker_embedding = speaker_mapping[choise_speaker]['embedding']\n", - " speaker_embedding_dim = len(speaker_embedding)\n", - "\n", - "# load the model\n", - "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n", - "model = setup_model(num_chars, num_speakers, C, speaker_embedding_dim)\n", - "cp = torch.load(MODEL_PATH, map_location=torch.device('cpu'))\n", - "model.load_state_dict(cp['model'])\n", - "model.eval()\n", - "\n", - "if USE_CUDA:\n", - " model.cuda()\n", - "\n", - "model.decoder.set_r(cp['r'])\n", - "\n", - "# load vocoder model\n", - "if VOCODER_PATH!= \"\":\n", - " VC = load_config(VOCODER_CONFIG_PATH)\n", - " vocoder_model = setup_generator(VC)\n", - " vocoder_model.load_state_dict(torch.load(VOCODER_PATH, map_location=\"cpu\")[\"model\"])\n", - " vocoder_model.remove_weight_norm()\n", - " if USE_CUDA:\n", - " vocoder_model.cuda()\n", - " vocoder_model.eval()\n", - "else:\n", - " vocoder_model = None\n", - " VC = None\n", - "\n", - "# synthesize voice\n", - "use_griffin_lim = VOCODER_PATH== \"\"\n", - "\n", - "if not C.use_external_speaker_embedding_file:\n", - " if SPEAKER_FILEID.isdigit():\n", - " SPEAKER_FILEID = int(SPEAKER_FILEID)\n", - " else:\n", - " SPEAKER_FILEID = None\n", - "else:\n", - " SPEAKER_FILEID = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tNvVEoE30qY6" - }, - "source": [ - "Synthesize sentence with Speaker\n", - "\n", - "> Stop running the cell to leave!\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2o8fXkVSyXOa" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",choise_speaker.split('_')[0], \"(this speaker seen in training)\")\n", - "gst_style = 'gst-style-example.wav'\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vnV-FigfvsS2" - }, - "source": [ - "# **Select Speaker**\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RuCGOnJ_fgDV" - }, - "outputs": [], - "source": [ - "\n", - "# VCTK speakers not seen in training (new speakers)\n", - "VCTK_test_Speakers = [\"p225\", \"p234\", \"p238\", \"p245\", \"p248\", \"p261\", \"p294\", \"p302\", \"p326\", \"p335\", \"p347\"]\n", - "\n", - "# VCTK speakers seen in training\n", - "VCTK_train_Speakers = ['p244', 'p300', 'p303', 'p273', 'p292', 'p252', 'p254', 'p269', 'p345', 'p274', 'p363', 'p285', 'p351', 'p361', 'p295', 'p266', 'p307', 'p230', 'p339', 'p253', 'p310', 'p241', 'p256', 'p323', 'p237', 'p229', 'p298', 'p336', 'p276', 'p305', 'p255', 'p278', 'p299', 'p265', 'p267', 'p280', 'p260', 'p272', 'p262', 'p334', 'p283', 'p247', 'p246', 'p374', 'p297', 'p249', 'p250', 'p304', 'p240', 'p236', 'p312', 'p286', 'p263', 'p258', 'p313', 'p376', 'p279', 'p340', 'p362', 'p284', 'p231', 'p308', 'p277', 'p275', 'p333', 'p314', 'p330', 'p264', 'p226', 'p288', 'p343', 'p239', 'p232', 'p268', 'p270', 'p329', 'p227', 'p271', 'p228', 'p311', 'p301', 'p293', 'p364', 'p251', 'p317', 'p360', 'p281', 'p243', 'p287', 'p233', 'p259', 'p316', 'p257', 'p282', 'p306', 'p341', 'p318']\n", - "\n", - "\n", - "num_samples_speaker = 2 # In theory the more samples of the speaker the more similar to the real voice it will be!\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hkvv7gRcx4WV" - }, - "source": [ - "## **Example select a VCTK seen speaker in training**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BviNMI9UyCYz" - }, - "outputs": [], - "source": [ - "# get embedding\n", - "Speaker_choise = VCTK_train_Speakers[0] # choise one of training speakers\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " if C.use_external_speaker_embedding_file:\n", - " speaker_embeddings = []\n", - " for key in list(speaker_mapping.keys()):\n", - " if Speaker_choise in key:\n", - " if len(speaker_embeddings) < num_samples_speaker:\n", - " speaker_embeddings.append(speaker_mapping[key]['embedding'])\n", - " # takes the average of the embedings samples of the announcers\n", - " speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5e5_XnLsx3jg" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker seen in training)\")\n", - "gst_style = 'gst-style-example.wav'\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJ6VgT2a4vHW" - }, - "source": [ - "## **Example select a VCTK not seen speaker in training (new Speakers)**\n", - "\n", - "\n", - "> Fitting new Speakers :)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SZS57ZK-4vHa" - }, - "outputs": [], - "source": [ - "# get embedding\n", - "Speaker_choise = VCTK_test_Speakers[0] # choise one of training speakers\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " if C.use_external_speaker_embedding_file:\n", - " speaker_embeddings = []\n", - " for key in list(speaker_mapping.keys()):\n", - " if Speaker_choise in key:\n", - " if len(speaker_embeddings) < num_samples_speaker:\n", - " speaker_embeddings.append(speaker_mapping[key]['embedding'])\n", - " # takes the average of the embedings samples of the announcers\n", - " speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bbs85vzz4vHo" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "gst_style = 'gst-style-example.wav'\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_G_HweN04W-" - }, - "source": [ - "# **Changing GST tokens manually (without wav reference)**" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jyFP5syW2bjt" - }, - "source": [ - "You can define tokens manually, this way you can increase/decrease the function of a given GST token. For example a token is responsible for the length of the speaker's pauses, if you increase the value of that token you will have longer pauses and if you decrease it you will have shorter pauses." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SpwjDjCM2a3Y" - }, - "outputs": [], - "source": [ - "# set gst tokens, in this model we have 5 tokens\n", - "gst_style = {\"0\": 0, \"1\": 0, \"3\": 0, \"4\": 0}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qWChMbI_0z5X" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uFjUi9xQ3mG3" - }, - "outputs": [], - "source": [ - "gst_style = {\"0\": 0.9, \"1\": 0, \"3\": 0, \"4\": 0}\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Uw0d6gWg4L27" - }, - "outputs": [], - "source": [ - "gst_style = {\"0\": -0.9, \"1\": 0, \"3\": 0, \"4\": 0}\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "V9izw4-54-Tl" - }, - "outputs": [], - "source": [ - "gst_style = {\"0\": 0, \"1\": 0.9, \"3\": 0, \"4\": 0}\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LEE6mQLh5Who" - }, - "source": [ - "# **Example Synthesizing with your own voice :)**\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "La70gSB65nrs" - }, - "source": [ - " Download and load GE2E Speaker Encoder " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "r0IEFZ0B5vQg" - }, - "outputs": [], - "source": [ - "!wget -c -q --show-progress -O ./SpeakerEncoder-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/GE2E-SpeakerEncoder-iter25k.zip\n", - "!unzip ./SpeakerEncoder-checkpoint.zip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jEH8HCTh5mF6" - }, - "outputs": [], - "source": [ - "SE_MODEL_RUN_PATH = \"GE2E-SpeakerEncoder/\"\n", - "SE_MODEL_PATH = os.path.join(SE_MODEL_RUN_PATH, \"best_model.pth.tar\")\n", - "SE_CONFIG_PATH =os.path.join(SE_MODEL_RUN_PATH, \"config.json\")\n", - "USE_CUDA = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tOwkfQqT6-Qo" - }, - "outputs": [], - "source": [ - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.speaker_encoder.model import SpeakerEncoder\n", - "se_config = load_config(SE_CONFIG_PATH)\n", - "se_ap = AudioProcessor(**se_config['audio'])\n", - "\n", - "se_model = SpeakerEncoder(**se_config.model)\n", - "se_model.load_state_dict(torch.load(SE_MODEL_PATH)['model'])\n", - "se_model.eval()\n", - "if USE_CUDA:\n", - " se_model.cuda()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TLlbUFG8O36" - }, - "source": [ - "Upload one or more wav audio files in your voice.\n", - "\n", - "\n", - "> We recommend files longer than 3 seconds, the bigger the file the closer to your voice :)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_FWwHPjJ8NXl" - }, - "outputs": [], - "source": [ - "# select one or more wav files\n", - "from google.colab import files\n", - "file_list = files.upload()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WWOf6sgbBbGY" - }, - "outputs": [], - "source": [ - "# extract embedding from wav files\n", - "speaker_embeddings = []\n", - "for name in file_list.keys():\n", - " if '.wav' in name:\n", - " mel_spec = se_ap.melspectrogram(se_ap.load_wav(name, sr=se_ap.sample_rate)).T\n", - " mel_spec = torch.FloatTensor(mel_spec[None, :, :])\n", - " if USE_CUDA:\n", - " mel_spec = mel_spec.cuda()\n", - " embedd = se_model.compute_embedding(mel_spec).cpu().detach().numpy().reshape(-1)\n", - " speaker_embeddings.append(embedd)\n", - " else:\n", - " print(\"You need upload Wav files, others files is not supported !!\")\n", - "\n", - "# takes the average of the embedings samples of the announcers\n", - "speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AQ7eP31d9yzq" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n", - "gst_style = {\"0\": 0, \"1\": 0.0, \"3\": 0, \"4\": 0}\n", - "gst_style = 'gst-style-example.wav'\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "11i10yE1-LMJ" - }, - "source": [ - "Uploading your own GST reference wav file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eKohSQG1-KkT" - }, - "outputs": [], - "source": [ - "# select one wav file for GST reference\n", - "from google.colab import files\n", - "file_list = files.upload()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xmItcGac5WiG" - }, - "outputs": [], - "source": [ - "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n", - "gst_style = list(file_list.keys())[0]\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [ - "yZK6UdwSFnOO", - "ENA2OumIVeMA", - "dV6cXXlfi72r", - "vnV-FigfvsS2", - "g_G_HweN04W-", - "LEE6mQLh5Who" - ], - "name": "Demo-Mozilla-TTS-MultiSpeaker-jia-et-al-2018-With-GST.ipynb", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/requirements.txt b/requirements.txt index fafd5112..c6ce7672 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ librosa==0.8.0 matplotlib numpy==1.18.5 pandas -phonemizer>=2.2.0 pypinyin pysbd pyyaml From a14fcf2a13fb76dbd64ecb10212db42c682ab864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 18 May 2021 15:11:12 +0200 Subject: [PATCH 81/87] remove text_processing test --- TTS/tts/utils/text/cleaners.py | 2 +- tests/test_text_processing.py | 173 --------------------------------- 2 files changed, 1 insertion(+), 174 deletions(-) delete mode 100644 tests/test_text_processing.py diff --git a/TTS/tts/utils/text/cleaners.py b/TTS/tts/utils/text/cleaners.py index d61738a6..2eddcdb8 100644 --- a/TTS/tts/utils/text/cleaners.py +++ b/TTS/tts/utils/text/cleaners.py @@ -14,7 +14,7 @@ import re from unidecode import unidecode -from TTS.tts.utils.chinese_mandarin.numbers import replace_numbers_to_characters_in_text +from TTS.tts.utils.text.chinese_mandarin.numbers import replace_numbers_to_characters_in_text from .abbreviations import abbreviations_en, abbreviations_fr from .number_norm import normalize_numbers diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py deleted file mode 100644 index 711021ab..00000000 --- a/tests/test_text_processing.py +++ /dev/null @@ -1,173 +0,0 @@ -# pylint: disable=unused-wildcard-import -# pylint: disable=wildcard-import -# pylint: disable=unused-import -from TTS.tts.configs import TacotronConfig -from TTS.tts.utils.text import * - -conf = TacotronConfig() - - -def test_phoneme_to_sequence(): - - text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" - text_cleaner = ["phoneme_cleaners"] - lang = "en-us" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "ɹiːsənt ɹᵻsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪŋkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹᵻspɑːnsᵻbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjʊleɪʃən ænd lɜːnɪŋ!" - assert text_hat == text_hat_with_params == gt - - # multiple punctuations - text = "Be a voice, not an! echo?" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ?" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # not ending with punctuation - text = "Be a voice, not an! echo" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # original - text = "Be a voice, not an echo!" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # extra space after the sentence - text = "Be a voice, not an! echo. " - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ." - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # extra space after the sentence - text = "Be a voice, not an! echo. " - sequence = phoneme_to_sequence(text, text_cleaner, lang, True) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "^biː ɐ vɔɪs, nɑːt æn! ɛkoʊ.~" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # padding char - text = "_Be a _voice, not an! echo_" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - -def test_phoneme_to_sequence_with_blank_token(): - - text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" - text_cleaner = ["phoneme_cleaners"] - lang = "en-us" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "ɹiːsənt ɹᵻsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪŋkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹᵻspɑːnsᵻbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjʊleɪʃən ænd lɜːnɪŋ!" - assert text_hat == text_hat_with_params == gt - - # multiple punctuations - text = "Be a voice, not an! echo?" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ?" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # not ending with punctuation - text = "Be a voice, not an! echo" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # original - text = "Be a voice, not an echo!" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # extra space after the sentence - text = "Be a voice, not an! echo. " - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ." - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # extra space after the sentence - text = "Be a voice, not an! echo. " - sequence = phoneme_to_sequence(text, text_cleaner, lang, True) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "^biː ɐ vɔɪs, nɑːt æn! ɛkoʊ.~" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # padding char - text = "_Be a _voice, not an! echo_" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - -def test_text2phone(): - text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" - gt = "ɹ|iː|s|ə|n|t| |ɹ|ᵻ|s|ɜː|tʃ| |æ|t| |h|ɑːɹ|v|ɚ|d| |h|ɐ|z| |ʃ|oʊ|n| |m|ɛ|d|ᵻ|t|eɪ|ɾ|ɪ|ŋ| |f|ɔː|ɹ| |æ|z| |l|ɪ|ɾ|əl| |æ|z| |eɪ|t| |w|iː|k|s| |k|æ|n| |æ|k|tʃ|uː|əl|i| |ɪ|ŋ|k|ɹ|iː|s|,| |ð|ə| |ɡ|ɹ|eɪ| |m|æ|ɾ|ɚ|ɹ| |ɪ|n|ð|ə| |p|ɑːɹ|t|s| |ʌ|v|ð|ə| |b|ɹ|eɪ|n| |ɹ|ᵻ|s|p|ɑː|n|s|ᵻ|b|əl| |f|ɔː|ɹ| |ɪ|m|oʊ|ʃ|ə|n|əl| |ɹ|ɛ|ɡ|j|ʊ|l|eɪ|ʃ|ə|n| |æ|n|d| |l|ɜː|n|ɪ|ŋ|!" - lang = "en-us" - ph = text2phone(text, lang) - assert gt == ph From ccfaa6b1d5f0ede98f715df4dba950459d2bba70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Tue, 18 May 2021 15:38:22 +0200 Subject: [PATCH 82/87] add `needs_phonemizer` field to models.json. If set true these models are only compatible with v0.0.13 or below. --- TTS/.models.json | 37 +++++++++++++++++++++++----------- TTS/tts/utils/text/__init__.py | 2 +- TTS/utils/manage.py | 3 +++ tests/test_synthesize.py | 30 +++++++++++++-------------- 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/TTS/.models.json b/TTS/.models.json index 5fcfa86b..b926f120 100644 --- a/TTS/.models.json +++ b/TTS/.models.json @@ -6,7 +6,8 @@ "description": "EK1 en-rp tacotron2 by NMStoker", "github_rls_url": "https://github.com/coqui-ai/TTS/releases/download/v0.0.10/tts_models--en--ek1--tacotron2.zip", "default_vocoder": "vocoder_models/en/ek1/wavegrad", - "commit": "c802255" + "commit": "c802255", + "needs_phonemizer": true } }, "ljspeech":{ @@ -17,7 +18,8 @@ "commit": "bae2ad0f", "author": "Eren Gölge @erogol", "license": "", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": false }, "glow-tts":{ "description": "", @@ -27,7 +29,8 @@ "commit": "", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true }, "tacotron2-DCA": { "description": "", @@ -36,7 +39,8 @@ "commit": "", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true }, "speedy-speech-wn":{ "description": "Speedy Speech model with wavenet decoder.", @@ -45,7 +49,8 @@ "commit": "77b6145", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true } }, "vctk":{ @@ -56,7 +61,9 @@ "commit": "b531fa69", "author": "Edresson Casanova", "license": "", - "contact":"" + "contact":"", + "needs_phonemizer": true + } }, @@ -68,7 +75,8 @@ "commit": "bae2ad0f", "author": "Eren Gölge @erogol", "license": "", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true } } }, @@ -80,7 +88,8 @@ "commit": "", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true } } }, @@ -92,7 +101,8 @@ "commit": "", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true } } }, @@ -112,7 +122,8 @@ "author": "@r-dh", "default_vocoder": "vocoder_models/nl/mai/parallel-wavegan", "stats_file": null, - "commit": "540d811" + "commit": "540d811", + "needs_phonemizer": true } } }, @@ -123,7 +134,8 @@ "author": "@erogol", "default_vocoder": "vocoder_models/universal/libri-tts/fullband-melgan", "license":"", - "contact": "egolge@coqui.com" + "contact": "egolge@coqui.com", + "needs_phonemizer": true } } }, @@ -133,7 +145,8 @@ "github_rls_url": "https://github.com/coqui-ai/TTS/releases/download/v0.0.11/tts_models--de--thorsten--tacotron2-DCA.zip", "default_vocoder": "vocoder_models/de/thorsten/wavegrad", "author": "@thorstenMueller", - "commit": "unknown" + "commit": "unknown", + "needs_phonemizer": true } } } diff --git a/TTS/tts/utils/text/__init__.py b/TTS/tts/utils/text/__init__.py index 2b73d4e4..2ead9561 100644 --- a/TTS/tts/utils/text/__init__.py +++ b/TTS/tts/utils/text/__init__.py @@ -39,7 +39,7 @@ def text2phone(text, language): if language == "zh-CN": ph = chinese_text_to_phonemes(text) return ph - raise ValueError(f" [!] Language {language} is nor supported for phonemization.") + raise ValueError(f" [!] Language {language} is not supported for phonemization.") def intersperse(sequence, token): diff --git a/TTS/utils/manage.py b/TTS/utils/manage.py index fdc141ec..9630873f 100644 --- a/TTS/utils/manage.py +++ b/TTS/utils/manage.py @@ -101,6 +101,9 @@ class ModelManager(object): output_path = os.path.join(self.output_prefix, model_full_name) output_model_path = os.path.join(output_path, "model_file.pth.tar") output_config_path = os.path.join(output_path, "config.json") + # NOTE : band-aid for removing phoneme support + if 'needs_phonemizer' in model_item and model_item['needs_phonemizer']: + raise RuntimeError(' [!] Use 🐸TTS <= v0.0.13 for this model. Current version does not support phoneme based models.') if os.path.exists(output_path): print(f" > {model_name} is already downloaded.") else: diff --git a/tests/test_synthesize.py b/tests/test_synthesize.py index 526f7dc8..62eb6dbe 100644 --- a/tests/test_synthesize.py +++ b/tests/test_synthesize.py @@ -10,19 +10,19 @@ def test_synthesize(): # single speaker model run_cli(f'tts --text "This is an example." --out_path "{output_path}"') - run_cli( - "tts --model_name tts_models/en/ljspeech/speedy-speech-wn " - f'--text "This is an example." --out_path "{output_path}"' - ) - run_cli( - "tts --model_name tts_models/en/ljspeech/speedy-speech-wn " - "--vocoder_name vocoder_models/en/ljspeech/multiband-melgan " - f'--text "This is an example." --out_path "{output_path}"' - ) + # run_cli( + # "tts --model_name tts_models/en/ljspeech/speedy-speech-wn " + # f'--text "This is an example." --out_path "{output_path}"' + # ) + # run_cli( + # "tts --model_name tts_models/en/ljspeech/speedy-speech-wn " + # "--vocoder_name vocoder_models/en/ljspeech/multiband-melgan " + # f'--text "This is an example." --out_path "{output_path}"' + # ) - # multi-speaker model - run_cli("tts --model_name tts_models/en/vctk/sc-glow-tts --list_speaker_idxs") - run_cli( - f'tts --model_name tts_models/en/vctk/sc-glow-tts --speaker_idx "p304" ' - f'--text "This is an example." --out_path "{output_path}"' - ) + # # multi-speaker model + # run_cli("tts --model_name tts_models/en/vctk/sc-glow-tts --list_speaker_idxs") + # run_cli( + # f'tts --model_name tts_models/en/vctk/sc-glow-tts --speaker_idx "p304" ' + # f'--text "This is an example." --out_path "{output_path}"' + # ) From faedea4b60c828cdc1e430f44652b09e60f53d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 19 May 2021 00:08:07 +0200 Subject: [PATCH 83/87] set use_phonemes False in configs --- tests/inputs/test_glow_tts.json | 2 +- tests/inputs/test_speedy_speech.json | 2 +- tests/inputs/test_tacotron2_config.json | 2 +- tests/inputs/test_tacotron_bd_config.json | 2 +- tests/inputs/test_tacotron_config.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/inputs/test_glow_tts.json b/tests/inputs/test_glow_tts.json index 0ee9395b..64cc3822 100644 --- a/tests/inputs/test_glow_tts.json +++ b/tests/inputs/test_glow_tts.json @@ -128,7 +128,7 @@ // PHONEMES "phoneme_cache_path": "tests/outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST diff --git a/tests/inputs/test_speedy_speech.json b/tests/inputs/test_speedy_speech.json index c4e27737..a29fc992 100644 --- a/tests/inputs/test_speedy_speech.json +++ b/tests/inputs/test_speedy_speech.json @@ -132,7 +132,7 @@ // PHONEMES "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronoun[ciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronoun[ciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST diff --git a/tests/inputs/test_tacotron2_config.json b/tests/inputs/test_tacotron2_config.json index 2bf1f840..cc2c1bb5 100644 --- a/tests/inputs/test_tacotron2_config.json +++ b/tests/inputs/test_tacotron2_config.json @@ -141,7 +141,7 @@ // PHONEMES "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST diff --git a/tests/inputs/test_tacotron_bd_config.json b/tests/inputs/test_tacotron_bd_config.json index b6092f4f..9d2935aa 100644 --- a/tests/inputs/test_tacotron_bd_config.json +++ b/tests/inputs/test_tacotron_bd_config.json @@ -141,7 +141,7 @@ // PHONEMES "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST diff --git a/tests/inputs/test_tacotron_config.json b/tests/inputs/test_tacotron_config.json index 12da4762..c8fae623 100644 --- a/tests/inputs/test_tacotron_config.json +++ b/tests/inputs/test_tacotron_config.json @@ -141,7 +141,7 @@ // PHONEMES "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST From 8a7c40736ced7e3529a18c5732a1fb72bd696442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 19 May 2021 01:27:26 +0200 Subject: [PATCH 84/87] set use_phonemes false --- TTS/tts/datasets/TTSDataset.py | 2 +- TTS/tts/utils/text/__init__.py | 14 +++++++------- TTS/utils/manage.py | 6 ++++-- tests/tts_tests/test_align_tts_train.py | 2 +- tests/tts_tests/test_glow_tts_train.py | 2 +- tests/tts_tests/test_speedy_speech_train.py | 2 +- tests/tts_tests/test_tacotron2_train.py | 2 +- tests/tts_tests/test_tacotron_train.py | 2 +- 8 files changed, 17 insertions(+), 15 deletions(-) diff --git a/TTS/tts/datasets/TTSDataset.py b/TTS/tts/datasets/TTSDataset.py index b613e37c..4ca93232 100644 --- a/TTS/tts/datasets/TTSDataset.py +++ b/TTS/tts/datasets/TTSDataset.py @@ -25,7 +25,7 @@ class MyDataset(Dataset): batch_group_size=0, min_seq_len=0, max_seq_len=float("inf"), - use_phonemes=True, + use_phonemes=False, phoneme_cache_path=None, phoneme_language="en-us", enable_eos_bos=False, diff --git a/TTS/tts/utils/text/__init__.py b/TTS/tts/utils/text/__init__.py index 2ead9561..9367e6e2 100644 --- a/TTS/tts/utils/text/__init__.py +++ b/TTS/tts/utils/text/__init__.py @@ -4,8 +4,8 @@ import re from packaging import version -from TTS.tts.utils.text.chinese_mandarin.phonemizer import chinese_text_to_phonemes from TTS.tts.utils.text import cleaners +from TTS.tts.utils.text.chinese_mandarin.phonemizer import chinese_text_to_phonemes from TTS.tts.utils.text.symbols import _bos, _eos, _punctuations, make_symbols, phonemes, symbols # pylint: disable=unnecessary-comprehension @@ -27,12 +27,12 @@ PHONEME_PUNCTUATION_PATTERN = r"[" + _punctuations.replace(" ", "") + "]+" def text2phone(text, language): """Convert graphemes to phonemes. - Parameters: - text (str): text to phonemize - language (str): language of the text - Returns: - ph (str): phonemes as a string seperated by "|" - ph = "ɪ|g|ˈ|z|æ|m|p|ə|l" + Parameters: + text (str): text to phonemize + language (str): language of the text + Returns: + ph (str): phonemes as a string seperated by "|" + ph = "ɪ|g|ˈ|z|æ|m|p|ə|l" """ # TO REVIEW : How to have a good implementation for this? diff --git a/TTS/utils/manage.py b/TTS/utils/manage.py index 9630873f..2e3caa81 100644 --- a/TTS/utils/manage.py +++ b/TTS/utils/manage.py @@ -102,8 +102,10 @@ class ModelManager(object): output_model_path = os.path.join(output_path, "model_file.pth.tar") output_config_path = os.path.join(output_path, "config.json") # NOTE : band-aid for removing phoneme support - if 'needs_phonemizer' in model_item and model_item['needs_phonemizer']: - raise RuntimeError(' [!] Use 🐸TTS <= v0.0.13 for this model. Current version does not support phoneme based models.') + if "needs_phonemizer" in model_item and model_item["needs_phonemizer"]: + raise RuntimeError( + " [!] Use 🐸TTS <= v0.0.13 for this model. Current version does not support phoneme based models." + ) if os.path.exists(output_path): print(f" > {model_name} is already downloaded.") else: diff --git a/tests/tts_tests/test_align_tts_train.py b/tests/tts_tests/test_align_tts_train.py index 97ffc7a7..848f46c1 100644 --- a/tests/tts_tests/test_align_tts_train.py +++ b/tests/tts_tests/test_align_tts_train.py @@ -15,7 +15,7 @@ config = AlignTTSConfig( num_loader_workers=0, num_val_loader_workers=0, text_cleaner="english_cleaners", - use_phonemes=True, + use_phonemes=False, phoneme_language="en-us", phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), run_eval=True, diff --git a/tests/tts_tests/test_glow_tts_train.py b/tests/tts_tests/test_glow_tts_train.py index a92d837f..8d9b2982 100644 --- a/tests/tts_tests/test_glow_tts_train.py +++ b/tests/tts_tests/test_glow_tts_train.py @@ -15,7 +15,7 @@ config = GlowTTSConfig( num_loader_workers=0, num_val_loader_workers=0, text_cleaner="english_cleaners", - use_phonemes=True, + use_phonemes=False, phoneme_language="en-us", phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), run_eval=True, diff --git a/tests/tts_tests/test_speedy_speech_train.py b/tests/tts_tests/test_speedy_speech_train.py index 19d24ab3..b76f568a 100644 --- a/tests/tts_tests/test_speedy_speech_train.py +++ b/tests/tts_tests/test_speedy_speech_train.py @@ -15,7 +15,7 @@ config = SpeedySpeechConfig( num_loader_workers=0, num_val_loader_workers=0, text_cleaner="english_cleaners", - use_phonemes=True, + use_phonemes=False, phoneme_language="en-us", phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), run_eval=True, diff --git a/tests/tts_tests/test_tacotron2_train.py b/tests/tts_tests/test_tacotron2_train.py index 94e02646..dbec309b 100644 --- a/tests/tts_tests/test_tacotron2_train.py +++ b/tests/tts_tests/test_tacotron2_train.py @@ -16,7 +16,7 @@ config = Tacotron2Config( num_loader_workers=0, num_val_loader_workers=0, text_cleaner="english_cleaners", - use_phonemes=True, + use_phonemes=False, phoneme_language="en-us", phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), run_eval=True, diff --git a/tests/tts_tests/test_tacotron_train.py b/tests/tts_tests/test_tacotron_train.py index 0f651f27..34ee6e06 100644 --- a/tests/tts_tests/test_tacotron_train.py +++ b/tests/tts_tests/test_tacotron_train.py @@ -15,7 +15,7 @@ config = TacotronConfig( num_loader_workers=0, num_val_loader_workers=0, text_cleaner="english_cleaners", - use_phonemes=True, + use_phonemes=False, phoneme_language="en-us", phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), run_eval=True, From 9b706c5583de53cb18ccba615e98c2e52710abf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 19 May 2021 03:15:54 +0200 Subject: [PATCH 85/87] enable phonemes in test to match the attention masks --- tests/tts_tests/test_glow_tts_train.py | 6 +++--- tests/tts_tests/test_speedy_speech_train.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/tts_tests/test_glow_tts_train.py b/tests/tts_tests/test_glow_tts_train.py index 8d9b2982..00c7e852 100644 --- a/tests/tts_tests/test_glow_tts_train.py +++ b/tests/tts_tests/test_glow_tts_train.py @@ -15,9 +15,9 @@ config = GlowTTSConfig( num_loader_workers=0, num_val_loader_workers=0, text_cleaner="english_cleaners", - use_phonemes=False, - phoneme_language="en-us", - phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + use_phonemes=True, + phoneme_language="zh-CN", + phoneme_cache_path='tests/data/ljspeech/phoneme_cache/', run_eval=True, test_delay_epochs=-1, epochs=1, diff --git a/tests/tts_tests/test_speedy_speech_train.py b/tests/tts_tests/test_speedy_speech_train.py index b76f568a..cc2845c2 100644 --- a/tests/tts_tests/test_speedy_speech_train.py +++ b/tests/tts_tests/test_speedy_speech_train.py @@ -15,9 +15,9 @@ config = SpeedySpeechConfig( num_loader_workers=0, num_val_loader_workers=0, text_cleaner="english_cleaners", - use_phonemes=False, - phoneme_language="en-us", - phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + use_phonemes=True, + phoneme_language="zh-CN", + phoneme_cache_path='tests/data/ljspeech/phoneme_cache/', run_eval=True, test_delay_epochs=-1, epochs=1, From f5a9950a2e31b26ecc73cca2f3ac2b6e26d88c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20G=C3=B6lge?= Date: Wed, 19 May 2021 03:16:57 +0200 Subject: [PATCH 86/87] phoneme files for testing --- .../ljspeech/phoneme_cache/LJ001-0001_phoneme.npy | Bin 0 -> 700 bytes .../ljspeech/phoneme_cache/LJ001-0002_phoneme.npy | Bin 0 -> 244 bytes .../ljspeech/phoneme_cache/LJ001-0003_phoneme.npy | Bin 0 -> 704 bytes .../ljspeech/phoneme_cache/LJ001-0004_phoneme.npy | Bin 0 -> 440 bytes .../ljspeech/phoneme_cache/LJ001-0005_phoneme.npy | Bin 0 -> 652 bytes .../ljspeech/phoneme_cache/LJ001-0006_phoneme.npy | Bin 0 -> 412 bytes .../ljspeech/phoneme_cache/LJ001-0007_phoneme.npy | Bin 0 -> 588 bytes .../ljspeech/phoneme_cache/LJ001-0008_phoneme.npy | Bin 0 -> 208 bytes .../ljspeech/phoneme_cache/LJ001-0009_phoneme.npy | Bin 0 -> 536 bytes .../ljspeech/phoneme_cache/LJ001-0010_phoneme.npy | Bin 0 -> 576 bytes .../ljspeech/phoneme_cache/LJ001-0011_phoneme.npy | Bin 0 -> 396 bytes .../ljspeech/phoneme_cache/LJ001-0012_phoneme.npy | Bin 0 -> 532 bytes .../ljspeech/phoneme_cache/LJ001-0013_phoneme.npy | Bin 0 -> 288 bytes .../ljspeech/phoneme_cache/LJ001-0014_phoneme.npy | Bin 0 -> 736 bytes .../ljspeech/phoneme_cache/LJ001-0015_phoneme.npy | Bin 0 -> 716 bytes .../ljspeech/phoneme_cache/LJ001-0016_phoneme.npy | Bin 0 -> 416 bytes .../ljspeech/phoneme_cache/LJ001-0017_phoneme.npy | Bin 0 -> 604 bytes .../ljspeech/phoneme_cache/LJ001-0018_phoneme.npy | Bin 0 -> 584 bytes .../ljspeech/phoneme_cache/LJ001-0019_phoneme.npy | Bin 0 -> 524 bytes .../ljspeech/phoneme_cache/LJ001-0020_phoneme.npy | Bin 0 -> 364 bytes .../ljspeech/phoneme_cache/LJ001-0021_phoneme.npy | Bin 0 -> 616 bytes .../ljspeech/phoneme_cache/LJ001-0022_phoneme.npy | Bin 0 -> 528 bytes .../ljspeech/phoneme_cache/LJ001-0023_phoneme.npy | Bin 0 -> 640 bytes .../ljspeech/phoneme_cache/LJ001-0024_phoneme.npy | Bin 0 -> 600 bytes .../ljspeech/phoneme_cache/LJ001-0025_phoneme.npy | Bin 0 -> 544 bytes .../ljspeech/phoneme_cache/LJ001-0026_phoneme.npy | Bin 0 -> 444 bytes .../ljspeech/phoneme_cache/LJ001-0027_phoneme.npy | Bin 0 -> 664 bytes .../ljspeech/phoneme_cache/LJ001-0028_phoneme.npy | Bin 0 -> 392 bytes .../ljspeech/phoneme_cache/LJ001-0029_phoneme.npy | Bin 0 -> 400 bytes .../ljspeech/phoneme_cache/LJ001-0030_phoneme.npy | Bin 0 -> 504 bytes .../ljspeech/phoneme_cache/LJ001-0031_phoneme.npy | Bin 0 -> 524 bytes .../ljspeech/phoneme_cache/LJ001-0032_phoneme.npy | Bin 0 -> 536 bytes 32 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0001_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0002_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0003_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0004_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0005_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0006_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0007_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0008_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0009_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0010_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0011_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0012_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0013_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0014_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0015_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0016_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0017_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0018_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0019_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0020_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0021_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0022_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0023_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0024_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0025_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0026_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0027_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0028_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0029_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0030_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0031_phoneme.npy create mode 100644 tests/data/ljspeech/phoneme_cache/LJ001-0032_phoneme.npy diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0001_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0001_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..fc024a6d0f4dee03a12a57645b754a8e3ce8bd28 GIT binary patch literal 700 zcmbV}KTE?v7>7?NMS`sg3W7!AbQLo+Z6_(Xb!&0SP#h%Kh=Q1ss7vYKXYdRAd*EKU z!Ks0Vd+&Ge|NJhC#lz~reY>q+SM77-?@~W|oBAyEFLm?TlpjxZv#!wR<$GI6Z(qwz zrTZ|SPO_UU-KBef$*FU0A@0SoI1l!<&aL>S4eImW31^|x_WcLkQad#O literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0002_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0002_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..9ac355d9f5d92ee7633adb4a30c9d83bd5db8336 GIT binary patch literal 244 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC%^qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I$7mO7d`3bhL411@_81_pj0ZUkZ$Am#$%d>{siD*`b{oCk;vfS3=61%X%w oh<$+=Bqj>PAhqH^%nQUI{UALcwHiPSGK&p}4S^VBPc0Aw02x6c0RR91 literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0003_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0003_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..ea3944c1ac5d41d67a213414d941a9294db2e363 GIT binary patch literal 704 zcmbV}K}!Nr6osE8CSsb96bLb`p3y9_280OOwNrHALR!R-BMH*POuCU4{s#YW=YzR0 z3s((%ym#Nb_uO;e9LBTJWL|MwxATi6|H%AP;Gb8K4+FnUv!5*fdQY=O!ue~w&J&&I zpYeC1xIc)Z@HPy7gMWXGx^wPMq+%rYrT9$yYjG~-f;-~-V&By|6DQ(Dv_(Up)f42C zLv6E&$!m#@px=W)15N6!4qx2A3G&gp5@^+mxT+P8eL+upTnKW}y%9r!&Z(gHrLcPQ zOg=dqfu7BP2kwr&5q;=^XJ%xEfk20Pn*)#NqQN`jlN@x*Z&{v=IhcDY&@?U1@L#T@ rKR!5n6zDU9y(hC$zZ3MtkNq!VUz{_K{To&^=uJ*X;Mv~UW@r5aS&2K? literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0004_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0004_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..ff7f0e3ea406fa39bee72aaae7afd9ac6a86b2f0 GIT binary patch literal 440 zcmbV{ISax-5QSF|i{OFa1%hn15mI=8B7&V=#3n_skYH96#K=Z%f`z}qKkR#uh14#5 z%*>nFIlkM#--RXd5}As1dNgV&)#@@+o>ZsAJWO;uBqr9>H_ zE&qQ`QwZUM5vYI;NI(m;!3PWg^YqK023)|pDzJK($NZb+dJlFD*efV#Q~3OE`?H2V zZ5=p(dG17w^X)!%=SiM_!g3~K+A-(@_S=IlDBi$6yxHE$I@^=88h|zI%lXXnZhIeZ F`vIaPEnxrv literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0005_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0005_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..d7c6b2e396f5ae723dc54f7e9e9a32e5c8a7ea53 GIT binary patch literal 652 zcmbV}L2JT55QVoOwurU}tsYt=J*`3xrtPT+-g;B&!Gm}xp{6K^kwmcXAfx8EfBXXal6KiR$bFz{b#cFN+zN1AOC>TmHrPgKvh z@iEc&FnW%{Q5Y0K?O$u)oSTZDGTtcm#giakigB5vSqsj|uSHL!f*QI;89R!mq1WPH zopY;4%I`(h1HR~IJ@D8pYu4MmRJB*qpe5dkmcSpoLoYNB!uGEFKUD7x)sJ1-1Aol0HzMx}&hWx}pdo)0 Gh4=#_jyK2v literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0006_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0006_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..4e7fa19a56bd0d2f9475f882e618a7917211cab3 GIT binary patch literal 412 zcmbV`JqyAx7=$B&NI^wV6cmY5Ds=D*1QFcaDmru!2Prn9AVykoDGvSy|FCzEfU|{% z_j1j9m(My1H@kv(id-c+JEm$b)#5T$fmA1xKGOJhFlnOcui|T_;b-Ugu5tEfqhJuo zm;C?px^c>+%V5vU{^K2XGnbtez}-D)SzX2~0`|EK_BfBbJ}`i}Az&8wJizyS0WyXwR{#J2 literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0007_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0007_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..b6db446041dfeaccd3455f447bd201a46683c036 GIT binary patch literal 588 zcmbV|J!=9%6h%i2B1TODF= zCxJ&@v;~@+c+!JLUNDC~{zpL#9eTO<#jP+s=BTj;n)Vj@Z2m=eyxAFzqPWbxDK4FB z=$SUX>}Px7K@K0w{eQ{7uL)FL)-lCb+?_!<4!$d5tDCYSzbwAnO8NG) zSXH{mvuT>#C1D@_`wND~m`ps13vubCJ@UHYgy7R=|Q=0;q*7@vMttk`>FCu(2C*u}rK(V;JL sZ}?yj^61h_j{W?6`n^~Dc@KXzuJncvblHVDoM(c$c=L1lN9grp4_m)7X#fBK literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0010_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0010_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..c53050da7a14296b04df8dfdfee094469ffec318 GIT binary patch literal 576 zcmbV}u}i~16vj_lL_)0!7Kb`yyNDS~a4Lmv-6}eC2o5DQCkkTZqAtOqe}n&UzaQkF zNT&vVyt{Yr`|jP#-@|h8SR9!j)7r{4Yi}p9oxa`MG`26b|MKO>s`iy5KQG@Kr+o8T zemd=wEX&iIH2#eb{-QHuOivV|6&rCa#@+f>u@?1ov@{2U==a9}DV;|9iq44LzV)icC4`f7jUeW^=o5 z%a{EBvsOZg9lU{YJBjyLThN7Jk}F`HKnQhc!0%icaSf`#x&y59Yz5fQUf$OM*3x&E wupa|w83B8X&;;Jcx$?lg3tnoULu>*2Sm&Hs;Q#P#=F&0i17JVrEo7jfus=pXidzysoF z;NiYG-{<^)&Zl3Cf;+gQUnTi6^W(rzHX|Pf{wK|Tvv~KNW~&7IN4(7w+4FV0PxRl3 z`cc>ogHv$#S9y2NwM0`4#9Z`6O;p66Xp7g2uIp`xg?PH?RPR&}`z(rry7;l7Ex#j8 zd{sOO>d@PA%F;(cUQb-tCI=hy(#LAQN#h#|;;(DayDo?&hFvTlAGy{KJ2#k#Ti@+0 i%*@VKpLr~YUM0bPFM^q<$vq#0-IqA-Y&|1s?sNvS)G|5% literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0013_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0013_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..f0e7c1a1e5112efa7257b606463fb1c519a63560 GIT binary patch literal 288 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC%^qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I#y20EHL3bhL411>oR1_pH?<_F?NAhw6HLE;8bF%clm4#XfiKB$-mkj@8U zK_E5+VqYMJsR;$LL3%-EN&_*-TnQis=?9q$;zt9qIS}&#F&7ZS)PTfc`f7n}04O&k A9{>OV literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0014_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0014_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..f0ed86b0bbd391456c09ba50bdd4f78c2f9474f5 GIT binary patch literal 736 zcmbV~J4?hs6op3+mDTkDzF%y)9qwo3^Tn-;^(1t$6oT zzSUaC=|z^DB+*y&?{9SEoZAvB;#*vcym|{7jHTed72bHWtKJg~dzFKLoyd!9HFyK|R)CP5j^k?KOcup3n-< z5HAGz)L>-adi?8uS$peZU(hReG7@Nn-dUdtyAO1!p9}Qt?k$I&>BoAePx8^DK6O~z zo9t*F4{^NG2m9fh;63bxGvwkEeRAm+PsF%K)3yJAy42vEd3!XdYd%?%I~92B#V?}* BK3M<& literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0015_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0015_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..381b981ab92d6de847ba4ad5d488ef7c7ae56129 GIT binary patch literal 716 zcmbV~OH0F05Jn@2kBUmc$3l_pu3{Ee3W5mk+*NeZMR6g)Mij(Is%}ab{s#ZB=YRu2 z1i`=|cjnIA`+bvqUJeIWqq*!oo2GF!xogsCp3WZosmRlP-MlvC^KIRXEAtoS)1)Fl zc_?2h;{9I#q&O(@S^n>*vzKMr5q!WB96}8PSc4sK?*_KN*InN3pTjzA!W%4rwX1Lp z*U*DFW5{|0Q8&-J1Kww!@d|8#Is4sLml{2H3fAs|cgI=!FsByxjn$w|V|BP*gwSno z3HtQ>@M67lFmBJMOC0^tYwcgYyf=J0i@3|Tp2M4G{q&W{nO_F|I0JQ<)2qF%+d9<~ iI<2+G`}Je}2G|qdwB2j%Z{IHF>8$JzE&daFn!y)6syhY% literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0016_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0016_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..5667d54bc702ba012028e21a9760254af2363e14 GIT binary patch literal 416 zcmbV|O$z~06o#)7lFx-wq}g3E3#62ko!!XBLKd1BQ8JprMizd9KkR)L_YatQ>YVqS z^PY3(yIcpWO~yRU%O0Y1pV+BqXXml?J$sCkdlFuEadL>z7vUw1q^GCw7AfyeMt;xt z-roOTv15!Gh>xg8@={^g_V+ xm5*$5K|Y(!o0-Nga+7Z-@Sx+je9Wu~W}zR;8~S-Svg7Byso7~6{>r$Q?*|xWHW~l` literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0018_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0018_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..b060181d526230633a49f23583a3e7381a852c8e GIT binary patch literal 584 zcmbV|K}!Nr6op?D8B$qLC@3+zr!b2~g&+cV?nZRsLR!R-6A9ADjJ6RM{)Yar=YzQ* zE*toG@1A$>opbNsXc~`aXYSW+{X8u`bN?Fnw?*HFf&a?#b)NiuWcfTrA12Epm0o-& zt5k72iuz$E40gf4zv`27?pf@`mFS5~Jcvhuej#S!TA-sfgyj<7i&zXqO;EcLmx6cG zt>2BbmY^pZaaT-4B(?(mTu{3!lS$ypOv*Kz>;e~yXW8Rp_c69*P C$u-mf literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0019_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0019_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..b0e818cc5de0af8165fb4b8500f8d285cf3d0995 GIT binary patch literal 524 zcmbV|Jxjw-6ozjRtb|%CR4gcRw`*wzEnQj!H@Aom9ZH81nkyDcBZ+hg9r_#m!@dvX zLU1(jkev6t=PN(2^W<&OGJ8|G%$MuJ&8&OcOk8Z;M_zmv>28%5nMZ#~x22a}ex_eu z@%?lf4`X|<|9_%eW6Xs(6-O};&*DnV1sXZDYcUq*;z3*re0aK|CvuSpw2r_>4li|) zz(ZY#@&4VTUu*ArGn@FV*62(9qhJ;@n!QGfx#_OX(VzMgL9cM@Qs3Ov^wEbq7J~la kj<&wT9%$4!!JH7YNAPnGH&eq6VMp@J;6I`><3?2C7Z}GblmGw# literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0020_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0020_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..472ac10be0384705feccf50c1f6ce0a3623c6533 GIT binary patch literal 364 zcmbWxI}5@v6b0Z2qEdVfii09?ib4mUgGF$2Q(PQ6Nw5(Gv685ZbnrL$hdl>ca5nIf z&XR}*x!<=}Emx_!#SQSXMI8T)?)iJhrtFQamMuz!HUz|biPWooB@6YD`z?X^q ze|TLX#1P(4fjWdR0%m0>!V((5EDs*Uz`O^{xvr(_66Ox@elyLodJFq8@GNT&D8TO< h?2G4_O@M!~KmNQF%U#>*Urd=3s*8Y69wVKOu7*l{RaPV?*ntd zozTF;z31F>&-+4Pw*rkj@g(TUiY7JmI2PQa|3;AC_vsba#I>Lo9`G3p)9!13E=-TU=-?Ago0U5H zlQZ3YPo|Hbc2#?5ZhH^bu`r6EYnw&OL literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0022_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0022_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..25273f681c6a8eaa3a0b268b55866afffc5015bb GIT binary patch literal 528 zcmbV|O$z~07=^FMS3Zj<(pX(Gi&0imc6P*K7P8P}MwEpVw z_q<>CyKuvr06?aOh7U%&7d_V=%KnRwAS{{smX!Ouu2E2cOl8K8r>Fa_s`F*gIK7ad>5IAeWZ4B@8m3XcsR=s4Ja^B literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0023_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0023_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..94b639df747a1c3a37dcc599bacddd1de979e74d GIT binary patch literal 640 zcmbV}y-EW?7)3`65)yw9)Ch{K)hKRZLJASV&Mp#*6k;KZSy2!-?21iD;WPNc-UD-C zYZngl&7JRmfA43L>El}X4$C+%o0mFXXYuBJ9P=!`R`pk1d_Gt8yhOh%KAKW`^HwZM z-Oq-jtNc9AR@uM5-f;+FM{I~6aVVa|QM%vimp%3fS_FTkyXC2wz6SW=VdGW(O6}($-?}2}T&WS&}V1LZs2+!nQ N;WHD=MSu9y_aE6KHVyy) literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0024_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0024_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..1178fc90b887ce9b28eb71e8b891fb9216c52988 GIT binary patch literal 600 zcmbV}!AiqG6h)_0RHC*DDuN=}O-r*dw2LA_cimNV(M50}!KAcM8cEcRbkT3{hrI{# zAnOJWGxO%YJ9j33&-2+!(sDoU*JpY4QToTwKdnYS3jLxezshv;UX)o*eVVSTT=i<1 ze&$;D;&>eOqi`GU{RIckxi@hocH&+<)bnGFLs5uJVH$ecVj`HMKNIxdiBo}gF0KXr zrU^7hcPNeo9At56op?AY!z)Q=wy*>&q4+*P9nIuDJ~A7g9MY(LNSt57jfus@DF=F@Ir7j z@bTU|@1Aq-%XzVyFQW%{a7Uk{#U}UTz)yZgJ`DUOKW`tE={;buQ{66GMTgCf>x0sEVUN zC)0Yiv~LPLc28DCM8dq~`*J3E_&6T3?hk in;TzOSSJ64S@D_1+d13+p`N|f;TZ^O=xKHMPT~S~!7>d1 literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0026_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0026_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..da78e6afc3f9b252f26604be60c60f929e0735e7 GIT binary patch literal 444 zcmbV{I}5@v7)2wBV(|%zn`BGH4hlk11UI*ei$igcVj~J-r4g6n;BW8`doQF=91R@4 ze2*k2pUp1Vh9&V7nToYJrfMP8@-kDNRHr0;q|xn=q_Jjw9bJt^-JGMlM(+A^Z{*3B z{C_!PAw(Mt!34ZP9dtklnt-)pzmL2OR-g;m--ChObC$iH%`1q!t6}pNVhwPHXW4f^ zVsp;XgUy?xYOKo0}>Ztnd1=M23s5aBI- literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0027_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0027_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..313f2b411321fa496bc143904200a14987bcec26 GIT binary patch literal 664 zcmbV}J4?h+5Jn@4%If+6U!X{=vf>u5E(jvn*{x`?#bP1BjVOqbL~VkFzrjE3IgkNs z1BW|z?l&{vz4^Hwk8UP2$!F4J)1rQ;vhy^%cphf`G4Niys!+!de{`>1(CP}gZ>u>^NIEF5C;2ZW}w~a@vA8-aeD8bx3gl3=E9@g4t z4JI%EbIY&*F*|h5zXf&nY9QvX5*6SsFdoMLB?W&jXutR(Y;dzO8 z3%rlHnB}T>`-=Szz?pyVsXym?H|Oe6%_3OiYEEy?Q1AIsySKM{HF~p`nyB$ikM>d< fzSYEkh?();o^>JKV~#l9&3^jSZ|L>t`!w(i8!b45 literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0028_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0028_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..d6cf441c01998270b6bb4ae8763e9607ba2dc434 GIT binary patch literal 392 zcmbV`I}5@<5QI+@i=c?0HW9h*6jJyif(Ul$B3f8T3JK;!L5y6)CZzB;_=lZ=99Vns zu{XQ3kI!ZwZVwgl61hrrb~0)y)#^G|o>b@5JWYH*rY6zM2k|Y_$Y+=Mp`nMq?~OcJ z$p4>H6GB9w3*Lab3f84=BZj4JA<{PiE})-2=K1dc*6Ht;n)7BL1q+Y^yT^LQ$Tfdz q+{v2V%el6Xz8>H_&Ypq>uyywFo;u*32n+ykwm0(T9dLjP3h)IJ-6}T# literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0029_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0029_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..0c0d379da1771fc0466d915ff2dea5e4130f9487 GIT binary patch literal 400 zcmbV{PYVH26vdyC82<|?rP;laSs)=PJ3C@wFq)O%mR)USK) z>D_bhd^h{`HpqyVNM)pwla_N=F0NzgyYd|Crw;GOSVxNfD!e5MesT#P3cWX(`90r# zyZ?VqRtRwbP4EF-Py>PK`>0hwe_*@1KRdBC0ajv{mma2Mxu;0FV9D(L_K literal 0 HcmV?d00001 diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0030_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0030_phoneme.npy new file mode 100644 index 0000000000000000000000000000000000000000..a8bcc0ae264bd3bbff8f6c9ebcf31659abe72bc2 GIT binary patch literal 504 zcmbV|J4*vm5Js;cA~8M+A4NpAyAikWk;Wj{*+sOl2o|!K6$No+S8PHGe}jM6bBck~ zUN|uKo0)Is-rw7Y$=!4%eW!Vz)!lQO$3?z+8|AXdFHQT^Rv%AIJFC@is`su&?_R6V zn)~o_R34YbqWJ&m<&@G79KbnDpa*K}umStffSUJfux)&xjr0%HO}F{O|bAc_=lZ=ETlH@ zk=ebS-M#y6_wLRssFzAj5GJSCEVWr(XU5Uy9L3MrzaOJG2$|pbwo5JguMk@yW71TEI;Mc5(+K@h<%T||o%sbn!L3gXJH+5`)KgMZj_AOl;w zaM<_mT;|Ta-`nZrZnlzrWs5Yg+Q%kc<>~t6A{BXhs+*6de0!*yd8IxsU)u`3eJU<(eQ2YU70z$t8ly52SD_Pk%# zHuw)guf9WA1$*qn0@RN{&H5D7ygOT*WB%?e(bEji=FGsky}P%wMBV Date: Wed, 19 May 2021 12:35:10 +0200 Subject: [PATCH 87/87] bump version number --- TTS/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TTS/_version.py b/TTS/_version.py index f0584d70..311f216e 100644 --- a/TTS/_version.py +++ b/TTS/_version.py @@ -1 +1 @@ -__version__ = "0.0.13.2" +__version__ = "0.0.14"