diff --git a/.compute b/.compute index cda787d2..9786a689 100644 --- a/.compute +++ b/.compute @@ -1,7 +1,6 @@ #!/bin/bash yes | apt-get install sox yes | apt-get install ffmpeg -yes | apt-get install espeak yes | apt-get install tmux yes | apt-get install zsh sh -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9fef1b84..74d5e85b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,6 +30,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + architecture: x64 - name: check OS run: cat /etc/os-release - name: Install dependencies diff --git a/.gitignore b/.gitignore index 1829dd93..1d3ab8c2 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,3 @@ notebooks/data/* TTS/tts/layers/glow_tts/monotonic_align/core.c .vscode-upload.json temp_build/* -recipes/* diff --git a/.pylintrc b/.pylintrc index 910a7a55..0bc0be4b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -563,7 +563,7 @@ max-branches=12 max-locals=15 # Maximum number of parents for a class (see R0901). -max-parents=7 +max-parents=15 # Maximum number of public methods for a class (see R0904). max-public-methods=20 diff --git a/Makefile b/Makefile index 650958ae..4dc2d588 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,6 @@ help: target_dirs := tests TTS notebooks system-deps: ## install linux system deps - sudo apt-get install -y espeak-ng sudo apt-get install -y libsndfile1-dev dev-deps: ## install development deps @@ -18,9 +17,12 @@ deps: ## install 🐸 requirements. pip install -r requirements.txt test: ## run tests. - nosetests -x --with-cov -cov --cover-erase --cover-package TTS tests --nologcapture + nosetests -x --with-cov -cov --cover-erase --cover-package TTS tests --nologcapture --with-id ./run_bash_tests.sh +test_failed: ## only run tests failed the last time. + nosetests -x --with-cov -cov --cover-erase --cover-package TTS tests --nologcapture --failed + style: ## update code style. black ${target_dirs} isort ${target_dirs} diff --git a/TTS/.models.json b/TTS/.models.json index 5fcfa86b..b926f120 100644 --- a/TTS/.models.json +++ b/TTS/.models.json @@ -6,7 +6,8 @@ "description": "EK1 en-rp tacotron2 by NMStoker", "github_rls_url": "https://github.com/coqui-ai/TTS/releases/download/v0.0.10/tts_models--en--ek1--tacotron2.zip", "default_vocoder": "vocoder_models/en/ek1/wavegrad", - "commit": "c802255" + "commit": "c802255", + "needs_phonemizer": true } }, "ljspeech":{ @@ -17,7 +18,8 @@ "commit": "bae2ad0f", "author": "Eren Gölge @erogol", "license": "", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": false }, "glow-tts":{ "description": "", @@ -27,7 +29,8 @@ "commit": "", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true }, "tacotron2-DCA": { "description": "", @@ -36,7 +39,8 @@ "commit": "", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true }, "speedy-speech-wn":{ "description": "Speedy Speech model with wavenet decoder.", @@ -45,7 +49,8 @@ "commit": "77b6145", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true } }, "vctk":{ @@ -56,7 +61,9 @@ "commit": "b531fa69", "author": "Edresson Casanova", "license": "", - "contact":"" + "contact":"", + "needs_phonemizer": true + } }, @@ -68,7 +75,8 @@ "commit": "bae2ad0f", "author": "Eren Gölge @erogol", "license": "", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true } } }, @@ -80,7 +88,8 @@ "commit": "", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true } } }, @@ -92,7 +101,8 @@ "commit": "", "author": "Eren Gölge @erogol", "license": "MPL", - "contact":"egolge@coqui.com" + "contact":"egolge@coqui.com", + "needs_phonemizer": true } } }, @@ -112,7 +122,8 @@ "author": "@r-dh", "default_vocoder": "vocoder_models/nl/mai/parallel-wavegan", "stats_file": null, - "commit": "540d811" + "commit": "540d811", + "needs_phonemizer": true } } }, @@ -123,7 +134,8 @@ "author": "@erogol", "default_vocoder": "vocoder_models/universal/libri-tts/fullband-melgan", "license":"", - "contact": "egolge@coqui.com" + "contact": "egolge@coqui.com", + "needs_phonemizer": true } } }, @@ -133,7 +145,8 @@ "github_rls_url": "https://github.com/coqui-ai/TTS/releases/download/v0.0.11/tts_models--de--thorsten--tacotron2-DCA.zip", "default_vocoder": "vocoder_models/de/thorsten/wavegrad", "author": "@thorstenMueller", - "commit": "unknown" + "commit": "unknown", + "needs_phonemizer": true } } } diff --git a/TTS/__init__.py b/TTS/__init__.py index e69de29b..8dee4bf8 100644 --- a/TTS/__init__.py +++ b/TTS/__init__.py @@ -0,0 +1 @@ +from ._version import __version__ diff --git a/TTS/_version.py b/TTS/_version.py new file mode 100644 index 00000000..311f216e --- /dev/null +++ b/TTS/_version.py @@ -0,0 +1 @@ +__version__ = "0.0.14" diff --git a/TTS/bin/compute_statistics.py b/TTS/bin/compute_statistics.py index 9e2b7415..f3234c2a 100755 --- a/TTS/bin/compute_statistics.py +++ b/TTS/bin/compute_statistics.py @@ -8,31 +8,38 @@ import os import numpy as np from tqdm import tqdm +# from TTS.utils.io import load_config +from TTS.config import load_config from TTS.tts.datasets.preprocess import load_meta_data from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config def main(): """Run preprocessing process.""" parser = argparse.ArgumentParser(description="Compute mean and variance of spectrogtram features.") + parser.add_argument("config_path", type=str, help="TTS config file path to define audio processin parameters.") + parser.add_argument("out_path", type=str, help="save path (directory and filename).") parser.add_argument( - "--config_path", type=str, required=True, help="TTS config file path to define audio processin parameters." + "--data_path", + type=str, + required=False, + help="folder including the target set of wavs overriding dataset config.", ) - parser.add_argument("--out_path", type=str, required=True, help="save path (directory and filename).") - args = parser.parse_args() + args, overrides = parser.parse_known_args() + + CONFIG = load_config(args.config_path) + CONFIG.parse_known_args(overrides, relaxed_parser=True) # load config - CONFIG = load_config(args.config_path) - CONFIG.audio["signal_norm"] = False # do not apply earlier normalization - CONFIG.audio["stats_path"] = None # discard pre-defined stats + CONFIG.audio.signal_norm = False # do not apply earlier normalization + CONFIG.audio.stats_path = None # discard pre-defined stats # load audio processor - ap = AudioProcessor(**CONFIG.audio) + ap = AudioProcessor(**CONFIG.audio.to_dict()) # load the meta data of target dataset - if "data_path" in CONFIG.keys(): - dataset_items = glob.glob(os.path.join(CONFIG.data_path, "**", "*.wav"), recursive=True) + if args.data_path: + dataset_items = glob.glob(os.path.join(args.data_path, "**", "*.wav"), recursive=True) else: dataset_items = load_meta_data(CONFIG.datasets)[0] # take only train data print(f" > There are {len(dataset_items)} files.") @@ -73,14 +80,14 @@ def main(): print(f" > Avg lienar spec scale: {linear_scale.mean()}") # set default config values for mean-var scaling - CONFIG.audio["stats_path"] = output_file_path - CONFIG.audio["signal_norm"] = True + CONFIG.audio.stats_path = output_file_path + CONFIG.audio.signal_norm = True # remove redundant values - del CONFIG.audio["max_norm"] - del CONFIG.audio["min_level_db"] - del CONFIG.audio["symmetric_norm"] - del CONFIG.audio["clip_norm"] - stats["audio_config"] = CONFIG.audio + del CONFIG.audio.max_norm + del CONFIG.audio.min_level_db + del CONFIG.audio.symmetric_norm + del CONFIG.audio.clip_norm + stats["audio_config"] = CONFIG.audio.to_dict() np.save(output_file_path, stats, allow_pickle=True) print(f" > stats saved to {output_file_path}") diff --git a/TTS/bin/extract_tts_spectrograms.py b/TTS/bin/extract_tts_spectrograms.py new file mode 100755 index 00000000..ace7464a --- /dev/null +++ b/TTS/bin/extract_tts_spectrograms.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Extract Mel spectrograms with teacher forcing.""" + +import argparse +import os + +import numpy as np +import torch +from torch.utils.data import DataLoader +from tqdm import tqdm + +from TTS.config import load_config +from TTS.tts.datasets.preprocess import load_meta_data +from TTS.tts.datasets.TTSDataset import MyDataset +from TTS.tts.utils.generic_utils import setup_model +from TTS.tts.utils.speakers import parse_speakers +from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols +from TTS.utils.audio import AudioProcessor +from TTS.utils.generic_utils import count_parameters + +use_cuda = torch.cuda.is_available() + + +def setup_loader(ap, r, verbose=False): + dataset = MyDataset( + r, + c.text_cleaner, + compute_linear_spec=False, + meta_data=meta_data, + ap=ap, + tp=c.characters if "characters" in c.keys() else None, + add_blank=c["add_blank"] if "add_blank" in c.keys() else False, + batch_group_size=0, + min_seq_len=c.min_seq_len, + max_seq_len=c.max_seq_len, + phoneme_cache_path=c.phoneme_cache_path, + use_phonemes=c.use_phonemes, + phoneme_language=c.phoneme_language, + enable_eos_bos=c.enable_eos_bos_chars, + use_noise_augment=False, + verbose=verbose, + speaker_mapping=speaker_mapping if c.use_speaker_embedding and c.use_external_speaker_embedding_file else None, + ) + + if c.use_phonemes and c.compute_input_seq_cache: + # precompute phonemes to have a better estimate of sequence lengths. + dataset.compute_input_seq(c.num_loader_workers) + dataset.sort_items() + + loader = DataLoader( + dataset, + batch_size=c.batch_size, + shuffle=False, + collate_fn=dataset.collate_fn, + drop_last=False, + sampler=None, + num_workers=c.num_loader_workers, + pin_memory=False, + ) + return loader + + +def set_filename(wav_path, out_path): + wav_file = os.path.basename(wav_path) + file_name = wav_file.split(".")[0] + os.makedirs(os.path.join(out_path, "quant"), exist_ok=True) + os.makedirs(os.path.join(out_path, "mel"), exist_ok=True) + os.makedirs(os.path.join(out_path, "wav_gl"), exist_ok=True) + os.makedirs(os.path.join(out_path, "wav"), exist_ok=True) + wavq_path = os.path.join(out_path, "quant", file_name) + mel_path = os.path.join(out_path, "mel", file_name) + wav_gl_path = os.path.join(out_path, "wav_gl", file_name + ".wav") + wav_path = os.path.join(out_path, "wav", file_name + ".wav") + return file_name, wavq_path, mel_path, wav_gl_path, wav_path + + +def format_data(data): + # setup input data + text_input = data[0] + text_lengths = data[1] + speaker_names = data[2] + mel_input = data[4] + mel_lengths = data[5] + item_idx = data[7] + attn_mask = data[9] + avg_text_length = torch.mean(text_lengths.float()) + avg_spec_length = torch.mean(mel_lengths.float()) + + if c.use_speaker_embedding: + if c.use_external_speaker_embedding_file: + speaker_embeddings = data[8] + speaker_ids = None + else: + speaker_ids = [speaker_mapping[speaker_name] for speaker_name in speaker_names] + speaker_ids = torch.LongTensor(speaker_ids) + speaker_embeddings = None + else: + speaker_embeddings = None + speaker_ids = None + + # dispatch data to GPU + if use_cuda: + text_input = text_input.cuda(non_blocking=True) + text_lengths = text_lengths.cuda(non_blocking=True) + mel_input = mel_input.cuda(non_blocking=True) + mel_lengths = mel_lengths.cuda(non_blocking=True) + if speaker_ids is not None: + speaker_ids = speaker_ids.cuda(non_blocking=True) + if speaker_embeddings is not None: + speaker_embeddings = speaker_embeddings.cuda(non_blocking=True) + + if attn_mask is not None: + attn_mask = attn_mask.cuda(non_blocking=True) + return ( + text_input, + text_lengths, + mel_input, + mel_lengths, + speaker_ids, + speaker_embeddings, + avg_text_length, + avg_spec_length, + attn_mask, + item_idx, + ) + + +@torch.no_grad() +def inference( + model_name, + model, + ap, + text_input, + text_lengths, + mel_input, + mel_lengths, + attn_mask=None, + speaker_ids=None, + speaker_embeddings=None, +): + if model_name == "glow_tts": + mel_input = mel_input.permute(0, 2, 1) # B x D x T + speaker_c = None + if speaker_ids is not None: + speaker_c = speaker_ids + elif speaker_embeddings is not None: + speaker_c = speaker_embeddings + + model_output, *_ = model.inference_with_MAS( + text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c + ) + model_output = model_output.transpose(1, 2).detach().cpu().numpy() + + elif "tacotron" in model_name: + _, postnet_outputs, *_ = model( + text_input, + text_lengths, + mel_input, + mel_lengths, + speaker_ids=speaker_ids, + speaker_embeddings=speaker_embeddings, + ) + # normalize tacotron output + if model_name == "tacotron": + mel_specs = [] + postnet_outputs = postnet_outputs.data.cpu().numpy() + for b in range(postnet_outputs.shape[0]): + postnet_output = postnet_outputs[b] + mel_specs.append(torch.FloatTensor(ap.out_linear_to_mel(postnet_output.T).T)) + model_output = torch.stack(mel_specs).cpu().numpy() + + elif model_name == "tacotron2": + model_output = postnet_outputs.detach().cpu().numpy() + return model_output + + +def extract_spectrograms( + data_loader, model, ap, output_path, quantized_wav=False, save_audio=False, debug=False, metada_name="metada.txt" +): + model.eval() + export_metadata = [] + for _, data in tqdm(enumerate(data_loader), total=len(data_loader)): + + # format data + ( + text_input, + text_lengths, + mel_input, + mel_lengths, + speaker_ids, + speaker_embeddings, + _, + _, + attn_mask, + item_idx, + ) = format_data(data) + + model_output = inference( + c.model.lower(), + model, + ap, + text_input, + text_lengths, + mel_input, + mel_lengths, + attn_mask, + speaker_ids, + speaker_embeddings, + ) + + for idx in range(text_input.shape[0]): + wav_file_path = item_idx[idx] + wav = ap.load_wav(wav_file_path) + _, wavq_path, mel_path, wav_gl_path, wav_path = set_filename(wav_file_path, output_path) + + # quantize and save wav + if quantized_wav: + wavq = ap.quantize(wav) + np.save(wavq_path, wavq) + + # save TTS mel + mel = model_output[idx] + mel_length = mel_lengths[idx] + mel = mel[:mel_length, :].T + np.save(mel_path, mel) + + export_metadata.append([wav_file_path, mel_path]) + if save_audio: + ap.save_wav(wav, wav_path) + + if debug: + print("Audio for debug saved at:", wav_gl_path) + wav = ap.inv_melspectrogram(mel) + ap.save_wav(wav, wav_gl_path) + + with open(os.path.join(output_path, metada_name), "w") as f: + for data in export_metadata: + f.write(f"{data[0]}|{data[1]+'.npy'}\n") + + +def main(args): # pylint: disable=redefined-outer-name + # pylint: disable=global-variable-undefined + global meta_data, symbols, phonemes, model_characters, speaker_mapping + + # Audio processor + ap = AudioProcessor(**c.audio) + if "characters" in c.keys() and c["characters"]: + symbols, phonemes = make_symbols(**c.characters) + + # set model characters + model_characters = phonemes if c.use_phonemes else symbols + num_chars = len(model_characters) + + # load data instances + meta_data_train, meta_data_eval = load_meta_data(c.datasets) + + # use eval and training partitions + meta_data = meta_data_train + meta_data_eval + + # parse speakers + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, None) + + # setup model + model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim) + + # restore model + checkpoint = torch.load(args.checkpoint_path, map_location="cpu") + model.load_state_dict(checkpoint["model"]) + + if use_cuda: + model.cuda() + + num_params = count_parameters(model) + print("\n > Model has {} parameters".format(num_params), flush=True) + # set r + r = 1 if c.model.lower() == "glow_tts" else model.decoder.r + own_loader = setup_loader(ap, r, verbose=True) + + extract_spectrograms( + own_loader, + model, + ap, + args.output_path, + quantized_wav=args.quantized, + save_audio=args.save_audio, + debug=args.debug, + metada_name="metada.txt", + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--config_path", type=str, help="Path to config file for training.", required=True) + parser.add_argument("--checkpoint_path", type=str, help="Model file to be restored.", required=True) + parser.add_argument("--output_path", type=str, help="Path to save mel specs", required=True) + parser.add_argument("--debug", default=False, action="store_true", help="Save audio files for debug") + parser.add_argument("--save_audio", default=False, action="store_true", help="Save audio files") + parser.add_argument("--quantized", action="store_true", help="Save quantized audio files") + args = parser.parse_args() + + c = load_config(args.config_path) + main(args) diff --git a/TTS/bin/train_align_tts.py b/TTS/bin/train_align_tts.py index 6f268ed3..7e3921b0 100644 --- a/TTS/bin/train_align_tts.py +++ b/TTS/bin/train_align_tts.py @@ -23,169 +23,304 @@ from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.synthesis import synthesis from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed, reduce_tensor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict from TTS.utils.radam import RAdam from TTS.utils.training import NoamLR, setup_torch_training_env -if __name__ == "__main__": - use_cuda, num_gpus = setup_torch_training_env(True, False) - # torch.autograd.set_detect_anomaly(True) +use_cuda, num_gpus = setup_torch_training_env(True, False) +# torch.autograd.set_detect_anomaly(True) - def setup_loader(ap, r, is_val=False, verbose=False): - if is_val and not c.run_eval: - loader = None + +def setup_loader(ap, r, is_val=False, verbose=False): + if is_val and not config.run_eval: + loader = None + else: + dataset = MyDataset( + r, + config.text_cleaner, + compute_linear_spec=False, + meta_data=meta_data_eval if is_val else meta_data_train, + ap=ap, + tp=config.characters, + add_blank=config["add_blank"], + batch_group_size=0 if is_val else config.batch_group_size * config.batch_size, + min_seq_len=config.min_seq_len, + max_seq_len=config.max_seq_len, + phoneme_cache_path=config.phoneme_cache_path, + use_phonemes=config.use_phonemes, + phoneme_language=config.phoneme_language, + enable_eos_bos=config.enable_eos_bos_chars, + use_noise_augment=not is_val, + verbose=verbose, + speaker_mapping=speaker_mapping + if config.use_speaker_embedding and config.use_external_speaker_embedding_file + else None, + ) + + if config.use_phonemes and config.compute_input_seq_cache: + # precompute phonemes to have a better estimate of sequence lengths. + dataset.compute_input_seq(config.num_loader_workers) + dataset.sort_items() + + sampler = DistributedSampler(dataset) if num_gpus > 1 else None + loader = DataLoader( + dataset, + batch_size=config.eval_batch_size if is_val else config.batch_size, + shuffle=False, + collate_fn=dataset.collate_fn, + drop_last=False, + sampler=sampler, + num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers, + pin_memory=False, + ) + return loader + + +def format_data(data): + # setup input data + text_input = data[0] + text_lengths = data[1] + speaker_names = data[2] + mel_input = data[4].permute(0, 2, 1) # B x D x T + mel_lengths = data[5] + item_idx = data[7] + avg_text_length = torch.mean(text_lengths.float()) + avg_spec_length = torch.mean(mel_lengths.float()) + + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: + # return precomputed embedding vector + speaker_c = data[8] else: - dataset = MyDataset( - r, - c.text_cleaner, - compute_linear_spec=False, - meta_data=meta_data_eval if is_val else meta_data_train, - ap=ap, - tp=c.characters if "characters" in c.keys() else None, - add_blank=c["add_blank"] if "add_blank" in c.keys() else False, - batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, - min_seq_len=c.min_seq_len, - max_seq_len=c.max_seq_len, - phoneme_cache_path=c.phoneme_cache_path, - use_phonemes=c.use_phonemes, - phoneme_language=c.phoneme_language, - enable_eos_bos=c.enable_eos_bos_chars, - use_noise_augment=not is_val, - verbose=verbose, - speaker_mapping=speaker_mapping - if c.use_speaker_embedding and c.use_external_speaker_embedding_file - else None, + # return speaker_id to be used by an embedding layer + speaker_c = [speaker_mapping[speaker_name] for speaker_name in speaker_names] + speaker_c = torch.LongTensor(speaker_c) + else: + speaker_c = None + # dispatch data to GPU + if use_cuda: + text_input = text_input.cuda(non_blocking=True) + text_lengths = text_lengths.cuda(non_blocking=True) + mel_input = mel_input.cuda(non_blocking=True) + mel_lengths = mel_lengths.cuda(non_blocking=True) + if speaker_c is not None: + speaker_c = speaker_c.cuda(non_blocking=True) + return text_input, text_lengths, mel_input, mel_lengths, speaker_c, avg_text_length, avg_spec_length, item_idx + + +def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, training_phase): + + model.train() + epoch_time = 0 + keep_avg = KeepAverage() + if use_cuda: + batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus)) + else: + batch_n_iter = int(len(data_loader.dataset) / config.batch_size) + end_time = time.time() + c_logger.print_train_start() + scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None + for num_iter, data in enumerate(data_loader): + start_time = time.time() + + # format data + ( + text_input, + text_lengths, + mel_targets, + mel_lengths, + speaker_c, + avg_text_length, + avg_spec_length, + _, + ) = format_data(data) + + loader_time = time.time() - end_time + + global_step += 1 + optimizer.zero_grad() + + # forward pass model + with torch.cuda.amp.autocast(enabled=config.mixed_precision): + decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward( + text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase ) - if c.use_phonemes and c.compute_input_seq_cache: - # precompute phonemes to have a better estimate of sequence lengths. - dataset.compute_input_seq(c.num_loader_workers) - dataset.sort_items() - - sampler = DistributedSampler(dataset) if num_gpus > 1 else None - loader = DataLoader( - dataset, - batch_size=c.eval_batch_size if is_val else c.batch_size, - shuffle=False, - collate_fn=dataset.collate_fn, - drop_last=False, - sampler=sampler, - num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers, - pin_memory=False, + # compute loss + loss_dict = criterion( + logp, + decoder_output, + mel_targets, + mel_lengths, + dur_output, + dur_mas_output, + text_lengths, + global_step, + phase=training_phase, ) - return loader - def format_data(data): - # setup input data - text_input = data[0] - text_lengths = data[1] - speaker_names = data[2] - mel_input = data[4].permute(0, 2, 1) # B x D x T - mel_lengths = data[5] - item_idx = data[7] - avg_text_length = torch.mean(text_lengths.float()) - avg_spec_length = torch.mean(mel_lengths.float()) + # backward pass with loss scaling + if config.mixed_precision: + scaler.scale(loss_dict["loss"]).backward() + scaler.unscale_(optimizer) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) + scaler.step(optimizer) + scaler.update() + else: + loss_dict["loss"].backward() + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) + optimizer.step() - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: - # return precomputed embedding vector - speaker_c = data[8] + # setup lr + if config.noam_schedule: + scheduler.step() + + # current_lr + current_lr = optimizer.param_groups[0]["lr"] + + # compute alignment error (the lower the better ) + align_error = 1 - alignment_diagonal_score(alignments, binary=True) + loss_dict["align_error"] = align_error + + step_time = time.time() - start_time + epoch_time += step_time + + # aggregate losses from processes + if num_gpus > 1: + loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus) + loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, num_gpus) + loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, num_gpus) + loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) + + # detach loss values + loss_dict_new = dict() + for key, value in loss_dict.items(): + if isinstance(value, (int, float)): + loss_dict_new[key] = value else: - # return speaker_id to be used by an embedding layer - speaker_c = [speaker_mapping[speaker_name] for speaker_name in speaker_names] - speaker_c = torch.LongTensor(speaker_c) - else: - speaker_c = None - # dispatch data to GPU - if use_cuda: - text_input = text_input.cuda(non_blocking=True) - text_lengths = text_lengths.cuda(non_blocking=True) - mel_input = mel_input.cuda(non_blocking=True) - mel_lengths = mel_lengths.cuda(non_blocking=True) - if speaker_c is not None: - speaker_c = speaker_c.cuda(non_blocking=True) - return text_input, text_lengths, mel_input, mel_lengths, speaker_c, avg_text_length, avg_spec_length, item_idx + loss_dict_new[key] = value.item() + loss_dict = loss_dict_new - def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, training_phase): + # update avg stats + update_train_values = dict() + for key, value in loss_dict.items(): + update_train_values["avg_" + key] = value + update_train_values["avg_loader_time"] = loader_time + update_train_values["avg_step_time"] = step_time + keep_avg.update_values(update_train_values) - model.train() - epoch_time = 0 - keep_avg = KeepAverage() - if use_cuda: - batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus)) - else: - batch_n_iter = int(len(data_loader.dataset) / c.batch_size) + # print training progress + if global_step % config.print_step == 0: + log_dict = { + "avg_spec_length": [avg_spec_length, 1], # value, precision + "avg_text_length": [avg_text_length, 1], + "step_time": [step_time, 4], + "loader_time": [loader_time, 2], + "current_lr": current_lr, + } + c_logger.print_train_step(batch_n_iter, num_iter, global_step, log_dict, loss_dict, keep_avg.avg_values) + + if args.rank == 0: + # Plot Training Iter Stats + # reduce TB load + if global_step % config.tb_plot_step == 0: + iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time} + iter_stats.update(loss_dict) + tb_logger.tb_train_iter_stats(global_step, iter_stats) + + if global_step % config.save_step == 0: + if config.checkpoint: + # save model + save_checkpoint( + model, + optimizer, + global_step, + epoch, + 1, + OUT_PATH, + model_characters, + model_loss=loss_dict["loss"], + ) + + # wait all kernels to be completed + torch.cuda.synchronize() + + # Diagnostic visualizations + if decoder_output is not None: + idx = np.random.randint(mel_targets.shape[0]) + pred_spec = decoder_output[idx].detach().data.cpu().numpy().T + gt_spec = mel_targets[idx].data.cpu().numpy().T + align_img = alignments[idx].data.cpu() + + figures = { + "prediction": plot_spectrogram(pred_spec, ap), + "ground_truth": plot_spectrogram(gt_spec, ap), + "alignment": plot_alignment(align_img), + } + + tb_logger.tb_train_figures(global_step, figures) + + # Sample audio + train_audio = ap.inv_melspectrogram(pred_spec.T) + tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() - c_logger.print_train_start() - scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None + + # print epoch stats + c_logger.print_train_epoch_end(global_step, epoch, epoch_time, keep_avg) + + # Plot Epoch Stats + if args.rank == 0: + epoch_stats = {"epoch_time": epoch_time} + epoch_stats.update(keep_avg.avg_values) + tb_logger.tb_train_epoch_stats(global_step, epoch_stats) + if config.tb_model_param_stats: + tb_logger.tb_model_weights(model, global_step) + return keep_avg.avg_values, global_step + + +@torch.no_grad() +def evaluate(data_loader, model, criterion, ap, global_step, epoch, training_phase): + model.eval() + epoch_time = 0 + keep_avg = KeepAverage() + c_logger.print_eval_start() + if data_loader is not None: for num_iter, data in enumerate(data_loader): start_time = time.time() # format data - ( - text_input, - text_lengths, - mel_targets, - mel_lengths, - speaker_c, - avg_text_length, - avg_spec_length, - _, - ) = format_data(data) - - loader_time = time.time() - end_time - - global_step += 1 - optimizer.zero_grad() + text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _ = format_data(data) # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward( text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase ) - # compute loss - loss_dict = criterion( - logp, - decoder_output, - mel_targets, - mel_lengths, - dur_output, - dur_mas_output, - text_lengths, - global_step, - phase=training_phase, - ) - - # backward pass with loss scaling - if c.mixed_precision: - scaler.scale(loss_dict["loss"]).backward() - scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) - scaler.step(optimizer) - scaler.update() - else: - loss_dict["loss"].backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) - optimizer.step() - - # setup lr - if c.noam_schedule: - scheduler.step() - - # current_lr - current_lr = optimizer.param_groups[0]["lr"] - - # compute alignment error (the lower the better ) - align_error = 1 - alignment_diagonal_score(alignments, binary=True) - loss_dict["align_error"] = align_error + # compute loss + loss_dict = criterion( + logp, + decoder_output, + mel_targets, + mel_lengths, + dur_output, + dur_mas_output, + text_lengths, + global_step, + phase=training_phase, + ) + # step time step_time = time.time() - start_time epoch_time += step_time + # compute alignment score + align_error = 1 - alignment_diagonal_score(alignments, binary=True) + loss_dict["align_error"] = align_error + # aggregate losses from processes if num_gpus > 1: loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus) @@ -206,357 +341,222 @@ if __name__ == "__main__": update_train_values = dict() for key, value in loss_dict.items(): update_train_values["avg_" + key] = value - update_train_values["avg_loader_time"] = loader_time - update_train_values["avg_step_time"] = step_time keep_avg.update_values(update_train_values) - # print training progress - if global_step % c.print_step == 0: - log_dict = { - "avg_spec_length": [avg_spec_length, 1], # value, precision - "avg_text_length": [avg_text_length, 1], - "step_time": [step_time, 4], - "loader_time": [loader_time, 2], - "current_lr": current_lr, - } - c_logger.print_train_step(batch_n_iter, num_iter, global_step, log_dict, loss_dict, keep_avg.avg_values) + if config.print_eval: + c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) - if args.rank == 0: - # Plot Training Iter Stats - # reduce TB load - if global_step % c.tb_plot_step == 0: - iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time} - iter_stats.update(loss_dict) - tb_logger.tb_train_iter_stats(global_step, iter_stats) - - if global_step % c.save_step == 0: - if c.checkpoint: - # save model - save_checkpoint( - model, - optimizer, - global_step, - epoch, - 1, - OUT_PATH, - model_characters, - model_loss=loss_dict["loss"], - ) - - # wait all kernels to be completed - torch.cuda.synchronize() - - # Diagnostic visualizations - if decoder_output is not None: - idx = np.random.randint(mel_targets.shape[0]) - pred_spec = decoder_output[idx].detach().data.cpu().numpy().T - gt_spec = mel_targets[idx].data.cpu().numpy().T - align_img = alignments[idx].data.cpu() - - figures = { - "prediction": plot_spectrogram(pred_spec, ap), - "ground_truth": plot_spectrogram(gt_spec, ap), - "alignment": plot_alignment(align_img), - } - - tb_logger.tb_train_figures(global_step, figures) - - # Sample audio - train_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"]) - end_time = time.time() - - # print epoch stats - c_logger.print_train_epoch_end(global_step, epoch, epoch_time, keep_avg) - - # Plot Epoch Stats if args.rank == 0: - epoch_stats = {"epoch_time": epoch_time} - epoch_stats.update(keep_avg.avg_values) - tb_logger.tb_train_epoch_stats(global_step, epoch_stats) - if c.tb_model_param_stats: - tb_logger.tb_model_weights(model, global_step) - return keep_avg.avg_values, global_step + # Diagnostic visualizations + idx = np.random.randint(mel_targets.shape[0]) + pred_spec = decoder_output[idx].detach().data.cpu().numpy().T + gt_spec = mel_targets[idx].data.cpu().numpy().T + align_img = alignments[idx].data.cpu() - @torch.no_grad() - def evaluate(data_loader, model, criterion, ap, global_step, epoch, training_phase): - model.eval() - epoch_time = 0 - keep_avg = KeepAverage() - c_logger.print_eval_start() - if data_loader is not None: - for num_iter, data in enumerate(data_loader): - start_time = time.time() + eval_figures = { + "prediction": plot_spectrogram(pred_spec, ap, output_fig=False), + "ground_truth": plot_spectrogram(gt_spec, ap, output_fig=False), + "alignment": plot_alignment(align_img, output_fig=False), + } - # format data - text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _ = format_data(data) + # Sample audio + eval_audio = ap.inv_melspectrogram(pred_spec.T) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) - # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): - decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward( - text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase - ) + # Plot Validation Stats + tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) + tb_logger.tb_eval_figures(global_step, eval_figures) - # compute loss - loss_dict = criterion( - logp, - decoder_output, - mel_targets, - mel_lengths, - dur_output, - dur_mas_output, - text_lengths, - global_step, - phase=training_phase, + if args.rank == 0 and epoch >= config.test_delay_epochs: + if config.test_sentences_file: + with open(config.test_sentences_file, "r") as f: + test_sentences = [s.strip() for s in f.readlines()] + else: + test_sentences = [ + "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", + "Be a voice, not an echo.", + "I'm sorry Dave. I'm afraid I can't do that.", + "This cake is great. It's so delicious and moist.", + "Prior to November 22, 1963.", + ] + + # test sentences + test_audios = {} + test_figures = {} + print(" | > Synthesizing test sentences") + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: + speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][ + "embedding" + ] + speaker_id = None + else: + speaker_id = 0 + speaker_embedding = None + else: + speaker_id = None + speaker_embedding = None + + for idx, test_sentence in enumerate(test_sentences): + try: + wav, alignment, _, postnet_output, _, _ = synthesis( + model, + test_sentence, + config, + use_cuda, + ap, + speaker_id=speaker_id, + speaker_embedding=speaker_embedding, + style_wav=None, + truncated=False, + enable_eos_bos_chars=config.enable_eos_bos_chars, # pylint: disable=unused-argument + use_griffin_lim=True, + do_trim_silence=False, ) - # step time - step_time = time.time() - start_time - epoch_time += step_time - - # compute alignment score - align_error = 1 - alignment_diagonal_score(alignments, binary=True) - loss_dict["align_error"] = align_error - - # aggregate losses from processes - if num_gpus > 1: - loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus) - loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, num_gpus) - loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, num_gpus) - loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) - - # detach loss values - loss_dict_new = dict() - for key, value in loss_dict.items(): - if isinstance(value, (int, float)): - loss_dict_new[key] = value - else: - loss_dict_new[key] = value.item() - loss_dict = loss_dict_new - - # update avg stats - update_train_values = dict() - for key, value in loss_dict.items(): - update_train_values["avg_" + key] = value - keep_avg.update_values(update_train_values) - - if c.print_eval: - c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) - - if args.rank == 0: - # Diagnostic visualizations - idx = np.random.randint(mel_targets.shape[0]) - pred_spec = decoder_output[idx].detach().data.cpu().numpy().T - gt_spec = mel_targets[idx].data.cpu().numpy().T - align_img = alignments[idx].data.cpu() - - eval_figures = { - "prediction": plot_spectrogram(pred_spec, ap, output_fig=False), - "ground_truth": plot_spectrogram(gt_spec, ap, output_fig=False), - "alignment": plot_alignment(align_img, output_fig=False), - } - - # Sample audio - eval_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"]) - - # Plot Validation Stats - tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) - tb_logger.tb_eval_figures(global_step, eval_figures) - - if args.rank == 0 and epoch >= c.test_delay_epochs: - if c.test_sentences_file is None: - test_sentences = [ - "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", - "Be a voice, not an echo.", - "I'm sorry Dave. I'm afraid I can't do that.", - "This cake is great. It's so delicious and moist.", - "Prior to November 22, 1963.", - ] - else: - with open(c.test_sentences_file, "r") as f: - test_sentences = [s.strip() for s in f.readlines()] - - # test sentences - test_audios = {} - test_figures = {} - print(" | > Synthesizing test sentences") - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: - speaker_embedding = speaker_mapping[ - list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)] - ]["embedding"] - speaker_id = None - else: - speaker_id = 0 - speaker_embedding = None - else: - speaker_id = None - speaker_embedding = None - - style_wav = c.get("style_wav_for_test") - for idx, test_sentence in enumerate(test_sentences): - try: - wav, alignment, _, postnet_output, _, _ = synthesis( - model, - test_sentence, - c, - use_cuda, - ap, - speaker_id=speaker_id, - speaker_embedding=speaker_embedding, - style_wav=style_wav, - truncated=False, - enable_eos_bos_chars=c.enable_eos_bos_chars, # pylint: disable=unused-argument - use_griffin_lim=True, - do_trim_silence=False, - ) - - file_path = os.path.join(AUDIO_PATH, str(global_step)) - os.makedirs(file_path, exist_ok=True) - file_path = os.path.join(file_path, "TestSentence_{}.wav".format(idx)) - ap.save_wav(wav, file_path) - test_audios["{}-audio".format(idx)] = wav - test_figures["{}-prediction".format(idx)] = plot_spectrogram(postnet_output, ap) - test_figures["{}-alignment".format(idx)] = plot_alignment(alignment) - except: # pylint: disable=bare-except - print(" !! Error creating Test Sentence -", idx) - traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"]) - tb_logger.tb_test_figures(global_step, test_figures) - return keep_avg.avg_values - - def main(args): # pylint: disable=redefined-outer-name - # pylint: disable=global-variable-undefined - global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping - # Audio processor - ap = AudioProcessor(**c.audio) - if "characters" in c.keys(): - symbols, phonemes = make_symbols(**c.characters) - - # DISTRUBUTED - if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"]) - - # set model characters - model_characters = phonemes if c.use_phonemes else symbols - num_chars = len(model_characters) - - # load data instances - meta_data_train, meta_data_eval = load_meta_data(c.datasets, eval_split=True) - - # set the portion of the data used for training if set in config.json - if "train_portion" in c.keys(): - meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)] - if "eval_portion" in c.keys(): - meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)] - - # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH) - - # setup model - model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim) - optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) - criterion = AlignTTSLoss(c) - - if args.restore_path: - print(f" > Restoring from {os.path.basename(args.restore_path)} ...") - checkpoint = torch.load(args.restore_path, map_location="cpu") - try: - # TODO: fix optimizer init, model.cuda() needs to be called before - # optimizer restore - optimizer.load_state_dict(checkpoint["optimizer"]) - if c.reinit_layers: - raise RuntimeError - model.load_state_dict(checkpoint["model"]) + file_path = os.path.join(AUDIO_PATH, str(global_step)) + os.makedirs(file_path, exist_ok=True) + file_path = os.path.join(file_path, "TestSentence_{}.wav".format(idx)) + ap.save_wav(wav, file_path) + test_audios["{}-audio".format(idx)] = wav + test_figures["{}-prediction".format(idx)] = plot_spectrogram(postnet_output, ap) + test_figures["{}-alignment".format(idx)] = plot_alignment(alignment) except: # pylint: disable=bare-except - print(" > Partial model initialization.") - model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint["model"], c) - model.load_state_dict(model_dict) - del model_dict + print(" !! Error creating Test Sentence -", idx) + traceback.print_exc() + tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"]) + tb_logger.tb_test_figures(global_step, test_figures) + return keep_avg.avg_values - for group in optimizer.param_groups: - group["initial_lr"] = c.lr - print(" > Model restored from step %d" % checkpoint["step"], flush=True) - args.restore_step = checkpoint["step"] - else: - args.restore_step = 0 - if use_cuda: - model.cuda() - criterion.cuda() +def main(args): # pylint: disable=redefined-outer-name + # pylint: disable=global-variable-undefined + global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping + # Audio processor + ap = AudioProcessor(**config.audio.to_dict()) + if config.has("characters") and config.characters: + symbols, phonemes = make_symbols(**config.characters.to_dict()) - # DISTRUBUTED - if num_gpus > 1: - model = DDP_th(model, device_ids=[args.rank]) + # DISTRUBUTED + if num_gpus > 1: + init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"]) - if c.noam_schedule: - scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1) - else: - scheduler = None + # set model characters + model_characters = phonemes if config.use_phonemes else symbols + num_chars = len(model_characters) - num_params = count_parameters(model) - print("\n > Model has {} parameters".format(num_params), flush=True) + # load data instances + meta_data_train, meta_data_eval = load_meta_data(config.datasets, eval_split=True) - if args.restore_step == 0 or not args.best_path: - best_loss = float("inf") - print(" > Starting with inf best loss.") - else: - print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") - best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] - print(f" > Starting with loaded last best loss {best_loss}.") - keep_all_best = c.get("keep_all_best", False) - keep_after = c.get("keep_after", 10000) # void if keep_all_best False + # parse speakers + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) - # define dataloaders - train_loader = setup_loader(ap, 1, is_val=False, verbose=True) - eval_loader = setup_loader(ap, 1, is_val=True, verbose=True) + # setup model + model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim) + optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) + criterion = AlignTTSLoss(config) - global_step = args.restore_step + if args.restore_path: + print(f" > Restoring from {os.path.basename(args.restore_path)} ...") + checkpoint = torch.load(args.restore_path, map_location="cpu") + try: + # TODO: fix optimizer init, model.cuda() needs to be called before + # optimizer restore + optimizer.load_state_dict(checkpoint["optimizer"]) + if config.reinit_layers: + raise RuntimeError + model.load_state_dict(checkpoint["model"]) + except: # pylint: disable=bare-except + print(" > Partial model initialization.") + model_dict = model.state_dict() + model_dict = set_init_dict(model_dict, checkpoint["model"], config) + model.load_state_dict(model_dict) + del model_dict - def set_phase(): - """Set AlignTTS training phase""" - if isinstance(c.phase_start_steps, list): - vals = [i < global_step for i in c.phase_start_steps] - if not True in vals: - phase = 0 - else: - phase = ( - len(c.phase_start_steps) - [i < global_step for i in c.phase_start_steps][::-1].index(True) - 1 - ) + for group in optimizer.param_groups: + group["initial_lr"] = config.lr + print(" > Model restored from step %d" % checkpoint["step"], flush=True) + args.restore_step = checkpoint["step"] + else: + args.restore_step = 0 + + if use_cuda: + model.cuda() + criterion.cuda() + + # DISTRUBUTED + if num_gpus > 1: + model = DDP_th(model, device_ids=[args.rank]) + + if config.noam_schedule: + scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1) + else: + scheduler = None + + num_params = count_parameters(model) + print("\n > Model has {} parameters".format(num_params), flush=True) + + if args.restore_step == 0 or not args.best_path: + best_loss = float("inf") + print(" > Starting with inf best loss.") + else: + print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") + best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] + print(f" > Starting with loaded last best loss {best_loss}.") + keep_all_best = config.keep_all_best + keep_after = config.keep_after # void if keep_all_best False + + # define dataloaders + train_loader = setup_loader(ap, 1, is_val=False, verbose=True) + eval_loader = setup_loader(ap, 1, is_val=True, verbose=True) + + global_step = args.restore_step + + def set_phase(): + """Set AlignTTS training phase""" + if isinstance(config.phase_start_steps, list): + vals = [i < global_step for i in config.phase_start_steps] + if not True in vals: + phase = 0 else: - phase = None - return phase + phase = ( + len(config.phase_start_steps) + - [i < global_step for i in config.phase_start_steps][::-1].index(True) + - 1 + ) + else: + phase = None + return phase - for epoch in range(0, c.epochs): - cur_phase = set_phase() - print(f"\n > Current AlignTTS phase: {cur_phase}") - c_logger.print_epoch_start(epoch, c.epochs) - train_avg_loss_dict, global_step = train( - train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, cur_phase - ) - eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch, cur_phase) - c_logger.print_epoch_end(epoch, eval_avg_loss_dict) - target_loss = train_avg_loss_dict["avg_loss"] - if c.run_eval: - target_loss = eval_avg_loss_dict["avg_loss"] - best_loss = save_best_model( - target_loss, - best_loss, - model, - optimizer, - global_step, - epoch, - 1, - OUT_PATH, - model_characters, - keep_all_best=keep_all_best, - keep_after=keep_after, - ) + for epoch in range(0, config.epochs): + cur_phase = set_phase() + print(f"\n > Current AlignTTS phase: {cur_phase}") + c_logger.print_epoch_start(epoch, config.epochs) + train_avg_loss_dict, global_step = train( + train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, cur_phase + ) + eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch, cur_phase) + c_logger.print_epoch_end(epoch, eval_avg_loss_dict) + target_loss = train_avg_loss_dict["avg_loss"] + if config.run_eval: + target_loss = eval_avg_loss_dict["avg_loss"] + best_loss = save_best_model( + target_loss, + best_loss, + model, + optimizer, + global_step, + epoch, + 1, + OUT_PATH, + model_characters, + keep_all_best=keep_all_best, + keep_after=keep_after, + ) - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts") + +if __name__ == "__main__": + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) diff --git a/TTS/bin/train_encoder.py b/TTS/bin/train_encoder.py index 3a3f876e..66ecddfe 100644 --- a/TTS/bin/train_encoder.py +++ b/TTS/bin/train_encoder.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import argparse import os import sys import time @@ -13,20 +12,13 @@ from torch.utils.data import DataLoader from TTS.speaker_encoder.dataset import MyDataset from TTS.speaker_encoder.losses import AngleProtoLoss, GE2ELoss from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.speaker_encoder.utils.generic_utils import check_config_speaker_encoder, save_best_model +from TTS.speaker_encoder.utils.io import save_best_model, save_checkpoint from TTS.speaker_encoder.utils.visual import plot_embeddings from TTS.tts.datasets.preprocess import load_meta_data +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor -from TTS.utils.generic_utils import ( - count_parameters, - create_experiment_folder, - get_git_branch, - remove_experiment_folder, - set_init_dict, -) -from TTS.utils.io import copy_model_files, load_config +from TTS.utils.generic_utils import count_parameters, remove_experiment_folder, set_init_dict from TTS.utils.radam import RAdam -from TTS.utils.tensorboard_logger import TensorboardLogger from TTS.utils.training import NoamLR, check_update torch.backends.cudnn.enabled = True @@ -105,8 +97,9 @@ def train(model, criterion, optimizer, scheduler, ap, global_step): # Averaged Loss and Averaged Loader Time avg_loss = 0.01 * loss.item() + 0.99 * avg_loss if avg_loss != 0 else loss.item() + num_loader_workers = c.num_loader_workers if c.num_loader_workers > 0 else 1 avg_loader_time = ( - 1 / c.num_loader_workers * loader_time + (c.num_loader_workers - 1) / c.num_loader_workers * avg_loader_time + 1 / num_loader_workers * loader_time + (num_loader_workers - 1) / num_loader_workers * avg_loader_time if avg_loader_time != 0 else loader_time ) @@ -139,8 +132,13 @@ def train(model, criterion, optimizer, scheduler, ap, global_step): # save best model best_loss = save_best_model(model, optimizer, avg_loss, best_loss, OUT_PATH, global_step) - end_time = time.time() + + # checkpoint and check stop train cond. + if global_step >= c.max_train_step or global_step % c.save_step == 0: + save_checkpoint(model, optimizer, avg_loss, OUT_PATH, global_step) + if global_step >= c.max_train_step: + break return avg_loss, global_step @@ -149,12 +147,12 @@ def main(args): # pylint: disable=redefined-outer-name global meta_data_train global meta_data_eval - ap = AudioProcessor(**c.audio) + ap = AudioProcessor(**c.audio.to_dict()) model = SpeakerEncoder( - input_dim=c.model["input_dim"], - proj_dim=c.model["proj_dim"], - lstm_dim=c.model["lstm_dim"], - num_lstm_layers=c.model["num_lstm_layers"], + input_dim=c.model_params["input_dim"], + proj_dim=c.model_params["proj_dim"], + lstm_dim=c.model_params["lstm_dim"], + num_lstm_layers=c.model_params["num_lstm_layers"], ) optimizer = RAdam(model.parameters(), lr=c.lr) @@ -168,11 +166,6 @@ def main(args): # pylint: disable=redefined-outer-name if args.restore_path: checkpoint = torch.load(args.restore_path) try: - # TODO: fix optimizer init, model.cuda() needs to be called before - # optimizer restore - # optimizer.load_state_dict(checkpoint['optimizer']) - if c.reinit_layers: - raise RuntimeError model.load_state_dict(checkpoint["model"]) except KeyError: print(" > Partial model initialization.") @@ -207,47 +200,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--restore_path", type=str, help="Path to model outputs (checkpoint, tensorboard etc.).", default=0 - ) - parser.add_argument( - "--config_path", - type=str, - required=True, - help="Path to config file for training.", - ) - parser.add_argument("--debug", type=bool, default=True, help="Do not verify commit integrity to run training.") - parser.add_argument("--data_path", type=str, default="", help="Defines the data path. It overwrites config.json.") - parser.add_argument("--output_path", type=str, help="path for training outputs.", default="") - parser.add_argument("--output_folder", type=str, default="", help="folder name for training outputs.") - args = parser.parse_args() - - # setup output paths and read configs - c = load_config(args.config_path) - check_config_speaker_encoder(c) - _ = os.path.dirname(os.path.realpath(__file__)) - if args.data_path != "": - c.data_path = args.data_path - - if args.output_path == "": - OUT_PATH = os.path.join(_, c.output_path) - else: - OUT_PATH = args.output_path - - if args.output_folder == "": - OUT_PATH = create_experiment_folder(OUT_PATH, c.run_name, args.debug) - else: - OUT_PATH = os.path.join(OUT_PATH, args.output_folder) - - new_fields = {} - if args.restore_path: - new_fields["restore_path"] = args.restore_path - new_fields["github_branch"] = get_git_branch() - copy_model_files(c, args.config_path, OUT_PATH, new_fields) - - LOG_DIR = OUT_PATH - tb_logger = TensorboardLogger(LOG_DIR, model_name="Speaker_Encoder") + args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) diff --git a/TTS/bin/train_glow_tts.py b/TTS/bin/train_glow_tts.py index d3b3d0e2..e93a4e8a 100644 --- a/TTS/bin/train_glow_tts.py +++ b/TTS/bin/train_glow_tts.py @@ -24,7 +24,7 @@ from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.synthesis import synthesis from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed, reduce_tensor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -35,45 +35,45 @@ use_cuda, num_gpus = setup_torch_training_env(True, False) def setup_loader(ap, r, is_val=False, verbose=False): - if is_val and not c.run_eval: + if is_val and not config.run_eval: loader = None else: dataset = MyDataset( r, - c.text_cleaner, + config.text_cleaner, compute_linear_spec=False, meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, - tp=c.characters if "characters" in c.keys() else None, - add_blank=c["add_blank"] if "add_blank" in c.keys() else False, - batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, - min_seq_len=c.min_seq_len, - max_seq_len=c.max_seq_len, - phoneme_cache_path=c.phoneme_cache_path, - use_phonemes=c.use_phonemes, - phoneme_language=c.phoneme_language, - enable_eos_bos=c.enable_eos_bos_chars, - use_noise_augment=c["use_noise_augment"] and not is_val, + tp=config.characters, + add_blank=config["add_blank"], + batch_group_size=0 if is_val else config.batch_group_size * config.batch_size, + min_seq_len=config.min_seq_len, + max_seq_len=config.max_seq_len, + phoneme_cache_path=config.phoneme_cache_path, + use_phonemes=config.use_phonemes, + phoneme_language=config.phoneme_language, + enable_eos_bos=config.enable_eos_bos_chars, + use_noise_augment=not is_val, verbose=verbose, speaker_mapping=speaker_mapping - if c.use_speaker_embedding and c.use_external_speaker_embedding_file + if config.use_speaker_embedding and config.use_external_speaker_embedding_file else None, ) - if c.use_phonemes and c.compute_input_seq_cache: + if config.use_phonemes and config.compute_input_seq_cache: # precompute phonemes to have a better estimate of sequence lengths. - dataset.compute_input_seq(c.num_loader_workers) + dataset.compute_input_seq(config.num_loader_workers) dataset.sort_items() sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader( dataset, - batch_size=c.eval_batch_size if is_val else c.batch_size, + batch_size=config.eval_batch_size if is_val else config.batch_size, shuffle=False, collate_fn=dataset.collate_fn, drop_last=False, sampler=sampler, - num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers, + num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers, pin_memory=False, ) return loader @@ -91,8 +91,8 @@ def format_data(data): avg_text_length = torch.mean(text_lengths.float()) avg_spec_length = torch.mean(mel_lengths.float()) - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: # return precomputed embedding vector speaker_c = data[8] else: @@ -147,7 +147,7 @@ def data_depended_init(data_loader, model): # forward pass model _ = model.forward(text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=spekaer_embed) - if num_iter == c.data_dep_init_iter: + if num_iter == config.data_dep_init_steps: break num_iter += 1 @@ -168,12 +168,12 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch_time = 0 keep_avg = KeepAverage() if use_cuda: - batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus)) + batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus)) else: - batch_n_iter = int(len(data_loader.dataset) / c.batch_size) + batch_n_iter = int(len(data_loader.dataset) / config.batch_size) end_time = time.time() c_logger.print_train_start() - scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None + scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None for num_iter, data in enumerate(data_loader): start_time = time.time() @@ -196,7 +196,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, optimizer.zero_grad() # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): z, logdet, y_mean, y_log_scale, alignments, o_dur_log, o_total_dur = model.forward( text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c ) @@ -205,19 +205,19 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, loss_dict = criterion(z, y_mean, y_log_scale, logdet, mel_lengths, o_dur_log, o_total_dur, text_lengths) # backward pass with loss scaling - if c.mixed_precision: + if config.mixed_precision: scaler.scale(loss_dict["loss"]).backward() scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) scaler.step(optimizer) scaler.update() else: loss_dict["loss"].backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) optimizer.step() # setup lr - if c.noam_schedule: + if config.noam_schedule: scheduler.step() # current_lr @@ -254,7 +254,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, keep_avg.update_values(update_train_values) # print training progress - if global_step % c.print_step == 0: + if global_step % config.print_step == 0: log_dict = { "avg_spec_length": [avg_spec_length, 1], # value, precision "avg_text_length": [avg_text_length, 1], @@ -267,13 +267,13 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, if args.rank == 0: # Plot Training Iter Stats # reduce TB load - if global_step % c.tb_plot_step == 0: + if global_step % config.tb_plot_step == 0: iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time} iter_stats.update(loss_dict) tb_logger.tb_train_iter_stats(global_step, iter_stats) - if global_step % c.save_step == 0: - if c.checkpoint: + if global_step % config.save_step == 0: + if config.checkpoint: # save model save_checkpoint( model, @@ -314,7 +314,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, # Sample audio train_audio = ap.inv_melspectrogram(const_spec.T) - tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"]) + tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() # print epoch stats @@ -325,7 +325,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch_stats = {"epoch_time": epoch_time} epoch_stats.update(keep_avg.avg_values) tb_logger.tb_train_epoch_stats(global_step, epoch_stats) - if c.tb_model_param_stats: + if config.tb_model_param_stats: tb_logger.tb_model_weights(model, global_step) return keep_avg.avg_values, global_step @@ -380,7 +380,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): update_train_values["avg_" + key] = value keep_avg.update_values(update_train_values) - if c.print_eval: + if config.print_eval: c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) if args.rank == 0: @@ -406,14 +406,17 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # Sample audio eval_audio = ap.inv_melspectrogram(const_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"]) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) # Plot Validation Stats tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) tb_logger.tb_eval_figures(global_step, eval_figures) - if args.rank == 0 and epoch >= c.test_delay_epochs: - if c.test_sentences_file is None: + if args.rank == 0 and epoch >= config.test_delay_epochs: + if config.test_sentences_file: + with open(config.test_sentences_file, "r") as f: + test_sentences = [s.strip() for s in f.readlines()] + else: test_sentences = [ "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", "Be a voice, not an echo.", @@ -421,16 +424,13 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): "This cake is great. It's so delicious and moist.", "Prior to November 22, 1963.", ] - else: - with open(c.test_sentences_file, "r") as f: - test_sentences = [s.strip() for s in f.readlines()] # test sentences test_audios = {} test_figures = {} print(" | > Synthesizing test sentences") - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][ "embedding" ] @@ -442,20 +442,20 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): speaker_id = None speaker_embedding = None - style_wav = c.get("style_wav_for_test") + style_wav = config.style_wav_for_test for idx, test_sentence in enumerate(test_sentences): try: wav, alignment, _, postnet_output, _, _ = synthesis( model, test_sentence, - c, + config, use_cuda, ap, speaker_id=speaker_id, speaker_embedding=speaker_embedding, style_wav=style_wav, truncated=False, - enable_eos_bos_chars=c.enable_eos_bos_chars, # pylint: disable=unused-argument + enable_eos_bos_chars=config.enable_eos_bos_chars, # pylint: disable=unused-argument use_griffin_lim=True, do_trim_silence=False, ) @@ -470,7 +470,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): except: # pylint: disable=bare-except print(" !! Error creating Test Sentence -", idx) traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"]) + tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"]) tb_logger.tb_test_figures(global_step, test_figures) return keep_avg.avg_values @@ -479,33 +479,27 @@ def main(args): # pylint: disable=redefined-outer-name # pylint: disable=global-variable-undefined global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping # Audio processor - ap = AudioProcessor(**c.audio) - if "characters" in c.keys(): - symbols, phonemes = make_symbols(**c.characters) + ap = AudioProcessor(**config.audio.to_dict()) + if config.has("characters") and config.characters: + symbols, phonemes = make_symbols(**config.characters.to_dict()) # DISTRUBUTED if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"]) + init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"]) # set model characters - model_characters = phonemes if c.use_phonemes else symbols + model_characters = phonemes if config.use_phonemes else symbols num_chars = len(model_characters) # load data instances - meta_data_train, meta_data_eval = load_meta_data(c.datasets) - - # set the portion of the data used for training - if "train_portion" in c.keys(): - meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)] - if "eval_portion" in c.keys(): - meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)] + meta_data_train, meta_data_eval = load_meta_data(config.datasets) # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH) + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) # setup model - model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim) - optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) + model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim) + optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) criterion = GlowTTSLoss() if args.restore_path: @@ -515,18 +509,16 @@ def main(args): # pylint: disable=redefined-outer-name # TODO: fix optimizer init, model.cuda() needs to be called before # optimizer restore optimizer.load_state_dict(checkpoint["optimizer"]) - if c.reinit_layers: - raise RuntimeError model.load_state_dict(checkpoint["model"]) except: # pylint: disable=bare-except print(" > Partial model initialization.") model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint["model"], c) + model_dict = set_init_dict(model_dict, checkpoint["model"], config) model.load_state_dict(model_dict) del model_dict for group in optimizer.param_groups: - group["initial_lr"] = c.lr + group["initial_lr"] = config.lr print(f" > Model restored from step {checkpoint['step']:d}", flush=True) args.restore_step = checkpoint["step"] else: @@ -540,8 +532,8 @@ def main(args): # pylint: disable=redefined-outer-name if num_gpus > 1: model = DDP_th(model, device_ids=[args.rank]) - if c.noam_schedule: - scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1) + if config.noam_schedule: + scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1) else: scheduler = None @@ -555,8 +547,8 @@ def main(args): # pylint: disable=redefined-outer-name print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] print(f" > Starting with loaded last best loss {best_loss}.") - keep_all_best = c.get("keep_all_best", False) - keep_after = c.get("keep_after", 10000) # void if keep_all_best False + keep_all_best = config.keep_all_best + keep_after = config.keep_after # void if keep_all_best False # define dataloaders train_loader = setup_loader(ap, 1, is_val=False, verbose=True) @@ -564,15 +556,15 @@ def main(args): # pylint: disable=redefined-outer-name global_step = args.restore_step model = data_depended_init(train_loader, model) - for epoch in range(0, c.epochs): - c_logger.print_epoch_start(epoch, c.epochs) + for epoch in range(0, config.epochs): + c_logger.print_epoch_start(epoch, config.epochs) train_avg_loss_dict, global_step = train( train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch ) eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) target_loss = train_avg_loss_dict["avg_loss"] - if c.run_eval: + if config.run_eval: target_loss = eval_avg_loss_dict["avg_loss"] best_loss = save_best_model( target_loss, @@ -581,7 +573,7 @@ def main(args): # pylint: disable=redefined-outer-name optimizer, global_step, epoch, - c.r, + config.r, OUT_PATH, model_characters, keep_all_best=keep_all_best, @@ -590,8 +582,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts") + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) diff --git a/TTS/bin/train_speedy_speech.py b/TTS/bin/train_speedy_speech.py index 3adbe513..2fba3df1 100644 --- a/TTS/bin/train_speedy_speech.py +++ b/TTS/bin/train_speedy_speech.py @@ -25,7 +25,7 @@ from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.synthesis import synthesis from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed, reduce_tensor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -36,45 +36,45 @@ use_cuda, num_gpus = setup_torch_training_env(True, False) def setup_loader(ap, r, is_val=False, verbose=False): - if is_val and not c.run_eval: + if is_val and not config.run_eval: loader = None else: dataset = MyDataset( r, - c.text_cleaner, + config.text_cleaner, compute_linear_spec=False, meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, - tp=c.characters if "characters" in c.keys() else None, - add_blank=c["add_blank"] if "add_blank" in c.keys() else False, - batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, - min_seq_len=c.min_seq_len, - max_seq_len=c.max_seq_len, - phoneme_cache_path=c.phoneme_cache_path, - use_phonemes=c.use_phonemes, - phoneme_language=c.phoneme_language, - enable_eos_bos=c.enable_eos_bos_chars, + tp=config.characters, + add_blank=config["add_blank"], + batch_group_size=0 if is_val else config.batch_group_size * config.batch_size, + min_seq_len=config.min_seq_len, + max_seq_len=config.max_seq_len, + phoneme_cache_path=config.phoneme_cache_path, + use_phonemes=config.use_phonemes, + phoneme_language=config.phoneme_language, + enable_eos_bos=config.enable_eos_bos_chars, use_noise_augment=not is_val, verbose=verbose, speaker_mapping=speaker_mapping - if c.use_speaker_embedding and c.use_external_speaker_embedding_file + if config.use_speaker_embedding and config.use_external_speaker_embedding_file else None, ) - if c.use_phonemes and c.compute_input_seq_cache: + if config.use_phonemes and config.compute_input_seq_cache: # precompute phonemes to have a better estimate of sequence lengths. - dataset.compute_input_seq(c.num_loader_workers) + dataset.compute_input_seq(config.num_loader_workers) dataset.sort_items() sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader( dataset, - batch_size=c.eval_batch_size if is_val else c.batch_size, + batch_size=config.eval_batch_size if is_val else config.batch_size, shuffle=False, collate_fn=dataset.collate_fn, drop_last=False, sampler=sampler, - num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers, + num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers, pin_memory=False, ) return loader @@ -92,8 +92,8 @@ def format_data(data): avg_text_length = torch.mean(text_lengths.float()) avg_spec_length = torch.mean(mel_lengths.float()) - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: # return precomputed embedding vector speaker_c = data[8] else: @@ -150,12 +150,12 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch_time = 0 keep_avg = KeepAverage() if use_cuda: - batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus)) + batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus)) else: - batch_n_iter = int(len(data_loader.dataset) / c.batch_size) + batch_n_iter = int(len(data_loader.dataset) / config.batch_size) end_time = time.time() c_logger.print_train_start() - scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None + scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None for num_iter, data in enumerate(data_loader): start_time = time.time() @@ -179,7 +179,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, optimizer.zero_grad() # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): decoder_output, dur_output, alignments = model.forward( text_input, text_lengths, mel_lengths, dur_target, g=speaker_c ) @@ -190,19 +190,19 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, ) # backward pass with loss scaling - if c.mixed_precision: + if config.mixed_precision: scaler.scale(loss_dict["loss"]).backward() scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) scaler.step(optimizer) scaler.update() else: loss_dict["loss"].backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) optimizer.step() # setup lr - if c.noam_schedule: + if config.noam_schedule: scheduler.step() # current_lr @@ -240,7 +240,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, keep_avg.update_values(update_train_values) # print training progress - if global_step % c.print_step == 0: + if global_step % config.print_step == 0: log_dict = { "avg_spec_length": [avg_spec_length, 1], # value, precision "avg_text_length": [avg_text_length, 1], @@ -253,13 +253,13 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, if args.rank == 0: # Plot Training Iter Stats # reduce TB load - if global_step % c.tb_plot_step == 0: + if global_step % config.tb_plot_step == 0: iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time} iter_stats.update(loss_dict) tb_logger.tb_train_iter_stats(global_step, iter_stats) - if global_step % c.save_step == 0: - if c.checkpoint: + if global_step % config.save_step == 0: + if config.checkpoint: # save model save_checkpoint( model, @@ -291,7 +291,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, # Sample audio train_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"]) + tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() # print epoch stats @@ -302,7 +302,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch_stats = {"epoch_time": epoch_time} epoch_stats.update(keep_avg.avg_values) tb_logger.tb_train_epoch_stats(global_step, epoch_stats) - if c.tb_model_param_stats: + if config.tb_model_param_stats: tb_logger.tb_model_weights(model, global_step) return keep_avg.avg_values, global_step @@ -321,7 +321,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _, dur_target, _ = format_data(data) # forward pass model - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): decoder_output, dur_output, alignments = model.forward( text_input, text_lengths, mel_lengths, dur_target, g=speaker_c ) @@ -361,7 +361,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): update_train_values["avg_" + key] = value keep_avg.update_values(update_train_values) - if c.print_eval: + if config.print_eval: c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) if args.rank == 0: @@ -379,14 +379,17 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): # Sample audio eval_audio = ap.inv_melspectrogram(pred_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"]) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) # Plot Validation Stats tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) tb_logger.tb_eval_figures(global_step, eval_figures) - if args.rank == 0 and epoch >= c.test_delay_epochs: - if c.test_sentences_file is None: + if args.rank == 0 and epoch >= config.test_delay_epochs: + if config.test_sentences_file: + with open(config.test_sentences_file, "r") as f: + test_sentences = [s.strip() for s in f.readlines()] + else: test_sentences = [ "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", "Be a voice, not an echo.", @@ -394,16 +397,13 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): "This cake is great. It's so delicious and moist.", "Prior to November 22, 1963.", ] - else: - with open(c.test_sentences_file, "r") as f: - test_sentences = [s.strip() for s in f.readlines()] # test sentences test_audios = {} test_figures = {} print(" | > Synthesizing test sentences") - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][ "embedding" ] @@ -415,20 +415,19 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): speaker_id = None speaker_embedding = None - style_wav = c.get("style_wav_for_test") for idx, test_sentence in enumerate(test_sentences): try: wav, alignment, _, postnet_output, _, _ = synthesis( model, test_sentence, - c, + config, use_cuda, ap, speaker_id=speaker_id, speaker_embedding=speaker_embedding, - style_wav=style_wav, + style_wav=None, truncated=False, - enable_eos_bos_chars=c.enable_eos_bos_chars, # pylint: disable=unused-argument + enable_eos_bos_chars=config.enable_eos_bos_chars, # pylint: disable=unused-argument use_griffin_lim=True, do_trim_silence=False, ) @@ -443,7 +442,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): except: # pylint: disable=bare-except print(" !! Error creating Test Sentence -", idx) traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"]) + tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"]) tb_logger.tb_test_figures(global_step, test_figures) return keep_avg.avg_values @@ -453,34 +452,34 @@ def main(args): # pylint: disable=redefined-outer-name # pylint: disable=global-variable-undefined global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping # Audio processor - ap = AudioProcessor(**c.audio) - if "characters" in c.keys(): - symbols, phonemes = make_symbols(**c.characters) + ap = AudioProcessor(**config.audio.to_dict()) + if config.characters is not None: + symbols, phonemes = make_symbols(**config.characters.to_dict()) # DISTRUBUTED if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"]) + init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"]) # set model characters - model_characters = phonemes if c.use_phonemes else symbols + model_characters = phonemes if config.use_phonemes else symbols num_chars = len(model_characters) # load data instances - meta_data_train, meta_data_eval = load_meta_data(c.datasets, eval_split=True) + meta_data_train, meta_data_eval = load_meta_data(config.datasets, eval_split=True) # set the portion of the data used for training if set in config.json - if "train_portion" in c.keys(): - meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)] - if "eval_portion" in c.keys(): - meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)] + if config.has("train_portion"): + meta_data_train = meta_data_train[: int(len(meta_data_train) * config.train_portion)] + if config.has("eval_portion"): + meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * config.eval_portion)] # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH) + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) # setup model - model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim) - optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) - criterion = SpeedySpeechLoss(c) + model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim) + optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9) + criterion = SpeedySpeechLoss(config) if args.restore_path: print(f" > Restoring from {os.path.basename(args.restore_path)} ...") @@ -489,18 +488,18 @@ def main(args): # pylint: disable=redefined-outer-name # TODO: fix optimizer init, model.cuda() needs to be called before # optimizer restore optimizer.load_state_dict(checkpoint["optimizer"]) - if c.reinit_layers: + if config.reinit_layers: raise RuntimeError model.load_state_dict(checkpoint["model"]) except: # pylint: disable=bare-except print(" > Partial model initialization.") model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint["model"], c) + model_dict = set_init_dict(model_dict, checkpoint["model"], config) model.load_state_dict(model_dict) del model_dict for group in optimizer.param_groups: - group["initial_lr"] = c.lr + group["initial_lr"] = config.lr print(" > Model restored from step %d" % checkpoint["step"], flush=True) args.restore_step = checkpoint["step"] else: @@ -514,8 +513,8 @@ def main(args): # pylint: disable=redefined-outer-name if num_gpus > 1: model = DDP_th(model, device_ids=[args.rank]) - if c.noam_schedule: - scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1) + if config.noam_schedule: + scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1) else: scheduler = None @@ -529,23 +528,23 @@ def main(args): # pylint: disable=redefined-outer-name print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] print(f" > Starting with loaded last best loss {best_loss}.") - keep_all_best = c.get("keep_all_best", False) - keep_after = c.get("keep_after", 10000) # void if keep_all_best False + keep_all_best = config.keep_all_best + keep_after = config.keep_after # void if keep_all_best False # define dataloaders train_loader = setup_loader(ap, 1, is_val=False, verbose=True) eval_loader = setup_loader(ap, 1, is_val=True, verbose=True) global_step = args.restore_step - for epoch in range(0, c.epochs): - c_logger.print_epoch_start(epoch, c.epochs) + for epoch in range(0, config.epochs): + c_logger.print_epoch_start(epoch, config.epochs) train_avg_loss_dict, global_step = train( train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch ) eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) target_loss = train_avg_loss_dict["avg_loss"] - if c.run_eval: + if config.run_eval: target_loss = eval_avg_loss_dict["avg_loss"] best_loss = save_best_model( target_loss, @@ -554,7 +553,7 @@ def main(args): # pylint: disable=redefined-outer-name optimizer, global_step, epoch, - c.r, + config.r, OUT_PATH, model_characters, keep_all_best=keep_all_best, @@ -563,8 +562,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts") + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py old mode 100644 new mode 100755 index c8346c3a..9685d0d7 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -21,7 +21,7 @@ from TTS.tts.utils.speakers import parse_speakers from TTS.tts.utils.synthesis import synthesis from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols from TTS.tts.utils.visual import plot_alignment, plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import DistributedSampler, apply_gradient_allreduce, init_distributed, reduce_tensor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -39,45 +39,47 @@ use_cuda, num_gpus = setup_torch_training_env(True, False) def setup_loader(ap, r, is_val=False, verbose=False, dataset=None): - if is_val and not c.run_eval: + if is_val and not config.run_eval: loader = None else: if dataset is None: dataset = MyDataset( r, - c.text_cleaner, - compute_linear_spec=c.model.lower() == "tacotron", + config.text_cleaner, + compute_linear_spec=config.model.lower() == "tacotron", meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, - tp=c.characters if "characters" in c.keys() else None, - add_blank=c["add_blank"] if "add_blank" in c.keys() else False, - batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, - min_seq_len=c.min_seq_len, - max_seq_len=c.max_seq_len, - phoneme_cache_path=c.phoneme_cache_path, - use_phonemes=c.use_phonemes, - phoneme_language=c.phoneme_language, - enable_eos_bos=c.enable_eos_bos_chars, + tp=config.characters, + add_blank=config["add_blank"], + batch_group_size=0 if is_val else config.batch_group_size * config.batch_size, + min_seq_len=config.min_seq_len, + max_seq_len=config.max_seq_len, + phoneme_cache_path=config.phoneme_cache_path, + use_phonemes=config.use_phonemes, + phoneme_language=config.phoneme_language, + enable_eos_bos=config.enable_eos_bos_chars, verbose=verbose, speaker_mapping=( - speaker_mapping if (c.use_speaker_embedding and c.use_external_speaker_embedding_file) else None + speaker_mapping + if (config.use_speaker_embedding and config.use_external_speaker_embedding_file) + else None ), ) - if c.use_phonemes and c.compute_input_seq_cache: + if config.use_phonemes and config.compute_input_seq_cache: # precompute phonemes to have a better estimate of sequence lengths. - dataset.compute_input_seq(c.num_loader_workers) + dataset.compute_input_seq(config.num_loader_workers) dataset.sort_items() sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader( dataset, - batch_size=c.eval_batch_size if is_val else c.batch_size, + batch_size=config.eval_batch_size if is_val else config.batch_size, shuffle=False, collate_fn=dataset.collate_fn, drop_last=False, sampler=sampler, - num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers, + num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers, pin_memory=False, ) return loader @@ -88,15 +90,15 @@ def format_data(data): text_input = data[0] text_lengths = data[1] speaker_names = data[2] - linear_input = data[3] if c.model.lower() in ["tacotron"] else None + linear_input = data[3] if config.model.lower() in ["tacotron"] else None mel_input = data[4] mel_lengths = data[5] stop_targets = data[6] max_text_length = torch.max(text_lengths.float()) max_spec_length = torch.max(mel_lengths.float()) - if c.use_speaker_embedding: - if c.use_external_speaker_embedding_file: + if config.use_speaker_embedding: + if config.use_external_speaker_embedding_file: speaker_embeddings = data[8] speaker_ids = None else: @@ -108,7 +110,7 @@ def format_data(data): speaker_ids = None # set stop targets view, we predict a single stop token per iteration. - stop_targets = stop_targets.view(text_input.shape[0], stop_targets.size(1) // c.r, -1) + stop_targets = stop_targets.view(text_input.shape[0], stop_targets.size(1) // config.r, -1) stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float().squeeze(2) # dispatch data to GPU @@ -117,7 +119,7 @@ def format_data(data): text_lengths = text_lengths.cuda(non_blocking=True) mel_input = mel_input.cuda(non_blocking=True) mel_lengths = mel_lengths.cuda(non_blocking=True) - linear_input = linear_input.cuda(non_blocking=True) if c.model in ["Tacotron"] else None + linear_input = linear_input.cuda(non_blocking=True) if config.model.lower() in ["tacotron"] else None stop_targets = stop_targets.cuda(non_blocking=True) if speaker_ids is not None: speaker_ids = speaker_ids.cuda(non_blocking=True) @@ -143,9 +145,9 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, epoch_time = 0 keep_avg = KeepAverage() if use_cuda: - batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus)) + batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus)) else: - batch_n_iter = int(len(data_loader.dataset) / c.batch_size) + batch_n_iter = int(len(data_loader.dataset) / config.batch_size) end_time = time.time() c_logger.print_train_start() for num_iter, data in enumerate(data_loader): @@ -169,16 +171,16 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, global_step += 1 # setup lr - if c.noam_schedule: + if config.noam_schedule: scheduler.step() optimizer.zero_grad() if optimizer_st: optimizer_st.zero_grad() - with torch.cuda.amp.autocast(enabled=c.mixed_precision): + with torch.cuda.amp.autocast(enabled=config.mixed_precision): # forward pass model - if c.bidirectional_decoder or c.double_decoder_consistency: + if config.bidirectional_decoder or config.double_decoder_consistency: ( decoder_output, postnet_output, @@ -235,17 +237,17 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, raise RuntimeError(f"Detected NaN loss at step {global_step}.") # optimizer step - if c.mixed_precision: + if config.mixed_precision: # model optimizer step in mixed precision mode scaler.scale(loss_dict["loss"]).backward() scaler.unscale_(optimizer) optimizer, current_lr = adam_weight_decay(optimizer) - grad_norm, _ = check_update(model, c.grad_clip, ignore_stopnet=True) + grad_norm, _ = check_update(model, config.grad_clip, ignore_stopnet=True) scaler.step(optimizer) scaler.update() # stopnet optimizer step - if c.separate_stopnet: + if config.separate_stopnet: scaler_st.scale(loss_dict["stopnet_loss"]).backward() scaler.unscale_(optimizer_st) optimizer_st, _ = adam_weight_decay(optimizer_st) @@ -258,11 +260,11 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, # main model optimizer step loss_dict["loss"].backward() optimizer, current_lr = adam_weight_decay(optimizer) - grad_norm, _ = check_update(model, c.grad_clip, ignore_stopnet=True) + grad_norm, _ = check_update(model, config.grad_clip, ignore_stopnet=True) optimizer.step() # stopnet optimizer step - if c.separate_stopnet: + if config.separate_stopnet: loss_dict["stopnet_loss"].backward() optimizer_st, _ = adam_weight_decay(optimizer_st) grad_norm_st, _ = check_update(model.decoder.stopnet, 1.0) @@ -283,7 +285,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, loss_dict["decoder_loss"] = reduce_tensor(loss_dict["decoder_loss"].data, num_gpus) loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus) loss_dict["stopnet_loss"] = ( - reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) if c.stopnet else loss_dict["stopnet_loss"] + reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) if config.stopnet else loss_dict["stopnet_loss"] ) # detach loss values @@ -304,7 +306,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, keep_avg.update_values(update_train_values) # print training progress - if global_step % c.print_step == 0: + if global_step % config.print_step == 0: log_dict = { "max_spec_length": [max_spec_length, 1], # value, precision "max_text_length": [max_text_length, 1], @@ -317,7 +319,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, if args.rank == 0: # Plot Training Iter Stats # reduce TB load - if global_step % c.tb_plot_step == 0: + if global_step % config.tb_plot_step == 0: iter_stats = { "lr": current_lr, "grad_norm": grad_norm, @@ -327,8 +329,8 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, iter_stats.update(loss_dict) tb_logger.tb_train_iter_stats(global_step, iter_stats) - if global_step % c.save_step == 0: - if c.checkpoint: + if global_step % config.save_step == 0: + if config.checkpoint: # save model save_checkpoint( model, @@ -340,14 +342,14 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, optimizer_st=optimizer_st, model_loss=loss_dict["postnet_loss"], characters=model_characters, - scaler=scaler.state_dict() if c.mixed_precision else None, + scaler=scaler.state_dict() if config.mixed_precision else None, ) # Diagnostic visualizations const_spec = postnet_output[0].data.cpu().numpy() gt_spec = ( linear_input[0].data.cpu().numpy() - if c.model in ["Tacotron", "TacotronGST"] + if config.model in ["Tacotron", "TacotronGST"] else mel_input[0].data.cpu().numpy() ) align_img = alignments[0].data.cpu().numpy() @@ -358,7 +360,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, "alignment": plot_alignment(align_img, output_fig=False), } - if c.bidirectional_decoder or c.double_decoder_consistency: + if config.bidirectional_decoder or config.double_decoder_consistency: figures["alignment_backward"] = plot_alignment( alignments_backward[0].data.cpu().numpy(), output_fig=False ) @@ -366,11 +368,11 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, tb_logger.tb_train_figures(global_step, figures) # Sample audio - if c.model in ["Tacotron", "TacotronGST"]: + if config.model in ["Tacotron", "TacotronGST"]: train_audio = ap.inv_spectrogram(const_spec.T) else: train_audio = ap.inv_melspectrogram(const_spec.T) - tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"]) + tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"]) end_time = time.time() # print epoch stats @@ -381,7 +383,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap, epoch_stats = {"epoch_time": epoch_time} epoch_stats.update(keep_avg.avg_values) tb_logger.tb_train_epoch_stats(global_step, epoch_stats) - if c.tb_model_param_stats: + if config.tb_model_param_stats: tb_logger.tb_model_weights(model, global_step) return keep_avg.avg_values, global_step @@ -412,7 +414,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): assert mel_input.shape[1] % model.decoder.r == 0 # forward pass model - if c.bidirectional_decoder or c.double_decoder_consistency: + if config.bidirectional_decoder or config.double_decoder_consistency: ( decoder_output, postnet_output, @@ -466,7 +468,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): if num_gpus > 1: loss_dict["postnet_loss"] = reduce_tensor(loss_dict["postnet_loss"].data, num_gpus) loss_dict["decoder_loss"] = reduce_tensor(loss_dict["decoder_loss"].data, num_gpus) - if c.stopnet: + if config.stopnet: loss_dict["stopnet_loss"] = reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) # detach loss values @@ -484,7 +486,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): update_train_values["avg_" + key] = value keep_avg.update_values(update_train_values) - if c.print_eval: + if config.print_eval: c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) if args.rank == 0: @@ -493,7 +495,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): const_spec = postnet_output[idx].data.cpu().numpy() gt_spec = ( linear_input[idx].data.cpu().numpy() - if c.model in ["Tacotron", "TacotronGST"] + if config.model in ["Tacotron", "TacotronGST"] else mel_input[idx].data.cpu().numpy() ) align_img = alignments[idx].data.cpu().numpy() @@ -505,22 +507,25 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): } # Sample audio - if c.model in ["Tacotron", "TacotronGST"]: + if config.model.lower() in ["tacotron"]: eval_audio = ap.inv_spectrogram(const_spec.T) else: eval_audio = ap.inv_melspectrogram(const_spec.T) - tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"]) + tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"]) # Plot Validation Stats - if c.bidirectional_decoder or c.double_decoder_consistency: + if config.bidirectional_decoder or config.double_decoder_consistency: align_b_img = alignments_backward[idx].data.cpu().numpy() eval_figures["alignment2"] = plot_alignment(align_b_img, output_fig=False) tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) tb_logger.tb_eval_figures(global_step, eval_figures) - if args.rank == 0 and epoch > c.test_delay_epochs: - if c.test_sentences_file is None: + if args.rank == 0 and epoch > config.test_delay_epochs: + if config.test_sentences_file: + with open(config.test_sentences_file, "r") as f: + test_sentences = [s.strip() for s in f.readlines()] + else: test_sentences = [ "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", "Be a voice, not an echo.", @@ -528,41 +533,37 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): "This cake is great. It's so delicious and moist.", "Prior to November 22, 1963.", ] - else: - with open(c.test_sentences_file, "r") as f: - test_sentences = [s.strip() for s in f.readlines()] # test sentences test_audios = {} test_figures = {} print(" | > Synthesizing test sentences") - speaker_id = 0 if c.use_speaker_embedding else None + speaker_id = 0 if config.use_speaker_embedding else None speaker_embedding = ( speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]]["embedding"] - if c.use_external_speaker_embedding_file and c.use_speaker_embedding + if config.use_external_speaker_embedding_file and config.use_speaker_embedding else None ) - style_wav = c.get("gst_style_input") - if style_wav is None and c.use_gst: + style_wav = config.gst_style_input + if style_wav is None and config.gst is not None: # inicialize GST with zero dict. style_wav = {} print("WARNING: You don't provided a gst style wav, for this reason we use a zero tensor!") - for i in range(c.gst["gst_style_tokens"]): + for i in range(config.gst["gst_num_style_tokens"]): style_wav[str(i)] = 0 - style_wav = c.get("gst_style_input") for idx, test_sentence in enumerate(test_sentences): try: wav, alignment, decoder_output, postnet_output, stop_tokens, _ = synthesis( model, test_sentence, - c, + config, use_cuda, ap, speaker_id=speaker_id, speaker_embedding=speaker_embedding, style_wav=style_wav, truncated=False, - enable_eos_bos_chars=c.enable_eos_bos_chars, # pylint: disable=unused-argument + enable_eos_bos_chars=config.enable_eos_bos_chars, # pylint: disable=unused-argument use_griffin_lim=True, do_trim_silence=False, ) @@ -577,7 +578,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch): except: # pylint: disable=bare-except print(" !! Error creating Test Sentence -", idx) traceback.print_exc() - tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"]) + tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"]) tb_logger.tb_test_figures(global_step, test_figures) return keep_avg.avg_values @@ -586,45 +587,45 @@ def main(args): # pylint: disable=redefined-outer-name # pylint: disable=global-variable-undefined global meta_data_train, meta_data_eval, speaker_mapping, symbols, phonemes, model_characters # Audio processor - ap = AudioProcessor(**c.audio) + ap = AudioProcessor(**config.audio.to_dict()) # setup custom characters if set in config file. - if "characters" in c.keys(): - symbols, phonemes = make_symbols(**c.characters) + if config.characters is not None: + symbols, phonemes = make_symbols(**config.characters.to_dict()) # DISTRUBUTED if num_gpus > 1: - init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"]) - num_chars = len(phonemes) if c.use_phonemes else len(symbols) - model_characters = phonemes if c.use_phonemes else symbols + init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"]) + num_chars = len(phonemes) if config.use_phonemes else len(symbols) + model_characters = phonemes if config.use_phonemes else symbols # load data instances - meta_data_train, meta_data_eval = load_meta_data(c.datasets) + meta_data_train, meta_data_eval = load_meta_data(config.datasets) # set the portion of the data used for training - if "train_portion" in c.keys(): - meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)] - if "eval_portion" in c.keys(): - meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)] + if config.has("train_portion"): + meta_data_train = meta_data_train[: int(len(meta_data_train) * config.train_portion)] + if config.has("eval_portion"): + meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * config.eval_portion)] # parse speakers - num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH) + num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH) - model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim) + model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim) # scalers for mixed precision training - scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None - scaler_st = torch.cuda.amp.GradScaler() if c.mixed_precision and c.separate_stopnet else None + scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None + scaler_st = torch.cuda.amp.GradScaler() if config.mixed_precision and config.separate_stopnet else None - params = set_weight_decay(model, c.wd) - optimizer = RAdam(params, lr=c.lr, weight_decay=0) - if c.stopnet and c.separate_stopnet: - optimizer_st = RAdam(model.decoder.stopnet.parameters(), lr=c.lr, weight_decay=0) + params = set_weight_decay(model, config.wd) + optimizer = RAdam(params, lr=config.lr, weight_decay=0) + if config.stopnet and config.separate_stopnet: + optimizer_st = RAdam(model.decoder.stopnet.parameters(), lr=config.lr, weight_decay=0) else: optimizer_st = None # setup criterion - criterion = TacotronLoss(c, stopnet_pos_weight=c.stopnet_pos_weight, ga_sigma=0.4) + criterion = TacotronLoss(config, stopnet_pos_weight=config.stopnet_pos_weight, ga_sigma=0.4) if args.restore_path: print(f" > Restoring from {os.path.basename(args.restore_path)}...") checkpoint = torch.load(args.restore_path, map_location="cpu") @@ -634,22 +635,18 @@ def main(args): # pylint: disable=redefined-outer-name # optimizer restore print(" > Restoring Optimizer...") optimizer.load_state_dict(checkpoint["optimizer"]) - if "scaler" in checkpoint and c.mixed_precision: + if "scaler" in checkpoint and config.mixed_precision: print(" > Restoring AMP Scaler...") scaler.load_state_dict(checkpoint["scaler"]) - if c.reinit_layers: - raise RuntimeError except (KeyError, RuntimeError): print(" > Partial model initialization...") model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint["model"], c) - # torch.save(model_dict, os.path.join(OUT_PATH, 'state_dict.pt')) - # print("State Dict saved for debug in: ", os.path.join(OUT_PATH, 'state_dict.pt')) + model_dict = set_init_dict(model_dict, checkpoint["model"], config) model.load_state_dict(model_dict) del model_dict for group in optimizer.param_groups: - group["lr"] = c.lr + group["lr"] = config.lr print(" > Model restored from step %d" % checkpoint["step"], flush=True) args.restore_step = checkpoint["step"] else: @@ -663,8 +660,8 @@ def main(args): # pylint: disable=redefined-outer-name if num_gpus > 1: model = apply_gradient_allreduce(model) - if c.noam_schedule: - scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1) + if config.noam_schedule: + scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1) else: scheduler = None @@ -678,22 +675,22 @@ def main(args): # pylint: disable=redefined-outer-name print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"] print(f" > Starting with loaded last best loss {best_loss}.") - keep_all_best = c.get("keep_all_best", False) - keep_after = c.get("keep_after", 10000) # void if keep_all_best False + keep_all_best = config.keep_all_best + keep_after = config.keep_after # void if keep_all_best False # define data loaders train_loader = setup_loader(ap, model.decoder.r, is_val=False, verbose=True) eval_loader = setup_loader(ap, model.decoder.r, is_val=True) global_step = args.restore_step - for epoch in range(0, c.epochs): - c_logger.print_epoch_start(epoch, c.epochs) + for epoch in range(0, config.epochs): + c_logger.print_epoch_start(epoch, config.epochs) # set gradual training - if c.gradual_training is not None: - r, c.batch_size = gradual_training_scheduler(global_step, c) - c.r = r + if config.gradual_training is not None: + r, config.batch_size = gradual_training_scheduler(global_step, config) + config.r = r model.decoder.set_r(r) - if c.bidirectional_decoder: + if config.bidirectional_decoder: model.decoder_backward.set_r(r) train_loader.dataset.outputs_per_step = r eval_loader.dataset.outputs_per_step = r @@ -718,7 +715,7 @@ def main(args): # pylint: disable=redefined-outer-name eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) target_loss = train_avg_loss_dict["avg_postnet_loss"] - if c.run_eval: + if config.run_eval: target_loss = eval_avg_loss_dict["avg_postnet_loss"] best_loss = save_best_model( target_loss, @@ -727,19 +724,17 @@ def main(args): # pylint: disable=redefined-outer-name optimizer, global_step, epoch, - c.r, + config.r, OUT_PATH, model_characters, keep_all_best=keep_all_best, keep_after=keep_after, - scaler=scaler.state_dict() if c.mixed_precision else None, + scaler=scaler.state_dict() if config.mixed_precision else None, ) if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts") - + args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) except KeyboardInterrupt: diff --git a/TTS/bin/train_vocoder_gan.py b/TTS/bin/train_vocoder_gan.py old mode 100644 new mode 100755 index 730506c1..123d5a43 --- a/TTS/bin/train_vocoder_gan.py +++ b/TTS/bin/train_vocoder_gan.py @@ -2,6 +2,7 @@ # TODO: mixed precision training """Trains GAN based vocoder model.""" +import itertools import os import sys import time @@ -15,7 +16,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP_th from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -162,7 +163,6 @@ def train( y_hat_sub=y_hat_sub, y_sub=y_G_sub, ) - loss_G = loss_G_dict["G_loss"] # optimizer generator @@ -468,7 +468,7 @@ def main(args): # pylint: disable=redefined-outer-name eval_data, train_data = load_wav_data(c.data_path, c.eval_split_size) # setup audio processor - ap = AudioProcessor(**c.audio) + ap = AudioProcessor(**c.audio.to_dict()) # DISTRUBUTED if num_gpus > 1: @@ -495,7 +495,15 @@ def main(args): # pylint: disable=redefined-outer-name optimizer_gen = getattr(torch.optim, c.optimizer) optimizer_gen = optimizer_gen(model_gen.parameters(), lr=c.lr_gen, **c.optimizer_params) optimizer_disc = getattr(torch.optim, c.optimizer) - optimizer_disc = optimizer_disc(model_disc.parameters(), lr=c.lr_disc, **c.optimizer_params) + + if c.discriminator_model == "hifigan_discriminator": + optimizer_disc = optimizer_disc( + itertools.chain(model_disc.msd.parameters(), model_disc.mpd.parameters()), + lr=c.lr_disc, + **c.optimizer_params, + ) + else: + optimizer_disc = optimizer_disc(model_disc.parameters(), lr=c.lr_disc, **c.optimizer_params) # schedulers scheduler_gen = None @@ -615,13 +623,15 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder") - + args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) except KeyboardInterrupt: remove_experiment_folder(OUT_PATH) + try: + sys.exit(0) + except SystemExit: + os._exit(0) # pylint: disable=protected-access except Exception: # pylint: disable=broad-except remove_experiment_folder(OUT_PATH) traceback.print_exc() diff --git a/TTS/bin/train_vocoder_wavegrad.py b/TTS/bin/train_vocoder_wavegrad.py index 1f039a67..c0fcff51 100644 --- a/TTS/bin/train_vocoder_wavegrad.py +++ b/TTS/bin/train_vocoder_wavegrad.py @@ -15,7 +15,7 @@ from torch.optim import Adam from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.distribute import init_distributed from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict @@ -131,12 +131,12 @@ def train(model, criterion, optimizer, scheduler, scaler, ap, global_step, epoch if c.mixed_precision: scaler.scale(loss).backward() scaler.unscale_(optimizer) - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.clip_grad) + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip) scaler.step(optimizer) scaler.update() else: loss.backward() - grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.clip_grad) + grad_norm = torch.nn.utils.grad_clip_norm_(model.parameters(), c.clip_grad) optimizer.step() # schedule update @@ -311,7 +311,7 @@ def main(args): # pylint: disable=redefined-outer-name eval_data, train_data = load_wav_data(c.data_path, c.eval_split_size) # setup audio processor - ap = AudioProcessor(**c.audio) + ap = AudioProcessor(**c.audio.to_dict()) # DISTRUBUTED if num_gpus > 1: @@ -416,9 +416,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder") - + args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) except KeyboardInterrupt: diff --git a/TTS/bin/train_vocoder_wavernn.py b/TTS/bin/train_vocoder_wavernn.py index 8a2d8d3a..bcad9493 100644 --- a/TTS/bin/train_vocoder_wavernn.py +++ b/TTS/bin/train_vocoder_wavernn.py @@ -11,7 +11,7 @@ import torch from torch.utils.data import DataLoader from TTS.tts.utils.visual import plot_spectrogram -from TTS.utils.arguments import parse_arguments, process_args +from TTS.utils.arguments import init_training from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict from TTS.utils.radam import RAdam @@ -307,29 +307,7 @@ def main(args): # pylint: disable=redefined-outer-name global train_data, eval_data # setup audio processor - ap = AudioProcessor(**c.audio) - - # print(f" > Loading wavs from: {c.data_path}") - # if c.feature_path is not None: - # print(f" > Loading features from: {c.feature_path}") - # eval_data, train_data = load_wav_feat_data( - # c.data_path, c.feature_path, c.eval_split_size - # ) - # else: - # mel_feat_path = os.path.join(OUT_PATH, "mel") - # feat_data = find_feat_files(mel_feat_path) - # if feat_data: - # print(f" > Loading features from: {mel_feat_path}") - # eval_data, train_data = load_wav_feat_data( - # c.data_path, mel_feat_path, c.eval_split_size - # ) - # else: - # print(" > No feature data found. Preprocessing...") - # # preprocessing feature data from given wav files - # preprocess_wav_files(OUT_PATH, CONFIG, ap) - # eval_data, train_data = load_wav_feat_data( - # c.data_path, mel_feat_path, c.eval_split_size - # ) + ap = AudioProcessor(**c.audio.to_dict()) print(f" > Loading wavs from: {c.data_path}") if c.feature_path is not None: @@ -438,9 +416,7 @@ def main(args): # pylint: disable=redefined-outer-name if __name__ == "__main__": - args = parse_arguments(sys.argv) - c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder") - + args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv) try: main(args) except KeyboardInterrupt: diff --git a/TTS/config/__init__.py b/TTS/config/__init__.py new file mode 100644 index 00000000..b4f1cbea --- /dev/null +++ b/TTS/config/__init__.py @@ -0,0 +1,75 @@ +import json +import os +import re + +import yaml + +from TTS.config.shared_configs import * +from TTS.utils.generic_utils import find_module + + +def read_json_with_comments(json_path): + """for backward compat.""" + # fallback to json + with open(json_path, "r", encoding="utf-8") as f: + input_str = f.read() + # handle comments + input_str = re.sub(r"\\\n", "", input_str) + input_str = re.sub(r"//.*\n", "\n", input_str) + data = json.loads(input_str) + return data + + +def _search_configs(model_name): + config_class = None + paths = ["TTS.tts.configs", "TTS.vocoder.configs", "TTS.speaker_encoder"] + for path in paths: + try: + config_class = find_module(path, model_name + "_config") + except ModuleNotFoundError: + pass + if config_class is None: + raise ModuleNotFoundError(f" [!] Config for {model_name} cannot be found.") + return config_class + + +def _process_model_name(config_dict): + model_name = config_dict["model"] if "model" in config_dict else config_dict["generator_model"] + model_name = model_name.replace("_generator", "").replace("_discriminator", "") + return model_name + + +def load_config(config_path: str) -> None: + """Import `json` or `yaml` files as TTS configs. First, load the input file as a `dict` and check the model name + to find the corresponding Config class. Then initialize the Config. + + Args: + config_path (str): path to the config file. + + Raises: + TypeError: given config file has an unknown type. + + Returns: + Coqpit: TTS config object. + """ + config_dict = {} + ext = os.path.splitext(config_path)[1] + if ext in (".yml", ".yaml"): + with open(config_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + elif ext == ".json": + try: + with open(config_path, "r", encoding="utf-8") as f: + input_str = f.read() + data = json.loads(input_str) + except json.decoder.JSONDecodeError: + # backwards compat. + data = read_json_with_comments(config_path) + else: + raise TypeError(f" [!] Unknown config file type {ext}") + config_dict.update(data) + model_name = _process_model_name(config_dict) + config_class = _search_configs(model_name.lower()) + config = config_class() + config.from_dict(config_dict) + return config diff --git a/TTS/config/shared_configs.py b/TTS/config/shared_configs.py new file mode 100644 index 00000000..69f1ee31 --- /dev/null +++ b/TTS/config/shared_configs.py @@ -0,0 +1,253 @@ +from dataclasses import asdict, dataclass +from typing import List + +from coqpit import MISSING, Coqpit, check_argument + + +@dataclass +class BaseAudioConfig(Coqpit): + """Base config to definge audio processing parameters. It is used to initialize + ```TTS.utils.audio.AudioProcessor.``` + + Args: + fft_size (int): + Number of STFT frequency levels aka.size of the linear spectogram frame. Defaults to 1024. + win_length (int): + Each frame of audio is windowed by window of length ```win_length``` and then padded with zeros to match + ```fft_size```. Defaults to 1024. + hop_length (int): + Number of audio samples between adjacent STFT columns. Defaults to 1024. + frame_shift_ms (int): + Set ```hop_length``` based on milliseconds and sampling rate. + frame_length_ms (int): + Set ```win_length``` based on milliseconds and sampling rate. + stft_pad_mode (str): + Padding method used in STFT. 'reflect' or 'center'. Defaults to 'reflect'. + sample_rate (int): + Audio sampling rate. Defaults to 22050. + resample (bool): + Enable / Disable resampling audio to ```sample_rate```. Defaults to ```False```. + preemphasis (float): + Preemphasis coefficient. Defaults to 0.0. + ref_level_db (int): 20 + Reference Db level to rebase the audio signal and ignore the level below. 20Db is assumed the sound of air. + Defaults to 20. + do_sound_norm (bool): + Enable / Disable sound normalization to reconcile the volume differences among samples. Defaults to False. + do_trim_silence (bool): + Enable / Disable trimming silences at the beginning and the end of the audio clip. Defaults to ```True```. + trim_db (int): + Silence threshold used for silence trimming. Defaults to 45. + power (float): + Exponent used for expanding spectrogra levels before running Griffin Lim. It helps to reduce the + artifacts in the synthesized voice. Defaults to 1.5. + griffin_lim_iters (int): + Number of Griffing Lim iterations. Defaults to 60. + num_mels (int): + Number of mel-basis frames that defines the frame lengths of each mel-spectrogram frame. Defaults to 80. + mel_fmin (float): Min frequency level used for the mel-basis filters. ~50 for male and ~95 for female voices. + It needs to be adjusted for a dataset. Defaults to 0. + mel_fmax (float): + Max frequency level used for the mel-basis filters. It needs to be adjusted for a dataset. + spec_gain (int): + Gain applied when converting amplitude to DB. Defaults to 20. + signal_norm (bool): + enable/disable signal normalization. Defaults to True. + min_level_db (int): + minimum db threshold for the computed melspectrograms. Defaults to -100. + symmetric_norm (bool): + enable/disable symmetric normalization. If set True normalization is performed in the range [-k, k] else + [0, k], Defaults to True. + max_norm (float): + ```k``` defining the normalization range. Defaults to 4.0. + clip_norm (bool): + enable/disable clipping the our of range values in the normalized audio signal. Defaults to True. + stats_path (str): + Path to the computed stats file. Defaults to None. + """ + + # stft parameters + fft_size: int = 1024 + win_length: int = 1024 + hop_length: int = 256 + frame_shift_ms: int = None + frame_length_ms: int = None + stft_pad_mode: str = "reflect" + # audio processing parameters + sample_rate: int = 22050 + resample: bool = False + preemphasis: float = 0.0 + ref_level_db: int = 20 + do_sound_norm: bool = False + log_func = "np.log10" + # silence trimming + do_trim_silence: bool = True + trim_db: int = 45 + # griffin-lim params + power: float = 1.5 + griffin_lim_iters: int = 60 + # mel-spec params + num_mels: int = 80 + mel_fmin: float = 0.0 + mel_fmax: float = None + spec_gain: int = 20 + # normalization params + signal_norm: bool = True + min_level_db: int = -100 + symmetric_norm: bool = True + max_norm: float = 4.0 + clip_norm: bool = True + stats_path: str = None + + def check_values( + self, + ): + """Check config fields""" + c = asdict(self) + check_argument("num_mels", c, restricted=True, min_val=10, max_val=2056) + check_argument("fft_size", c, restricted=True, min_val=128, max_val=4058) + check_argument("sample_rate", c, restricted=True, min_val=512, max_val=100000) + check_argument( + "frame_length_ms", + c, + restricted=True, + min_val=10, + max_val=1000, + alternative="win_length", + ) + check_argument("frame_shift_ms", c, restricted=True, min_val=1, max_val=1000, alternative="hop_length") + check_argument("preemphasis", c, restricted=True, min_val=0, max_val=1) + check_argument("min_level_db", c, restricted=True, min_val=-1000, max_val=10) + check_argument("ref_level_db", c, restricted=True, min_val=0, max_val=1000) + check_argument("power", c, restricted=True, min_val=1, max_val=5) + check_argument("griffin_lim_iters", c, restricted=True, min_val=10, max_val=1000) + + # normalization parameters + check_argument("signal_norm", c, restricted=True) + check_argument("symmetric_norm", c, restricted=True) + check_argument("max_norm", c, restricted=True, min_val=0.1, max_val=1000) + check_argument("clip_norm", c, restricted=True) + check_argument("mel_fmin", c, restricted=True, min_val=0.0, max_val=1000) + check_argument("mel_fmax", c, restricted=True, min_val=500.0, allow_none=True) + check_argument("spec_gain", c, restricted=True, min_val=1, max_val=100) + check_argument("do_trim_silence", c, restricted=True) + check_argument("trim_db", c, restricted=True) + + +@dataclass +class BaseDatasetConfig(Coqpit): + """Base config for TTS datasets. + + Args: + name (str): + Dataset name that defines the preprocessor in use. Defaults to None. + path (str): + Root path to the dataset files. Defaults to None. + meta_file_train (str): + Name of the dataset meta file. Or a list of speakers to be ignored at training for multi-speaker datasets. + Defaults to None. + unused_speakers (List): + List of speakers IDs that are not used at the training. Default None. + meta_file_val (str): + Name of the dataset meta file that defines the instances used at validation. + meta_file_attn_mask (str): + Path to the file that lists the attention mask files used with models that require attention masks to + train the duration predictor. + """ + + name: str = "" + path: str = "" + meta_file_train: str = "" + ununsed_speakers: List[str] = None + meta_file_val: str = "" + meta_file_attn_mask: str = "" + + def check_values( + self, + ): + """Check config fields""" + c = asdict(self) + check_argument("name", c, restricted=True) + check_argument("path", c, restricted=True) + check_argument("meta_file_train", c, restricted=True) + check_argument("meta_file_val", c, restricted=False) + check_argument("meta_file_attn_mask", c, restricted=False) + + +@dataclass +class BaseTrainingConfig(Coqpit): + """Base config to define the basic training parameters that are shared + among all the models. + + Args: + batch_size (int): + Training batch size. + eval_batch_size (int): + Validation batch size. + mixed_precision (bool): + Enable / Disable mixed precision training. It reduces the VRAM use and allows larger batch sizes, however + it may also cause numerical unstability in some cases. + run_eval (bool): + Enable / Disable evaluation (validation) run. Defaults to True. + test_delay_epochs (int): + Number of epochs before starting to use evaluation runs. Initially, models do not generate meaningful + results, hence waiting for a couple of epochs might save some time. + print_eval (bool): + Enable / Disable console logging for evalutaion steps. If disabled then it only shows the final values at + the end of the evaluation. Default to ```False```. + print_step (int): + Number of steps required to print the next training log. + tb_plot_step (int): + Number of steps required to log training on Tensorboard. + tb_model_param_stats (bool): + Enable / Disable logging internal model stats for model diagnostic. It might be useful for model debugging. + Defaults to ```False```. + save_step (int):ipt + Number of steps required to save the next checkpoint. + checkpoint (bool): + Enable / Disable checkpointing. + keep_all_best (bool): + Enable / Disable keeping all the saved best models instead of overwriting the previous one. Defaults + to ```False```. + keep_after (int): + Number of steps to wait before saving all the best models. In use if ```keep_all_best == True```. Defaults + to 10000. + num_loader_workers (int): + Number of workers for training time dataloader. + num_val_loader_workers (int): + Number of workers for evaluation time dataloader. + output_path (str): + Path for training output folder. The nonexist part of the given path is created automatically. + All training outputs are saved there. + """ + + model: str = None + run_name: str = "" + run_description: str = "" + # training params + epochs: int = 10000 + batch_size: int = MISSING + eval_batch_size: int = None + mixed_precision: bool = False + # eval params + run_eval: bool = True + test_delay_epochs: int = 0 + print_eval: bool = False + # logging + print_step: int = 25 + tb_plot_step: int = 100 + tb_model_param_stats: bool = False + # checkpointing + save_step: int = 10000 + checkpoint: bool = True + keep_all_best: bool = False + keep_after: int = 10000 + # dataloading + num_loader_workers: int = MISSING + num_val_loader_workers: int = 0 + use_noise_augment: bool = False + # paths + output_path: str = None + # distributed + distributed_backend: str = "nccl" + distributed_url: str = "tcp://localhost:54321" diff --git a/TTS/server/README.md b/TTS/server/README.md index 51cedc05..89ee21eb 100644 --- a/TTS/server/README.md +++ b/TTS/server/README.md @@ -22,42 +22,3 @@ Run the server with the official models on a GPU. Run the server with a custom models. ```python TTS/server/server.py --tts_checkpoint /path/to/tts/model.pth.tar --tts_config /path/to/tts/config.json --vocoder_checkpoint /path/to/vocoder/model.pth.tar --vocoder_config /path/to/vocoder/config.json``` - - - - - - diff --git a/TTS/server/server.py b/TTS/server/server.py index f6335c42..15a6b292 100644 --- a/TTS/server/server.py +++ b/TTS/server/server.py @@ -9,7 +9,7 @@ from typing import Union from flask import Flask, render_template, request, send_file -from TTS.utils.io import load_config +from TTS.config import load_config from TTS.utils.manage import ModelManager from TTS.utils.synthesizer import Synthesizer diff --git a/TTS/speaker_encoder/config.json b/TTS/speaker_encoder/config.json deleted file mode 100644 index 4fbd84cc..00000000 --- a/TTS/speaker_encoder/config.json +++ /dev/null @@ -1,103 +0,0 @@ - -{ - "run_name": "mueller91", - "run_description": "train speaker encoder with voxceleb1, voxceleb2 and libriSpeech ", - "audio":{ - // Audio processing parameters - "num_mels": 40, // size of the mel spec frame. - "fft_size": 400, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 16000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "win_length": 400, // stft window length in ms. - "hop_length": 160, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "min_level_db": -100, // normalization range - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - // Normalization parameters - "signal_norm": true, // normalize the spec values in range [0, 1] - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "do_trim_silence": true, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60 // threshold for timming silence. Set this according to your dataset. - }, - "reinit_layers": [], - "loss": "angleproto", // "ge2e" to use Generalized End-to-End loss and "angleproto" to use Angular Prototypical loss (new SOTA) - "grad_clip": 3.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_decay": false, // if true, Noam learning rate decaying is applied through training. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "steps_plot_stats": 10, // number of steps to plot embeddings. - "num_speakers_in_batch": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "num_utters_per_speaker": 10, // - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "wd": 0.000001, // Weight decay weight. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. - "print_step": 20, // Number of steps to log traning on console. - "output_path": "../../MozillaTTSOutput/checkpoints/voxceleb_librispeech/speaker_encoder/", // DATASET-RELATED: output path for all training outputs. - "model": { - "input_dim": 40, - "proj_dim": 256, - "lstm_dim": 768, - "num_lstm_layers": 3, - "use_lstm_with_projection": true - }, - "storage": { - "sample_from_storage_p": 0.66, // the probability with which we'll sample from the DataSet in-memory storage - "storage_size": 15, // the size of the in-memory storage with respect to a single batch - "additive_noise": 1e-5 // add very small gaussian noise to the data in order to increase robustness - }, - "datasets": - [ - { - "name": "vctk_slim", - "path": "../../../audio-datasets/en/VCTK-Corpus/", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "libri_tts", - "path": "../../../audio-datasets/en/LibriTTS/train-clean-100", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "libri_tts", - "path": "../../../audio-datasets/en/LibriTTS/train-clean-360", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "libri_tts", - "path": "../../../audio-datasets/en/LibriTTS/train-other-500", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "voxceleb1", - "path": "../../../audio-datasets/en/voxceleb1/", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "voxceleb2", - "path": "../../../audio-datasets/en/voxceleb2/", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "common_voice", - "path": "../../../audio-datasets/en/MozillaCommonVoice", - "meta_file_train": "train.tsv", - "meta_file_val": "test.tsv" - } - ] -} \ No newline at end of file diff --git a/TTS/speaker_encoder/losses.py b/TTS/speaker_encoder/losses.py index 69264ab4..1b6b6e43 100644 --- a/TTS/speaker_encoder/losses.py +++ b/TTS/speaker_encoder/losses.py @@ -23,7 +23,7 @@ class GE2ELoss(nn.Module): self.b = nn.Parameter(torch.tensor(init_b)) self.loss_method = loss_method - print(" > Initialised Generalized End-to-End loss") + print(" > Initialized Generalized End-to-End loss") assert self.loss_method in ["softmax", "contrast"] @@ -136,7 +136,7 @@ class AngleProtoLoss(nn.Module): self.b = nn.Parameter(torch.tensor(init_b)) self.criterion = torch.nn.CrossEntropyLoss() - print(" > Initialised Angular Prototypical loss") + print(" > Initialized Angular Prototypical loss") def forward(self, x): """ diff --git a/TTS/speaker_encoder/speaker_encoder_config.py b/TTS/speaker_encoder/speaker_encoder_config.py new file mode 100644 index 00000000..dcba3b6c --- /dev/null +++ b/TTS/speaker_encoder/speaker_encoder_config.py @@ -0,0 +1,62 @@ +from dataclasses import asdict, dataclass, field +from typing import List + +from coqpit import MISSING + +from TTS.config.shared_configs import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig + + +@dataclass +class SpeakerEncoderConfig(BaseTrainingConfig): + """Defines parameters for Speaker Encoder model.""" + + model: str = "speaker_encoder" + audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) + datasets: List[BaseDatasetConfig] = field(default_factory=lambda: [BaseDatasetConfig()]) + + # model params + model_params: dict = field( + default_factory=lambda: { + "input_dim": 40, + "proj_dim": 256, + "lstm_dim": 768, + "num_lstm_layers": 3, + "use_lstm_with_projection": True, + } + ) + + storage: dict = field( + default_factory=lambda: { + "sample_from_storage_p": 0.66, # the probability with which we'll sample from the DataSet in-memory storage + "storage_size": 15, # the size of the in-memory storage with respect to a single batch + "additive_noise": 1e-5, # add very small gaussian noise to the data in order to increase robustness + } + ) + + # training params + max_train_step: int = 1000 # end training when number of training steps reaches this value. + loss: str = "angleproto" + grad_clip: float = 3.0 + lr: float = 0.0001 + lr_decay: bool = False + warmup_steps: int = 4000 + wd: float = 1e-6 + + # logging params + tb_model_param_stats: bool = False + steps_plot_stats: int = 10 + checkpoint: bool = True + save_step: int = 1000 + print_step: int = 20 + + # data loader + num_speakers_in_batch: int = MISSING + num_utters_per_speaker: int = MISSING + num_loader_workers: int = MISSING + + def check_values(self): + super().check_values() + c = asdict(self) + assert ( + c["model_params"]["input_dim"] == self.audio.num_mels + ), " [!] model input dimendion must be equal to melspectrogram dimension." diff --git a/TTS/speaker_encoder/utils/generic_utils.py b/TTS/speaker_encoder/utils/generic_utils.py index c9bfa679..eecf9086 100644 --- a/TTS/speaker_encoder/utils/generic_utils.py +++ b/TTS/speaker_encoder/utils/generic_utils.py @@ -1,11 +1,6 @@ -import datetime -import os import re -import torch - from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.utils.generic_utils import check_argument def to_camel(text): @@ -14,110 +9,10 @@ def to_camel(text): def setup_model(c): - model = SpeakerEncoder(c.model["input_dim"], c.model["proj_dim"], c.model["lstm_dim"], c.model["num_lstm_layers"]) + model = SpeakerEncoder( + c.model_params["input_dim"], + c.model_params["proj_dim"], + c.model_params["lstm_dim"], + c.model_params["num_lstm_layers"], + ) return model - - -def save_checkpoint(model, optimizer, model_loss, out_path, current_step, epoch): - checkpoint_path = "checkpoint_{}.pth.tar".format(current_step) - checkpoint_path = os.path.join(out_path, checkpoint_path) - print(" | | > Checkpoint saving : {}".format(checkpoint_path)) - - new_state_dict = model.state_dict() - state = { - "model": new_state_dict, - "optimizer": optimizer.state_dict() if optimizer is not None else None, - "step": current_step, - "epoch": epoch, - "loss": model_loss, - "date": datetime.date.today().strftime("%B %d, %Y"), - } - torch.save(state, checkpoint_path) - - -def save_best_model(model, optimizer, model_loss, best_loss, out_path, current_step): - if model_loss < best_loss: - new_state_dict = model.state_dict() - state = { - "model": new_state_dict, - "optimizer": optimizer.state_dict(), - "step": current_step, - "loss": model_loss, - "date": datetime.date.today().strftime("%B %d, %Y"), - } - best_loss = model_loss - bestmodel_path = "best_model.pth.tar" - bestmodel_path = os.path.join(out_path, bestmodel_path) - print("\n > BEST MODEL ({0:.5f}) : {1:}".format(model_loss, bestmodel_path)) - torch.save(state, bestmodel_path) - return best_loss - - -def check_config_speaker_encoder(c): - """Check the config.json file of the speaker encoder""" - check_argument("run_name", c, restricted=True, val_type=str) - check_argument("run_description", c, val_type=str) - - # audio processing parameters - check_argument("audio", c, restricted=True, val_type=dict) - check_argument("num_mels", c["audio"], restricted=True, val_type=int, min_val=10, max_val=2056) - check_argument("fft_size", c["audio"], restricted=True, val_type=int, min_val=128, max_val=4058) - check_argument("sample_rate", c["audio"], restricted=True, val_type=int, min_val=512, max_val=100000) - check_argument( - "frame_length_ms", - c["audio"], - restricted=True, - val_type=float, - min_val=10, - max_val=1000, - alternative="win_length", - ) - check_argument( - "frame_shift_ms", c["audio"], restricted=True, val_type=float, min_val=1, max_val=1000, alternative="hop_length" - ) - check_argument("preemphasis", c["audio"], restricted=True, val_type=float, min_val=0, max_val=1) - check_argument("min_level_db", c["audio"], restricted=True, val_type=int, min_val=-1000, max_val=10) - check_argument("ref_level_db", c["audio"], restricted=True, val_type=int, min_val=0, max_val=1000) - check_argument("power", c["audio"], restricted=True, val_type=float, min_val=1, max_val=5) - check_argument("griffin_lim_iters", c["audio"], restricted=True, val_type=int, min_val=10, max_val=1000) - - # training parameters - check_argument("loss", c, enum_list=["ge2e", "angleproto"], restricted=True, val_type=str) - check_argument("grad_clip", c, restricted=True, val_type=float) - check_argument("epochs", c, restricted=True, val_type=int, min_val=1) - check_argument("lr", c, restricted=True, val_type=float, min_val=0) - check_argument("lr_decay", c, restricted=True, val_type=bool) - check_argument("warmup_steps", c, restricted=True, val_type=int, min_val=0) - check_argument("tb_model_param_stats", c, restricted=True, val_type=bool) - check_argument("num_speakers_in_batch", c, restricted=True, val_type=int) - check_argument("num_loader_workers", c, restricted=True, val_type=int) - check_argument("wd", c, restricted=True, val_type=float, min_val=0.0, max_val=1.0) - - # checkpoint and output parameters - check_argument("steps_plot_stats", c, restricted=True, val_type=int) - check_argument("checkpoint", c, restricted=True, val_type=bool) - check_argument("save_step", c, restricted=True, val_type=int) - check_argument("print_step", c, restricted=True, val_type=int) - check_argument("output_path", c, restricted=True, val_type=str) - - # model parameters - check_argument("model", c, restricted=True, val_type=dict) - check_argument("input_dim", c["model"], restricted=True, val_type=int) - check_argument("proj_dim", c["model"], restricted=True, val_type=int) - check_argument("lstm_dim", c["model"], restricted=True, val_type=int) - check_argument("num_lstm_layers", c["model"], restricted=True, val_type=int) - check_argument("use_lstm_with_projection", c["model"], restricted=True, val_type=bool) - - # in-memory storage parameters - check_argument("storage", c, restricted=True, val_type=dict) - check_argument("sample_from_storage_p", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0) - check_argument("storage_size", c["storage"], restricted=True, val_type=int, min_val=1, max_val=100) - check_argument("additive_noise", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0) - - # datasets - checking only the first entry - check_argument("datasets", c, restricted=True, val_type=list) - for dataset_entry in c["datasets"]: - check_argument("name", dataset_entry, restricted=True, val_type=str) - check_argument("path", dataset_entry, restricted=True, val_type=str) - check_argument("meta_file_train", dataset_entry, restricted=True, val_type=[str, list]) - check_argument("meta_file_val", dataset_entry, restricted=True, val_type=str) diff --git a/TTS/speaker_encoder/utils/io.py b/TTS/speaker_encoder/utils/io.py index e69de29b..0479f1af 100644 --- a/TTS/speaker_encoder/utils/io.py +++ b/TTS/speaker_encoder/utils/io.py @@ -0,0 +1,38 @@ +import datetime +import os + +import torch + + +def save_checkpoint(model, optimizer, model_loss, out_path, current_step): + checkpoint_path = "checkpoint_{}.pth.tar".format(current_step) + checkpoint_path = os.path.join(out_path, checkpoint_path) + print(" | | > Checkpoint saving : {}".format(checkpoint_path)) + + new_state_dict = model.state_dict() + state = { + "model": new_state_dict, + "optimizer": optimizer.state_dict() if optimizer is not None else None, + "step": current_step, + "loss": model_loss, + "date": datetime.date.today().strftime("%B %d, %Y"), + } + torch.save(state, checkpoint_path) + + +def save_best_model(model, optimizer, model_loss, best_loss, out_path, current_step): + if model_loss < best_loss: + new_state_dict = model.state_dict() + state = { + "model": new_state_dict, + "optimizer": optimizer.state_dict(), + "step": current_step, + "loss": model_loss, + "date": datetime.date.today().strftime("%B %d, %Y"), + } + best_loss = model_loss + bestmodel_path = "best_model.pth.tar" + bestmodel_path = os.path.join(out_path, bestmodel_path) + print("\n > BEST MODEL ({0:.5f}) : {1:}".format(model_loss, bestmodel_path)) + torch.save(state, bestmodel_path) + return best_loss diff --git a/TTS/tts/configs/__init__.py b/TTS/tts/configs/__init__.py new file mode 100644 index 00000000..5ad4fe8c --- /dev/null +++ b/TTS/tts/configs/__init__.py @@ -0,0 +1,17 @@ +import importlib +import os +from inspect import isclass + +# import all files under configs/ +configs_dir = os.path.dirname(__file__) +for file in os.listdir(configs_dir): + path = os.path.join(configs_dir, file) + if not file.startswith("_") and not file.startswith(".") and (file.endswith(".py") or os.path.isdir(path)): + config_name = file[: file.find(".py")] if file.endswith(".py") else file + module = importlib.import_module("TTS.tts.configs." + config_name) + for attribute_name in dir(module): + attribute = getattr(module, attribute_name) + + if isclass(attribute): + # Add the class to this package's variables + globals()[attribute_name] = attribute diff --git a/TTS/tts/configs/align_tts_config.py b/TTS/tts/configs/align_tts_config.py new file mode 100644 index 00000000..2956d935 --- /dev/null +++ b/TTS/tts/configs/align_tts_config.py @@ -0,0 +1,104 @@ +from dataclasses import dataclass, field +from typing import List + +from TTS.tts.configs.shared_configs import BaseTTSConfig + + +@dataclass +class AlignTTSConfig(BaseTTSConfig): + """Defines parameters for AlignTTS model. + Example: + + >>> from TTS.tts.configs import AlignTTSConfig + >>> config = AlignTTSConfig() + + Args: + model(str): + Model name used for selecting the right model at initialization. Defaults to `align_tts`. + positional_encoding (bool): + enable / disable positional encoding applied to the encoder output. Defaults to True. + hidden_channels (int): + Base number of hidden channels. Defines all the layers expect ones defined by the specific encoder or decoder + parameters. Defaults to 256. + hidden_channels_dp (int): + Number of hidden channels of the duration predictor's layers. Defaults to 256. + encoder_type (str): + Type of the encoder used by the model. Look at `TTS.tts.layers.feed_forward.encoder` for more details. + Defaults to `fftransformer`. + encoder_params (dict): + Parameters used to define the encoder network. Look at `TTS.tts.layers.feed_forward.encoder` for more details. + Defaults to `{"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}`. + decoder_type (str): + Type of the decoder used by the model. Look at `TTS.tts.layers.feed_forward.decoder` for more details. + Defaults to `fftransformer`. + decoder_params (dict): + Parameters used to define the decoder network. Look at `TTS.tts.layers.feed_forward.decoder` for more details. + Defaults to `{"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}`. + phase_start_steps (List[int]): + A list of number of steps required to start the next training phase. AlignTTS has 4 different training + phases. Thus you need to define 4 different values to enable phase based training. If None, it + trains the whole model together. Defaults to None. + ssim_alpha (float): + Weight for the SSIM loss. If set <= 0, disables the SSIM loss. Defaults to 1.0. + duration_loss_alpha (float): + Weight for the duration predictor's loss. Defaults to 1.0. + mdn_alpha (float): + Weight for the MDN loss. Defaults to 1.0. + spec_loss_alpha (float): + Weight for the MSE spectrogram loss. If set <= 0, disables the L1 loss. Defaults to 1.0. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-3`. + wd (float): + Weight decay coefficient. Defaults to `1e-7`. + min_seq_len (int): + Minimum input sequence length to be used at training. + max_seq_len (int): + Maximum input sequence length to be used at training. Larger values result in more VRAM usage.""" + + model: str = "align_tts" + # model specific params + positional_encoding: bool = True + hidden_channels_dp: int = 256 + hidden_channels: int = 256 + encoder_type: str = "fftransformer" + encoder_params: dict = field( + default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1} + ) + decoder_type: str = "fftransformer" + decoder_params: dict = field( + default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1} + ) + phase_start_steps: List[int] = None + + ssim_alpha: float = 1.0 + spec_loss_alpha: float = 1.0 + dur_loss_alpha: float = 1.0 + mdn_alpha: float = 1.0 + + # multi-speaker settings + use_speaker_embedding: bool = False + use_external_speaker_embedding_file: bool = False + external_speaker_embedding_file: str = False + + # optimizer parameters + noam_schedule: bool = False + warmup_steps: int = 4000 + lr: float = 1e-4 + wd: float = 1e-6 + grad_clip: float = 5.0 + + # overrides + min_seq_len: int = 13 + max_seq_len: int = 200 + r: int = 1 diff --git a/TTS/tts/configs/config.json b/TTS/tts/configs/config.json deleted file mode 100644 index 4092a1b0..00000000 --- a/TTS/tts/configs/config.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "model": "Tacotron2", - "run_name": "ljspeech-ddc", - "run_description": "tacotron2 with DDC and differential spectral loss.", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. - "mixed_precision": true, // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate. - - // LOSS SETTINGS - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled - "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled - "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled - "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled - "ga_alpha": 5.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. - - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TACOTRON PRENET - "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. - "prenet_type": "original", // "original" or "bn". - "prenet_dropout": true, // enable/disable dropout at prenet. - - // TACOTRON ATTENTION - "attention_type": "original", // 'original' , 'graves', 'dynamic_convolution' - "attention_heads": 4, // number of attention heads (only for 'graves') - "attention_norm": "sigmoid", // softmax or sigmoid. - "windowing": false, // Enables attention windowing. Used only in eval mode. - "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. - "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. - "transition_agent": false, // enable/disable transition agent of forward attention. - "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. - "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. - "double_decoder_consistency": true, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ - "ddc_r": 7, // reduction rate for coarse decoder. - - // STOPNET - "stopnet": true, // Train stopnet predicting the end of synthesis. - "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 4, //Number of batches to shuffle after bucketing. - "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 153, // DATASET-RELATED: maximum text length - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - "use_noise_augment": true, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": false, // use global style tokens - "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_style_tokens). - "gst_embedding_dim": 512, - "gst_num_heads": 4, - "gst_style_tokens": 10, - "gst_use_speaker_embedding": false - }, - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers - "meta_file_val": null - } - ] -} diff --git a/TTS/tts/configs/glow_tts_config.py b/TTS/tts/configs/glow_tts_config.py new file mode 100644 index 00000000..36ccb612 --- /dev/null +++ b/TTS/tts/configs/glow_tts_config.py @@ -0,0 +1,102 @@ +from dataclasses import dataclass, field + +from TTS.tts.configs.shared_configs import BaseTTSConfig + + +@dataclass +class GlowTTSConfig(BaseTTSConfig): + """Defines parameters for GlowTTS model. + + Example: + + >>> from TTS.tts.configs import GlowTTSConfig + >>> config = GlowTTSConfig() + + Args: + model(str): + Model name used for selecting the right model at initialization. Defaults to `glow_tts`. + encoder_type (str): + Type of the encoder used by the model. Look at `TTS.tts.layers.glow_tts.encoder` for more details. + Defaults to `rel_pos_transformers`. + encoder_params (dict): + Parameters used to define the encoder network. Look at `TTS.tts.layers.glow_tts.encoder` for more details. + Defaults to `{"kernel_size": 3, "dropout_p": 0.1, "num_layers": 6, "num_heads": 2, "hidden_channels_ffn": 768}` + use_encoder_prenet (bool): + enable / disable the use of a prenet for the encoder. Defaults to True. + hidden_channels_encoder (int): + Number of base hidden channels used by the encoder network. It defines the input and the output channel sizes, + and for some encoder types internal hidden channels sizes too. Defaults to 192. + hidden_channels_decoder (int): + Number of base hidden channels used by the decoder WaveNet network. Defaults to 192 as in the original work. + hidden_channels_duration_predictor (int): + Number of layer channels of the duration predictor network. Defaults to 256 as in the original work. + data_dep_init_steps (int): + Number of steps used for computing normalization parameters at the beginning of the training. GlowTTS uses + Activation Normalization that pre-computes normalization stats at the beginning and use the same values + for the rest. Defaults to 10. + style_wav_for_test (str): + Path to the wav file used for changing the style of the speech. Defaults to None. + inference_noise_scale (float): + Variance used for sampling the random noise added to the decoder's input at inference. Defaults to 0.0. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-3`. + wd (float): + Weight decay coefficient. Defaults to `1e-7`. + min_seq_len (int): + Minimum input sequence length to be used at training. + max_seq_len (int): + Maximum input sequence length to be used at training. Larger values result in more VRAM usage. + """ + + model: str = "glow_tts" + + # model params + encoder_type: str = "rel_pos_transformer" + encoder_params: dict = field( + default_factory=lambda: { + "kernel_size": 3, + "dropout_p": 0.1, + "num_layers": 6, + "num_heads": 2, + "hidden_channels_ffn": 768, + } + ) + use_encoder_prenet: bool = True + hidden_channels_encoder: int = 192 + hidden_channels_decoder: int = 192 + hidden_channels_duration_predictor: int = 256 + + # training params + data_dep_init_steps: int = 10 + + # inference params + style_wav_for_test: str = None + inference_noise_scale: float = 0.0 + + # multi-speaker settings + use_speaker_embedding: bool = False + use_external_speaker_embedding_file: bool = False + external_speaker_embedding_file: str = False + + # optimizer params + noam_schedule: bool = True + warmup_steps: int = 4000 + grad_clip: float = 5.0 + lr: float = 1e-3 + wd: float = 0.000001 + + # overrides + min_seq_len: int = 3 + max_seq_len: int = 500 + r: int = 1 # DO NOT CHANGE - TODO: make this immutable once coqpit implements it. diff --git a/TTS/tts/configs/glow_tts_gated_conv.json b/TTS/tts/configs/glow_tts_gated_conv.json deleted file mode 100644 index c4d7b1e5..00000000 --- a/TTS/tts/configs/glow_tts_gated_conv.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "model": "glow_tts", - "run_name": "glow-tts-gatedconv", - "run_description": "glow-tts model training with gated conv.", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Griffin-Lim - "power": 1.1, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 1.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model. - - // DISTRIBUTED TRAINING - "apex_amp_level": null, // APEX amp optimization level. "O1" is currently supported. - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54323" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // MODEL PARAMETERS - "use_mas": false, // use Monotonic Alignment Search if true. Otherwise use pre-computed attention alignments. - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 1, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "loss_masking": true, // enable / disable loss masking against the sequence padding. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 0, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": true, // use noam warmup and lr schedule. - "grad_clip": 5.0, // upper limit for gradients for clipping. - "epochs": 10000, // total number of epochs to train. - "lr": 1e-3, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - "encoder_type": "gatedconv", - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "apex_amp_level": null, - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 0, //Number of batches to shuffle after bucketing. - "min_seq_len": 3, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 500, // DATASET-RELATED: maximum text length - "compute_f0": false, // compute f0 values in data-loader - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "style_wav_for_test": null, // path to style wav file to be used in TacotronGST inference. - "use_gst": false, // TACOTRON ONLY: use global style tokens - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", - "meta_file_val": null - // "path_for_attn": "/home/erogol/Data/LJSpeech-1.1/alignments/" - } - ] -} - - diff --git a/TTS/tts/configs/glow_tts_ljspeech.json b/TTS/tts/configs/glow_tts_ljspeech.json deleted file mode 100644 index 5a4c47c2..00000000 --- a/TTS/tts/configs/glow_tts_ljspeech.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "model": "glow_tts", - "run_name": "glow-tts-residual_bn_conv", - "run_description": "glow-tts model training with residual BN conv.", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Griffin-Lim - "power": 1.1, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram.00 - - // Normalization parameters - "signal_norm": false, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 1.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model. - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // MODEL PARAMETERS - // "use_mas": false, // use Monotonic Alignment Search if true. Otherwise use pre-computed attention alignments. - "hidden_channels_encoder": 192, - "hidden_channels_decoder": 192, - "hidden_channels_duration_predictor": 256, - "use_encoder_prenet": true, - "encoder_type": "rel_pos_transformer", - "encoder_params": { - "kernel_size":3, - "dropout_p": 0.1, - "num_layers": 6, - "num_heads": 2, - "hidden_channels_ffn": 768, - "input_length": null - }, - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 1, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "mixed_precision": true, - "data_dep_init_iter": 10, - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 0, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": true, // use noam warmup and lr schedule. - "grad_clip": 5.0, // upper limit for gradients for clipping. - "epochs": 10000, // total number of epochs to train. - "lr": 1e-3, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 0, //Number of batches to shuffle after bucketing. - "min_seq_len": 3, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 500, // DATASET-RELATED: maximum text length - "compute_f0": false, // compute f0 values in data-loader - "use_noise_augment": true, //add a random noise to audio signal for augmentation at training . - "compute_input_seq_cache": true, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_external_speaker_embedding_file": false, - "style_wav_for_test": null, // path to style wav file to be used in TacotronGST inference. - "use_gst": false, // TACOTRON ONLY: use global style tokens - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", - "meta_file_val": null - // "path_for_attn": "/home/erogol/Data/LJSpeech-1.1/alignments/" - } - ] - } - - diff --git a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json b/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json deleted file mode 100644 index 11e42259..00000000 --- a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "model": "Tacotron2", - "run_name": "ljspeech-dcattn", - "run_description": "tacotron2 with dynamic convolution attention.", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. - "mixed_precision": true, // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate. - - // LOSS SETTINGS - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled - "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled - "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled - "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled - "ga_alpha": 0.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. - - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TACOTRON PRENET - "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. - "prenet_type": "original", // "original" or "bn". - "prenet_dropout": false, // enable/disable dropout at prenet. - - // TACOTRON ATTENTION - "attention_type": "dynamic_convolution", // 'original' , 'graves', 'dynamic_convolution' - "attention_heads": 4, // number of attention heads (only for 'graves') - "attention_norm": "softmax", // softmax or sigmoid. - "windowing": false, // Enables attention windowing. Used only in eval mode. - "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. - "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. - "transition_agent": false, // enable/disable transition agent of forward attention. - "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. - "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. - "double_decoder_consistency": false, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ - "ddc_r": 7, // reduction rate for coarse decoder. - - // STOPNET - "stopnet": true, // Train stopnet predicting the end of synthesis. - "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 4, //Number of batches to shuffle after bucketing. - "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 153, // DATASET-RELATED: maximum text length - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": false, // use global style tokens - "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_style_tokens). - "gst_embedding_dim": 512, - "gst_num_heads": 4, - "gst_style_tokens": 10, - "gst_use_speaker_embedding": false - }, - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers - "meta_file_val": null - } - ] -} - diff --git a/TTS/tts/configs/shared_configs.py b/TTS/tts/configs/shared_configs.py new file mode 100644 index 00000000..6c710ca2 --- /dev/null +++ b/TTS/tts/configs/shared_configs.py @@ -0,0 +1,157 @@ +from dataclasses import asdict, dataclass, field +from typing import List + +from coqpit import MISSING, Coqpit, check_argument + +from TTS.config import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig + + +@dataclass +class GSTConfig(Coqpit): + """Defines the Global Style Token Module + + Args: + gst_style_input_wav (str): + Path to the wav file used to define the style of the output speech at inference. Defaults to None. + gst_style_input_weights (dict): + Defines the weights for each style token used at inference. Defaults to None. + gst_embedding_dim (int): + Defines the size of the GST embedding vector dimensions. Defaults to 256. + gst_num_heads (int): + Number of attention heads used by the multi-head attention. Defaults to 4. + gst_num_style_tokens (int): + Number of style token vectors. Defaults to 10. + """ + + gst_style_input_wav: str = None + gst_style_input_weights: dict = None + gst_embedding_dim: int = 256 + gst_use_speaker_embedding: bool = False + gst_num_heads: int = 4 + gst_num_style_tokens: int = 10 + + def check_values( + self, + ): + """Check config fields""" + c = asdict(self) + super().check_values() + check_argument("gst_style_input_weights", c, restricted=False) + check_argument("gst_style_input_wav", c, restricted=False) + check_argument("gst_embedding_dim", c, restricted=True, min_val=0, max_val=1000) + check_argument("gst_use_speaker_embedding", c, restricted=False) + check_argument("gst_num_heads", c, restricted=True, min_val=2, max_val=10) + check_argument("gst_num_style_tokens", c, restricted=True, min_val=1, max_val=1000) + + +@dataclass +class CharactersConfig(Coqpit): + """Defines character or phoneme set used by the model + + Args: + pad (str): + characters in place of empty padding. Defaults to None. + eos (str): + characters showing the end of a sentence. Defaults to None. + bos (str): + characters showing the beginning of a sentence. Defaults to None. + characters (str): + character set used by the model. Characters not in this list are ignored when converting input text to + a list of sequence IDs. Defaults to None. + punctuations (str): + characters considered as punctuation as parsing the input sentence. Defaults to None. + phonemes (str): + characters considered as parsing phonemes. Defaults to None. + unique (bool): + remove any duplicate characters in the character lists. It is a bandaid for compatibility with the old + models trained with character lists with duplicates. + """ + + pad: str = None + eos: str = None + bos: str = None + characters: str = None + punctuations: str = None + phonemes: str = None + unique: bool = True # for backwards compatibility of models trained with char sets with duplicates + + def check_values( + self, + ): + """Check config fields""" + c = asdict(self) + check_argument("pad", c, "characters", restricted=True) + check_argument("eos", c, "characters", restricted=True) + check_argument("bos", c, "characters", restricted=True) + check_argument("characters", c, "characters", restricted=True) + check_argument("phonemes", c, restricted=True) + check_argument("punctuations", c, "characters", restricted=True) + + +@dataclass +class BaseTTSConfig(BaseTrainingConfig): + """Shared parameters among all the tts models. + + Args: + audio (BaseAudioConfig): + Audio processor config object instance. + use_phonemes (bool): + enable / disable phoneme use. + compute_input_seq_cache (bool): + enable / disable precomputation of the phoneme sequences. At the expense of some delay at the beginning of + the training, It allows faster data loader time and precise limitation with `max_seq_len` and + `min_seq_len`. + text_cleaner (str): + Name of the text cleaner used for cleaning and formatting transcripts. + enable_eos_bos_chars (bool): + enable / disable the use of eos and bos characters. + test_senteces_file (str): + Path to a txt file that has sentences used at test time. The file must have a sentence per line. + phoneme_cache_path (str): + Path to the output folder caching the computed phonemes for each sample. + characters (CharactersConfig): + Instance of a CharactersConfig class. + batch_group_size (int): + Size of the batch groups used for bucketing. By default, the dataloader orders samples by the sequence + length for a more efficient and stable training. If `batch_group_size > 1` then it performs bucketing to + prevent using the same batches for each epoch. + loss_masking (bool): + enable / disable masking loss values against padded segments of samples in a batch. + min_seq_len (int): + Minimum input sequence length to be used at training. + max_seq_len (int): + Maximum input sequence length to be used at training. Larger values result in more VRAM usage. + compute_f0 (int): + (Not in use yet). + use_noise_augment (bool): + Augment the input audio with random noise. + add_blank (bool): + Add blank characters between each other two characters. It improves performance for some models at expense + of slower run-time due to the longer input sequence. + datasets (List[BaseDatasetConfig]): + List of datasets used for training. If multiple datasets are provided, they are merged and used together + for training. + """ + + audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) + # phoneme settings + use_phonemes: bool = False + phoneme_language: str = None + compute_input_seq_cache: bool = False + text_cleaner: str = MISSING + enable_eos_bos_chars: bool = False + test_sentences_file: str = "" + phoneme_cache_path: str = None + # vocabulary parameters + characters: CharactersConfig = None + # training params + batch_group_size: int = 0 + loss_masking: bool = None + # dataloading + min_seq_len: int = 1 + max_seq_len: int = float("inf") + compute_f0: bool = False + use_noise_augment: bool = False + add_blank: bool = False + # dataset + datasets: List[BaseDatasetConfig] = field(default_factory=lambda: [BaseDatasetConfig()]) diff --git a/TTS/tts/configs/speedy_speech_config.py b/TTS/tts/configs/speedy_speech_config.py new file mode 100644 index 00000000..1b8f0c82 --- /dev/null +++ b/TTS/tts/configs/speedy_speech_config.py @@ -0,0 +1,116 @@ +from dataclasses import dataclass, field + +from TTS.tts.configs.shared_configs import BaseTTSConfig + + +@dataclass +class SpeedySpeechConfig(BaseTTSConfig): + """Defines parameters for Speedy Speech (feed-forward encoder-decoder) based models. + + Example: + + >>> from TTS.tts.configs import SpeedySpeechConfig + >>> config = SpeedySpeechConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `speedy_speech`. + positional_encoding (bool): + enable / disable positional encoding applied to the encoder output. Defaults to True. + hidden_channels (int): + Base number of hidden channels. Defines all the layers expect ones defined by the specific encoder or decoder + parameters. Defaults to 128. + encoder_type (str): + Type of the encoder used by the model. Look at `TTS.tts.layers.feed_forward.encoder` for more details. + Defaults to `residual_conv_bn`. + encoder_params (dict): + Parameters used to define the encoder network. Look at `TTS.tts.layers.feed_forward.encoder` for more details. + Defaults to `{"kernel_size": 4, "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1], "num_conv_blocks": 2, "num_res_blocks": 13}` + decoder_type (str): + Type of the decoder used by the model. Look at `TTS.tts.layers.feed_forward.decoder` for more details. + Defaults to `residual_conv_bn`. + decoder_params (dict): + Parameters used to define the decoder network. Look at `TTS.tts.layers.feed_forward.decoder` for more details. + Defaults to `{"kernel_size": 4, "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1], "num_conv_blocks": 2, "num_res_blocks": 17}` + hidden_channels_encoder (int): + Number of base hidden channels used by the encoder network. It defines the input and the output channel sizes, + and for some encoder types internal hidden channels sizes too. Defaults to 192. + hidden_channels_decoder (int): + Number of base hidden channels used by the decoder WaveNet network. Defaults to 192 as in the original work. + hidden_channels_duration_predictor (int): + Number of layer channels of the duration predictor network. Defaults to 256 as in the original work. + data_dep_init_steps (int): + Number of steps used for computing normalization parameters at the beginning of the training. GlowTTS uses + Activation Normalization that pre-computes normalization stats at the beginning and use the same values + for the rest. Defaults to 10. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-3`. + wd (float): + Weight decay coefficient. Defaults to `1e-7`. + ssim_alpha (float): + Weight for the SSIM loss. If set <= 0, disables the SSIM loss. Defaults to 1.0. + huber_alpha (float): + Weight for the duration predictor's loss. Defaults to 1.0. + l1_alpha (float): + Weight for the L1 spectrogram loss. If set <= 0, disables the L1 loss. Defaults to 1.0. + min_seq_len (int): + Minimum input sequence length to be used at training. + max_seq_len (int): + Maximum input sequence length to be used at training. Larger values result in more VRAM usage. + """ + + model: str = "speedy_speech" + # model specific params + positional_encoding: bool = True + hidden_channels: int = 128 + encoder_type: str = "residual_conv_bn" + encoder_params: dict = field( + default_factory=lambda: { + "kernel_size": 4, + "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1], + "num_conv_blocks": 2, + "num_res_blocks": 13, + } + ) + decoder_type: str = "residual_conv_bn" + decoder_params: dict = field( + default_factory=lambda: { + "kernel_size": 4, + "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1], + "num_conv_blocks": 2, + "num_res_blocks": 17, + } + ) + + # multi-speaker settings + use_speaker_embedding: bool = False + use_external_speaker_embedding_file: bool = False + external_speaker_embedding_file: str = False + + # optimizer parameters + noam_schedule: bool = False + warmup_steps: int = 4000 + lr: float = 1e-4 + wd: float = 1e-6 + grad_clip: float = 5.0 + + # loss params + ssim_alpha: float = 1.0 + huber_alpha: float = 1.0 + l1_alpha: float = 1.0 + + # overrides + min_seq_len: int = 13 + max_seq_len: int = 200 + r: int = 1 # DO NOT CHANGE diff --git a/TTS/tts/configs/speedy_speech_ljspeech.json b/TTS/tts/configs/speedy_speech_ljspeech.json deleted file mode 100644 index f61f35cd..00000000 --- a/TTS/tts/configs/speedy_speech_ljspeech.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "model": "speedy_speech", - "run_name": "speedy-speech-ljspeech", - "run_description": "speedy-speech model for LJSpeech dataset.", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "&", - // "bos": "*", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZÇÃÀÁÂÊÉÍÓÔÕÚÛabcdefghijklmnopqrstuvwxyzçãàáâêéíóôõúû!(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ'̃' " - // }, - - "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model. - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // MODEL PARAMETERS - "positional_encoding": true, - "hidden_channels": 128, // defined globally all the hidden channels of the model - 128 default - "encoder_type": "residual_conv_bn", - "encoder_params":{ - "kernel_size": 4, - "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1], - "num_conv_blocks": 2, - "num_res_blocks": 13 - }, - "decoder_type": "residual_conv_bn", - "decoder_params":{ - "kernel_size": 4, - "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1], - "num_conv_blocks": 2, - "num_res_blocks": 17 - }, - - // TRAINING - "batch_size":64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":32, - "r": 1, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "loss_masking": true, // enable / disable loss masking against the sequence padding. - - // LOSS PARAMETERS - "ssim_alpha": 1, - "l1_alpha": 1, - "huber_alpha": 1, - - // VALIDATION - "run_eval": true, - "test_delay_epochs": -1, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": true, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 10000, // total number of epochs to train. - "lr": 0.002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.:set n - "mixed_precision": false, - - // DATA LOADING - "text_cleaner": "english_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 8, // number of evaluation data loader processes. - "batch_group_size": 4, //Number of batches to shuffle after bucketing. - "min_seq_len": 2, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 300, // DATASET-RELATED: maximum text length - "compute_f0": false, // compute f0 values in data-loader - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - - // PATHS - "output_path": "/home/erogol/Models/ljspeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/ljspeech_phonemes/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronoun[ciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "external_speaker_embedding_file": "/home/erogol/Data/libritts/speakers.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - - - // DATASETS - "datasets": // List of datasets. They all merged and they get different s$ - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", - "meta_file_val": null, - "meta_file_attn_mask": "/home/erogol/Data/LJSpeech-1.1/metadata_attn_mask.txt" // created by bin/compute_attention_masks.py - } - ] -} \ No newline at end of file diff --git a/TTS/tts/configs/tacotron2_config.py b/TTS/tts/configs/tacotron2_config.py new file mode 100644 index 00000000..ea66fae8 --- /dev/null +++ b/TTS/tts/configs/tacotron2_config.py @@ -0,0 +1,118 @@ +from dataclasses import dataclass + +from TTS.tts.configs.tacotron_config import TacotronConfig + + +@dataclass +class Tacotron2Config(TacotronConfig): + """Defines parameters for Tacotron2 based models. + + Example: + + >>> from TTS.tts.configs import Tacotron2Config + >>> config = Tacotron2Config() + + Args: + model (str): + Model name used to select the right model class to initilize. Defaults to `Tacotron2`. + use_gst (bool): + enable / disable the use of Global Style Token modules. Defaults to False. + gst (GSTConfig): + Instance of `GSTConfig` class. + gst_style_input (str): + Path to the wav file used at inference to set the speech style through GST. If `GST` is enabled and + this is not defined, the model uses a zero vector as an input. Defaults to None. + r (int): + Number of output frames that the decoder computed per iteration. Larger values makes training and inference + faster but reduces the quality of the output frames. This needs to be tuned considering your own needs. + Defaults to 1. + gradual_trainin (List[List]): + Parameters for the gradual training schedule. It is in the form `[[a, b, c], [d ,e ,f] ..]` where `a` is + the step number to start using the rest of the values, `b` is the `r` value and `c` is the batch size. + If sets None, no gradual training is used. Defaults to None. + memory_size (int): + Defines the number of previous frames used by the Prenet. If set to < 0, then it uses only the last frame. + Defaults to -1. + prenet_type (str): + `original` or `bn`. `original` sets the default Prenet and `bn` uses Batch Normalization version of the + Prenet. Defaults to `original`. + prenet_dropout (bool): + enables / disables the use of dropout in the Prenet. Defaults to True. + prenet_dropout_at_inference (bool): + enable / disable the use of dropout in the Prenet at the inference time. Defaults to False. + stopnet (bool): + enable /disable the Stopnet that predicts the end of the decoder sequence. Defaults to True. + stopnet_pos_weight (float): + Weight that is applied to over-weight positive instances in the Stopnet loss. Use larger values with + datasets with longer sentences. Defaults to 10. + separate_stopnet (bool): + Use a distinct Stopnet which is trained separately from the rest of the model. Defaults to True. + attention_type (str): + attention type. Check ```TTS.tts.layers.attentions.init_attn```. Defaults to 'original'. + attention_heads (int): + Number of attention heads for GMM attention. Defaults to 5. + windowing (bool): + It especially useful at inference to keep attention alignment diagonal. Defaults to False. + use_forward_attn (bool): + It is only valid if ```attn_type``` is ```original```. Defaults to False. + forward_attn_mask (bool): + enable/disable extra masking over forward attention. It is useful at inference to prevent + possible attention failures. Defaults to False. + transition_agent (bool): + enable/disable transition agent in forward attention. Defaults to False. + location_attn (bool): + enable/disable location sensitive attention as in the original Tacotron2 paper. + It is only valid if ```attn_type``` is ```original```. Defaults to True. + bidirectional_decoder (bool): + enable/disable bidirectional decoding. Defaults to False. + double_decoder_consistency (bool): + enable/disable double decoder consistency. Defaults to False. + ddc_r (int): + reduction rate used by the coarse decoder when `double_decoder_consistency` is in use. Set this + as a multiple of the `r` value. Defaults to 6. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-4`. + wd (float): + Weight decay coefficient. Defaults to `1e-6`. + grad_clip (float): + Gradient clipping threshold. Defaults to `5`. + seq_len_notm (bool): + enable / disable the sequnce length normalization in the loss functions. If set True, loss of a sample + is divided by the sequence length. Defaults to False. + loss_masking (bool): + enable / disable masking the paddings of the samples in loss computation. Defaults to True. + decoder_loss_alpha (float): + Weight for the decoder loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_loss_alpha (float): + Weight for the postnet loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_diff_spec_alpha (float): + Weight for the postnet differential loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + decoder_diff_spec_alpha (float): + Weight for the decoder differential loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + decoder_ssim_alpha (float): + Weight for the decoder SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_ssim_alpha (float): + Weight for the postnet SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + ga_alpha (float): + Weight for the guided attention loss. If set less than or equal to zero, it disables the corresponding loss + function. Defaults to 5. + """ + + model: str = "tacotron2" diff --git a/TTS/tts/configs/tacotron_config.py b/TTS/tts/configs/tacotron_config.py new file mode 100644 index 00000000..d3a54269 --- /dev/null +++ b/TTS/tts/configs/tacotron_config.py @@ -0,0 +1,176 @@ +from dataclasses import dataclass +from typing import List + +from TTS.tts.configs.shared_configs import BaseTTSConfig, GSTConfig + + +@dataclass +class TacotronConfig(BaseTTSConfig): + """Defines parameters for Tacotron based models. + + Example: + + >>> from TTS.tts.configs import TacotronConfig + >>> config = TacotronConfig() + + Args: + model (str): + Model name used to select the right model class to initilize. Defaults to `Tacotron`. + use_gst (bool): + enable / disable the use of Global Style Token modules. Defaults to False. + gst (GSTConfig): + Instance of `GSTConfig` class. + gst_style_input (str): + Path to the wav file used at inference to set the speech style through GST. If `GST` is enabled and + this is not defined, the model uses a zero vector as an input. Defaults to None. + r (int): + Initial number of output frames that the decoder computed per iteration. Larger values makes training and inference + faster but reduces the quality of the output frames. This must be equal to the largest `r` value used in + `gradual_training` schedule. Defaults to 1. + gradual_training (List[List]): + Parameters for the gradual training schedule. It is in the form `[[a, b, c], [d ,e ,f] ..]` where `a` is + the step number to start using the rest of the values, `b` is the `r` value and `c` is the batch size. + If sets None, no gradual training is used. Defaults to None. + memory_size (int): + Defines the number of previous frames used by the Prenet. If set to < 0, then it uses only the last frame. + Defaults to -1. + prenet_type (str): + `original` or `bn`. `original` sets the default Prenet and `bn` uses Batch Normalization version of the + Prenet. Defaults to `original`. + prenet_dropout (bool): + enables / disables the use of dropout in the Prenet. Defaults to True. + prenet_dropout_at_inference (bool): + enable / disable the use of dropout in the Prenet at the inference time. Defaults to False. + stopnet (bool): + enable /disable the Stopnet that predicts the end of the decoder sequence. Defaults to True. + stopnet_pos_weight (float): + Weight that is applied to over-weight positive instances in the Stopnet loss. Use larger values with + datasets with longer sentences. Defaults to 10. + separate_stopnet (bool): + Use a distinct Stopnet which is trained separately from the rest of the model. Defaults to True. + attention_type (str): + attention type. Check ```TTS.tts.layers.attentions.init_attn```. Defaults to 'original'. + attention_heads (int): + Number of attention heads for GMM attention. Defaults to 5. + windowing (bool): + It especially useful at inference to keep attention alignment diagonal. Defaults to False. + use_forward_attn (bool): + It is only valid if ```attn_type``` is ```original```. Defaults to False. + forward_attn_mask (bool): + enable/disable extra masking over forward attention. It is useful at inference to prevent + possible attention failures. Defaults to False. + transition_agent (bool): + enable/disable transition agent in forward attention. Defaults to False. + location_attn (bool): + enable/disable location sensitive attention as in the original Tacotron2 paper. + It is only valid if ```attn_type``` is ```original```. Defaults to True. + bidirectional_decoder (bool): + enable/disable bidirectional decoding. Defaults to False. + double_decoder_consistency (bool): + enable/disable double decoder consistency. Defaults to False. + ddc_r (int): + reduction rate used by the coarse decoder when `double_decoder_consistency` is in use. Set this + as a multiple of the `r` value. Defaults to 6. + use_speaker_embedding (bool): + enable / disable using speaker embeddings for multi-speaker models. If set True, the model is + in the multi-speaker mode. Defaults to False. + use_external_speaker_embedding_file (bool): + enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False. + external_speaker_embedding_file (str): + Path to the file including pre-computed speaker embeddings. Defaults to None. + noam_schedule (bool): + enable / disable the use of Noam LR scheduler. Defaults to False. + warmup_steps (int): + Number of warm-up steps for the Noam scheduler. Defaults 4000. + lr (float): + Initial learning rate. Defaults to `1e-4`. + wd (float): + Weight decay coefficient. Defaults to `1e-6`. + grad_clip (float): + Gradient clipping threshold. Defaults to `5`. + seq_len_notm (bool): + enable / disable the sequnce length normalization in the loss functions. If set True, loss of a sample + is divided by the sequence length. Defaults to False. + loss_masking (bool): + enable / disable masking the paddings of the samples in loss computation. Defaults to True. + decoder_loss_alpha (float): + Weight for the decoder loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_loss_alpha (float): + Weight for the postnet loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_diff_spec_alpha (float): + Weight for the postnet differential loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + decoder_diff_spec_alpha (float): + Weight for the decoder differential loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + decoder_ssim_alpha (float): + Weight for the decoder SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + postnet_ssim_alpha (float): + Weight for the postnet SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the + corresponding loss function. Defaults to 0.25 + ga_alpha (float): + Weight for the guided attention loss. If set less than or equal to zero, it disables the corresponding loss + function. Defaults to 5. + """ + + model: str = "tacotron" + use_gst: bool = False + gst: GSTConfig = None + gst_style_input: str = None + # model specific params + r: int = 2 + gradual_training: List[List] = None + memory_size: int = -1 + prenet_type: str = "original" + prenet_dropout: bool = True + prenet_dropout_at_inference: bool = False + stopnet: bool = True + separate_stopnet: bool = True + stopnet_pos_weight: float = 10.0 + + # attention layers + attention_type: str = "original" + attention_heads: int = None + attention_norm: str = "sigmoid" + windowing: bool = False + use_forward_attn: bool = False + forward_attn_mask: bool = False + transition_agent: bool = False + location_attn: bool = True + + # advance methods + bidirectional_decoder: bool = False + double_decoder_consistency: bool = False + ddc_r: int = 6 + + # multi-speaker settings + use_speaker_embedding: bool = False + use_external_speaker_embedding_file: bool = False + external_speaker_embedding_file: str = False + + # optimizer parameters + noam_schedule: bool = False + warmup_steps: int = 4000 + lr: float = 1e-4 + wd: float = 1e-6 + grad_clip: float = 5.0 + seq_len_norm: bool = False + loss_masking: bool = True + + # loss params + decoder_loss_alpha: float = 0.25 + postnet_loss_alpha: float = 0.25 + postnet_diff_spec_alpha: float = 0.25 + decoder_diff_spec_alpha: float = 0.25 + decoder_ssim_alpha: float = 0.25 + postnet_ssim_alpha: float = 0.25 + ga_alpha: float = 5.0 + + def check_values(self): + if self.gradual_training: + assert ( + self.gradual_training[0][1] == self.r + ), f"[!] the first scheduled gradual training `r` must be equal to the model's `r` value. {self.gradual_training[0][1]} vs {self.r}" diff --git a/TTS/tts/datasets/TTSDataset.py b/TTS/tts/datasets/TTSDataset.py index b613e37c..4ca93232 100644 --- a/TTS/tts/datasets/TTSDataset.py +++ b/TTS/tts/datasets/TTSDataset.py @@ -25,7 +25,7 @@ class MyDataset(Dataset): batch_group_size=0, min_seq_len=0, max_seq_len=float("inf"), - use_phonemes=True, + use_phonemes=False, phoneme_cache_path=None, phoneme_language="en-us", enable_eos_bos=False, diff --git a/TTS/tts/datasets/preprocess.py b/TTS/tts/datasets/preprocess.py index 0f82814d..72ab160e 100644 --- a/TTS/tts/datasets/preprocess.py +++ b/TTS/tts/datasets/preprocess.py @@ -2,19 +2,41 @@ import os import re import sys import xml.etree.ElementTree as ET +from collections import Counter from glob import glob from pathlib import Path from typing import List +import numpy as np from tqdm import tqdm -from TTS.tts.utils.generic_utils import split_dataset - #################### # UTILITIES #################### +def split_dataset(items): + speakers = [item[-1] for item in items] + is_multi_speaker = len(set(speakers)) > 1 + eval_split_size = min(500, int(len(items) * 0.01)) + assert eval_split_size > 0, " [!] You do not have enough samples to train. You need at least 100 samples." + np.random.seed(0) + np.random.shuffle(items) + if is_multi_speaker: + items_eval = [] + speakers = [item[-1] for item in items] + speaker_counter = Counter(speakers) + while len(items_eval) < eval_split_size: + item_idx = np.random.randint(0, len(items)) + speaker_to_be_removed = items[item_idx][-1] + if speaker_counter[speaker_to_be_removed] > 1: + items_eval.append(items[item_idx]) + speaker_counter[speaker_to_be_removed] -= 1 + del items[item_idx] + return items_eval, items + return items[:eval_split_size], items[eval_split_size:] + + def load_meta_data(datasets, eval_split=True): meta_data_train_all = [] meta_data_eval_all = [] if eval_split else None @@ -30,19 +52,19 @@ def load_meta_data(datasets, eval_split=True): print(f" | > Found {len(meta_data_train)} files in {Path(root_path).resolve()}") # load evaluation split if set if eval_split: - if meta_file_val is None: - meta_data_eval, meta_data_train = split_dataset(meta_data_train) - else: + if meta_file_val: meta_data_eval = preprocessor(root_path, meta_file_val) + else: + meta_data_eval, meta_data_train = split_dataset(meta_data_train) meta_data_eval_all += meta_data_eval meta_data_train_all += meta_data_train # load attention masks for duration predictor training - if "meta_file_attn_mask" in dataset and dataset["meta_file_attn_mask"] is not None: + if dataset.meta_file_attn_mask: meta_data = dict(load_attention_mask_meta_data(dataset["meta_file_attn_mask"])) for idx, ins in enumerate(meta_data_train_all): attn_file = meta_data[ins[1]].strip() meta_data_train_all[idx].append(attn_file) - if meta_data_eval_all is not None: + if meta_data_eval_all: for idx, ins in enumerate(meta_data_eval_all): attn_file = meta_data[ins[1]].strip() meta_data_eval_all[idx].append(attn_file) diff --git a/TTS/tts/layers/tacotron/tacotron.py b/TTS/tts/layers/tacotron/tacotron.py index dcb5fdc5..dc38173f 100644 --- a/TTS/tts/layers/tacotron/tacotron.py +++ b/TTS/tts/layers/tacotron/tacotron.py @@ -463,7 +463,6 @@ class Decoder(nn.Module): stop_tokens = [] t = 0 self._init_states(inputs) - self.attention.init_win_idx() self.attention.init_states(inputs) while True: if t > 0: diff --git a/TTS/tts/layers/tacotron/tacotron2.py b/TTS/tts/layers/tacotron/tacotron2.py index df14aead..aeca8953 100644 --- a/TTS/tts/layers/tacotron/tacotron2.py +++ b/TTS/tts/layers/tacotron/tacotron2.py @@ -375,7 +375,6 @@ class Decoder(nn.Module): else: self._init_states(inputs, mask=None, keep_states=True) - self.attention.init_win_idx() self.attention.init_states(inputs) outputs, stop_tokens, alignments, t = [], [], [], 0 while True: diff --git a/TTS/tts/models/glow_tts.py b/TTS/tts/models/glow_tts.py old mode 100644 new mode 100755 index 1b84b451..19eb594a --- a/TTS/tts/models/glow_tts.py +++ b/TTS/tts/models/glow_tts.py @@ -179,6 +179,81 @@ class GlowTTS(nn.Module): attn = attn.squeeze(1).permute(0, 2, 1) return z, logdet, y_mean, y_log_scale, attn, o_dur_log, o_attn_dur + @torch.no_grad() + def inference_with_MAS(self, x, x_lengths, y=None, y_lengths=None, attn=None, g=None): + """ + It's similar to the teacher forcing in Tacotron. + It was proposed in: https://arxiv.org/abs/2104.05557 + Shapes: + x: [B, T] + x_lenghts: B + y: [B, C, T] + y_lengths: B + g: [B, C] or B + """ + y_max_length = y.size(2) + # norm speaker embeddings + if g is not None: + if self.external_speaker_embedding_dim: + g = F.normalize(g).unsqueeze(-1) + else: + g = F.normalize(self.emb_g(g)).unsqueeze(-1) # [b, h, 1] + + # embedding pass + o_mean, o_log_scale, o_dur_log, x_mask = self.encoder(x, x_lengths, g=g) + # drop redisual frames wrt num_squeeze and set y_lengths. + y, y_lengths, y_max_length, attn = self.preprocess(y, y_lengths, y_max_length, None) + # create masks + y_mask = torch.unsqueeze(sequence_mask(y_lengths, y_max_length), 1).to(x_mask.dtype) + attn_mask = torch.unsqueeze(x_mask, -1) * torch.unsqueeze(y_mask, 2) + # decoder pass + z, logdet = self.decoder(y, y_mask, g=g, reverse=False) + # find the alignment path between z and encoder output + o_scale = torch.exp(-2 * o_log_scale) + logp1 = torch.sum(-0.5 * math.log(2 * math.pi) - o_log_scale, [1]).unsqueeze(-1) # [b, t, 1] + logp2 = torch.matmul(o_scale.transpose(1, 2), -0.5 * (z ** 2)) # [b, t, d] x [b, d, t'] = [b, t, t'] + logp3 = torch.matmul((o_mean * o_scale).transpose(1, 2), z) # [b, t, d] x [b, d, t'] = [b, t, t'] + logp4 = torch.sum(-0.5 * (o_mean ** 2) * o_scale, [1]).unsqueeze(-1) # [b, t, 1] + logp = logp1 + logp2 + logp3 + logp4 # [b, t, t'] + attn = maximum_path(logp, attn_mask.squeeze(1)).unsqueeze(1).detach() + + y_mean, y_log_scale, o_attn_dur = self.compute_outputs(attn, o_mean, o_log_scale, x_mask) + attn = attn.squeeze(1).permute(0, 2, 1) + + # get predited aligned distribution + z = y_mean * y_mask + + # reverse the decoder and predict using the aligned distribution + y, logdet = self.decoder(z, y_mask, g=g, reverse=True) + + return y, logdet, y_mean, y_log_scale, attn, o_dur_log, o_attn_dur + + @torch.no_grad() + def decoder_inference(self, y, y_lengths=None, g=None): + """ + Shapes: + y: [B, C, T] + y_lengths: B + g: [B, C] or B + """ + y_max_length = y.size(2) + # norm speaker embeddings + if g is not None: + if self.external_speaker_embedding_dim: + g = F.normalize(g).unsqueeze(-1) + else: + g = F.normalize(self.emb_g(g)).unsqueeze(-1) # [b, h, 1] + + y_mask = torch.unsqueeze(sequence_mask(y_lengths, y_max_length), 1).to(y.dtype) + + # decoder pass + z, logdet = self.decoder(y, y_mask, g=g, reverse=False) + + # reverse decoder and predict + y, logdet = self.decoder(z, y_mask, g=g, reverse=True) + + return y, logdet + @torch.no_grad() def inference(self, x, x_lengths, g=None): if g is not None: diff --git a/TTS/tts/models/tacotron.py b/TTS/tts/models/tacotron.py index 297c8e3e..89d98e9f 100644 --- a/TTS/tts/models/tacotron.py +++ b/TTS/tts/models/tacotron.py @@ -41,11 +41,8 @@ class Tacotron(TacotronAbstract): encoder_in_features (int, optional): input channels for the encoder. Defaults to 512. decoder_in_features (int, optional): input channels for the decoder. Defaults to 512. speaker_embedding_dim (int, optional): external speaker conditioning vector channels. Defaults to None. - gst (bool, optional): enable/disable global style token learning. Defaults to False. - gst_embedding_dim (int, optional): size of channels for GST vectors. Defaults to 512. - gst_num_heads (int, optional): number of attention heads for GST. Defaults to 4. - gst_style_tokens (int, optional): number of GST tokens. Defaults to 10. - gst_use_speaker_embedding (bool, optional): enable/disable inputing speaker embedding to GST. Defaults to False. + use_gst (bool, optional): enable/disable Global style token module. + gst (Coqpit, optional): Coqpit to initialize the GST module. If `None`, GST is disabled. Defaults to None. memory_size (int, optional): size of the history queue fed to the prenet. Model feeds the last ```memory_size``` output frames to the prenet. """ @@ -75,12 +72,9 @@ class Tacotron(TacotronAbstract): encoder_in_features=256, decoder_in_features=256, speaker_embedding_dim=None, - gst=False, - gst_embedding_dim=256, - gst_num_heads=4, - gst_style_tokens=10, + use_gst=False, + gst=None, memory_size=5, - gst_use_speaker_embedding=False, ): super().__init__( num_chars, @@ -106,11 +100,8 @@ class Tacotron(TacotronAbstract): encoder_in_features, decoder_in_features, speaker_embedding_dim, + use_gst, gst, - gst_embedding_dim, - gst_num_heads, - gst_style_tokens, - gst_use_speaker_embedding, ) # speaker embedding layers @@ -154,15 +145,13 @@ class Tacotron(TacotronAbstract): self.decoder.prenet.dropout_at_inference = prenet_dropout_at_inference # global style token layers - if self.gst: + if self.gst and self.use_gst: self.gst_layer = GST( - num_mel=80, - num_heads=gst_num_heads, - num_style_tokens=gst_style_tokens, - gst_embedding_dim=self.gst_embedding_dim, - speaker_embedding_dim=speaker_embedding_dim - if self.embeddings_per_sample and self.gst_use_speaker_embedding - else None, + num_mel=decoder_output_dim, + speaker_embedding_dim=speaker_embedding_dim, + num_heads=gst.gst_num_heads, + num_style_tokens=gst.gst_num_style_tokens, + gst_embedding_dim=gst.gst_embedding_dim, ) # backward pass decoder if self.bidirectional_decoder: @@ -205,11 +194,9 @@ class Tacotron(TacotronAbstract): # sequence masking encoder_outputs = encoder_outputs * input_mask.unsqueeze(2).expand_as(encoder_outputs) # global style token - if self.gst: + if self.gst and self.use_gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, mel_specs, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, mel_specs, speaker_embeddings) # speaker embedding if self.num_speakers > 1: if not self.embeddings_per_sample: @@ -263,11 +250,9 @@ class Tacotron(TacotronAbstract): def inference(self, characters, speaker_ids=None, style_mel=None, speaker_embeddings=None): inputs = self.embedding(characters) encoder_outputs = self.encoder(inputs) - if self.gst: + if self.gst and self.use_gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings) if self.num_speakers > 1: if not self.embeddings_per_sample: # B x 1 x speaker_embed_dim diff --git a/TTS/tts/models/tacotron2.py b/TTS/tts/models/tacotron2.py index c015a195..fded8f87 100644 --- a/TTS/tts/models/tacotron2.py +++ b/TTS/tts/models/tacotron2.py @@ -41,11 +41,8 @@ class Tacotron2(TacotronAbstract): encoder_in_features (int, optional): input channels for the encoder. Defaults to 512. decoder_in_features (int, optional): input channels for the decoder. Defaults to 512. speaker_embedding_dim (int, optional): external speaker conditioning vector channels. Defaults to None. - gst (bool, optional): enable/disable global style token learning. Defaults to False. - gst_embedding_dim (int, optional): size of channels for GST vectors. Defaults to 512. - gst_num_heads (int, optional): number of attention heads for GST. Defaults to 4. - gst_style_tokens (int, optional): number of GST tokens. Defaults to 10. - gst_use_speaker_embedding (bool, optional): enable/disable inputing speaker embedding to GST. Defaults to False. + use_gst (bool, optional): enable/disable Global style token module. + gst (Coqpit, optional): Coqpit to initialize the GST module. If `None`, GST is disabled. Defaults to None. """ def __init__( @@ -73,11 +70,8 @@ class Tacotron2(TacotronAbstract): encoder_in_features=512, decoder_in_features=512, speaker_embedding_dim=None, - gst=False, - gst_embedding_dim=512, - gst_num_heads=4, - gst_style_tokens=10, - gst_use_speaker_embedding=False, + use_gst=False, + gst=None, ): super().__init__( num_chars, @@ -103,11 +97,8 @@ class Tacotron2(TacotronAbstract): encoder_in_features, decoder_in_features, speaker_embedding_dim, + use_gst, gst, - gst_embedding_dim, - gst_num_heads, - gst_style_tokens, - gst_use_speaker_embedding, ) # speaker embedding layer @@ -148,16 +139,15 @@ class Tacotron2(TacotronAbstract): self.decoder.prenet.dropout_at_inference = prenet_dropout_at_inference # global style token layers - if self.gst: + if self.gst and use_gst: self.gst_layer = GST( - num_mel=80, - num_heads=self.gst_num_heads, - num_style_tokens=self.gst_style_tokens, - gst_embedding_dim=self.gst_embedding_dim, - speaker_embedding_dim=speaker_embedding_dim - if self.embeddings_per_sample and self.gst_use_speaker_embedding - else None, + num_mel=decoder_output_dim, + speaker_embedding_dim=speaker_embedding_dim, + num_heads=gst.gst_num_heads, + num_style_tokens=gst.gst_num_style_tokens, + gst_embedding_dim=gst.gst_embedding_dim, ) + # backward pass decoder if self.bidirectional_decoder: self._init_backward_decoder() @@ -203,11 +193,9 @@ class Tacotron2(TacotronAbstract): embedded_inputs = self.embedding(text).transpose(1, 2) # B x T_in_max x D_en encoder_outputs = self.encoder(embedded_inputs, text_lengths) - if self.gst: + if self.gst and self.use_gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, mel_specs, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, mel_specs, speaker_embeddings) if self.num_speakers > 1: if not self.embeddings_per_sample: # B x 1 x speaker_embed_dim @@ -261,11 +249,9 @@ class Tacotron2(TacotronAbstract): embedded_inputs = self.embedding(text).transpose(1, 2) encoder_outputs = self.encoder.inference(embedded_inputs) - if self.gst: + if self.gst and self.use_gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings) if self.num_speakers > 1: if not self.embeddings_per_sample: speaker_embeddings = self.speaker_embedding(speaker_ids)[:, None] @@ -286,9 +272,7 @@ class Tacotron2(TacotronAbstract): if self.gst: # B x gst_dim - encoder_outputs = self.compute_gst( - encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None - ) + encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings) if self.num_speakers > 1: if not self.embeddings_per_sample: diff --git a/TTS/tts/models/tacotron_abstract.py b/TTS/tts/models/tacotron_abstract.py index 1dc49e1b..e684ce7c 100644 --- a/TTS/tts/models/tacotron_abstract.py +++ b/TTS/tts/models/tacotron_abstract.py @@ -33,11 +33,8 @@ class TacotronAbstract(ABC, nn.Module): encoder_in_features=512, decoder_in_features=512, speaker_embedding_dim=None, - gst=False, - gst_embedding_dim=512, - gst_num_heads=4, - gst_style_tokens=10, - gst_use_speaker_embedding=False, + use_gst=False, + gst=None, ): """Abstract Tacotron class""" super().__init__() @@ -45,11 +42,8 @@ class TacotronAbstract(ABC, nn.Module): self.r = r self.decoder_output_dim = decoder_output_dim self.postnet_output_dim = postnet_output_dim + self.use_gst = use_gst self.gst = gst - self.gst_embedding_dim = gst_embedding_dim - self.gst_num_heads = gst_num_heads - self.gst_style_tokens = gst_style_tokens - self.gst_use_speaker_embedding = gst_use_speaker_embedding self.num_speakers = num_speakers self.bidirectional_decoder = bidirectional_decoder self.double_decoder_consistency = double_decoder_consistency @@ -85,8 +79,8 @@ class TacotronAbstract(ABC, nn.Module): self.embeddings_per_sample = True # global style token - if self.gst: - self.decoder_in_features += gst_embedding_dim # add gst embedding dim + if self.gst and use_gst: + self.decoder_in_features += self.gst.gst_embedding_dim # add gst embedding dim self.gst_layer = None # model states @@ -194,18 +188,18 @@ class TacotronAbstract(ABC, nn.Module): """Compute global style token""" device = inputs.device if isinstance(style_input, dict): - query = torch.zeros(1, 1, self.gst_embedding_dim // 2).to(device) + query = torch.zeros(1, 1, self.gst.gst_embedding_dim // 2).to(device) if speaker_embedding is not None: query = torch.cat([query, speaker_embedding.reshape(1, 1, -1)], dim=-1) _GST = torch.tanh(self.gst_layer.style_token_layer.style_tokens) - gst_outputs = torch.zeros(1, 1, self.gst_embedding_dim).to(device) + gst_outputs = torch.zeros(1, 1, self.gst.gst_embedding_dim).to(device) for k_token, v_amplifier in style_input.items(): key = _GST[int(k_token)].unsqueeze(0).expand(1, -1, -1) gst_outputs_att = self.gst_layer.style_token_layer.attention(query, key) gst_outputs = gst_outputs + gst_outputs_att * v_amplifier elif style_input is None: - gst_outputs = torch.zeros(1, 1, self.gst_embedding_dim).to(device) + gst_outputs = torch.zeros(1, 1, self.gst.gst_embedding_dim).to(device) else: gst_outputs = self.gst_layer(style_input, speaker_embedding) # pylint: disable=not-callable inputs = self._concat_speaker_embedding(inputs, gst_outputs) diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index c58f37c9..b0e53f33 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -1,33 +1,6 @@ -import importlib -import re -from collections import Counter - -import numpy as np import torch -from TTS.utils.generic_utils import check_argument - - -def split_dataset(items): - speakers = [item[-1] for item in items] - is_multi_speaker = len(set(speakers)) > 1 - eval_split_size = min(500, int(len(items) * 0.01)) - assert eval_split_size > 0, " [!] You do not have enough samples to train. You need at least 100 samples." - np.random.seed(0) - np.random.shuffle(items) - if is_multi_speaker: - items_eval = [] - speakers = [item[-1] for item in items] - speaker_counter = Counter(speakers) - while len(items_eval) < eval_split_size: - item_idx = np.random.randint(0, len(items)) - speaker_to_be_removed = items[item_idx][-1] - if speaker_counter[speaker_to_be_removed] > 1: - items_eval.append(items[item_idx]) - speaker_counter[speaker_to_be_removed] -= 1 - del items[item_idx] - return items_eval, items - return items[:eval_split_size], items[eval_split_size:] +from TTS.utils.generic_utils import find_module # from https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1 @@ -39,17 +12,9 @@ def sequence_mask(sequence_length, max_len=None): return seq_range.unsqueeze(0) < sequence_length.unsqueeze(1) -def to_camel(text): - text = text.capitalize() - text = re.sub(r"(?!^)_([a-zA-Z])", lambda m: m.group(1).upper(), text) - text = text.replace("Tts", "TTS") - return text - - def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): print(" > Using model: {}".format(c.model)) - MyModel = importlib.import_module("TTS.tts.models." + c.model.lower()) - MyModel = getattr(MyModel, to_camel(c.model)) + MyModel = find_module("TTS.tts.models", c.model.lower()) if c.model.lower() in "tacotron": model = MyModel( num_chars=num_chars + getattr(c, "add_blank", False), @@ -57,18 +22,15 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): r=c.r, postnet_output_dim=int(c.audio["fft_size"] / 2 + 1), decoder_output_dim=c.audio["num_mels"], - gst=c.use_gst, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"], + use_gst=c.use_gst, + gst=c.gst, memory_size=c.memory_size, attn_type=c.attention_type, attn_win=c.windowing, attn_norm=c.attention_norm, prenet_type=c.prenet_type, prenet_dropout=c.prenet_dropout, - prenet_dropout_at_inference=c.prenet_dropout_at_inference if "prenet_dropout_at_inference" in c else False, + prenet_dropout_at_inference=c.prenet_dropout_at_inference, forward_attn=c.use_forward_attn, trans_agent=c.transition_agent, forward_attn_mask=c.forward_attn_mask, @@ -87,17 +49,14 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): r=c.r, postnet_output_dim=c.audio["num_mels"], decoder_output_dim=c.audio["num_mels"], - gst=c.use_gst, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"], + use_gst=c.use_gst, + gst=c.gst, attn_type=c.attention_type, attn_win=c.windowing, attn_norm=c.attention_norm, prenet_type=c.prenet_type, prenet_dropout=c.prenet_dropout, - prenet_dropout_at_inference=c.prenet_dropout_at_inference if "prenet_dropout_at_inference" in c else False, + prenet_dropout_at_inference=c.prenet_dropout_at_inference, forward_attn=c.use_forward_attn, trans_agent=c.transition_agent, forward_attn_mask=c.forward_attn_mask, @@ -119,7 +78,7 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None): encoder_type=c.encoder_type, encoder_params=c.encoder_params, use_encoder_prenet=c["use_encoder_prenet"], - inference_noise_scale=c.get("inference_noise_scale", 0.33), + inference_noise_scale=c.inference_noise_scale, num_flow_blocks_dec=12, kernel_size_dec=5, dilation_rate=1, @@ -164,202 +123,156 @@ def is_tacotron(c): return "tacotron" in c["model"].lower() -def check_config_tts(c): - check_argument( - "model", - c, - enum_list=["tacotron", "tacotron2", "glow_tts", "speedy_speech", "align_tts"], - restricted=True, - val_type=str, - ) - check_argument("run_name", c, restricted=True, val_type=str) - check_argument("run_description", c, val_type=str) +# def check_config_tts(c): +# check_argument('model', c, enum_list=['tacotron', 'tacotron2', 'glow_tts', 'speedy_speech', 'align_tts'], restricted=True, val_type=str) +# check_argument('run_name', c, restricted=True, val_type=str) +# check_argument('run_description', c, val_type=str) - # AUDIO - check_argument("audio", c, restricted=True, val_type=dict) +# # AUDIO +# # check_argument('audio', c, restricted=True, val_type=dict) - # audio processing parameters - check_argument("num_mels", c["audio"], restricted=True, val_type=int, min_val=10, max_val=2056) - check_argument("fft_size", c["audio"], restricted=True, val_type=int, min_val=128, max_val=4058) - check_argument("sample_rate", c["audio"], restricted=True, val_type=int, min_val=512, max_val=100000) - check_argument( - "frame_length_ms", - c["audio"], - restricted=True, - val_type=float, - min_val=10, - max_val=1000, - alternative="win_length", - ) - check_argument( - "frame_shift_ms", c["audio"], restricted=True, val_type=float, min_val=1, max_val=1000, alternative="hop_length" - ) - check_argument("preemphasis", c["audio"], restricted=True, val_type=float, min_val=0, max_val=1) - check_argument("min_level_db", c["audio"], restricted=True, val_type=int, min_val=-1000, max_val=10) - check_argument("ref_level_db", c["audio"], restricted=True, val_type=int, min_val=0, max_val=1000) - check_argument("power", c["audio"], restricted=True, val_type=float, min_val=1, max_val=5) - check_argument("griffin_lim_iters", c["audio"], restricted=True, val_type=int, min_val=10, max_val=1000) +# # audio processing parameters +# # check_argument('num_mels', c['audio'], restricted=True, val_type=int, min_val=10, max_val=2056) +# # check_argument('fft_size', c['audio'], restricted=True, val_type=int, min_val=128, max_val=4058) +# # check_argument('sample_rate', c['audio'], restricted=True, val_type=int, min_val=512, max_val=100000) +# # check_argument('frame_length_ms', c['audio'], restricted=True, val_type=float, min_val=10, max_val=1000, alternative='win_length') +# # check_argument('frame_shift_ms', c['audio'], restricted=True, val_type=float, min_val=1, max_val=1000, alternative='hop_length') +# # check_argument('preemphasis', c['audio'], restricted=True, val_type=float, min_val=0, max_val=1) +# # check_argument('min_level_db', c['audio'], restricted=True, val_type=int, min_val=-1000, max_val=10) +# # check_argument('ref_level_db', c['audio'], restricted=True, val_type=int, min_val=0, max_val=1000) +# # check_argument('power', c['audio'], restricted=True, val_type=float, min_val=1, max_val=5) +# # check_argument('griffin_lim_iters', c['audio'], restricted=True, val_type=int, min_val=10, max_val=1000) - # vocabulary parameters - check_argument("characters", c, restricted=False, val_type=dict) - check_argument( - "pad", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str - ) - check_argument( - "eos", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str - ) - check_argument( - "bos", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str - ) - check_argument( - "characters", - c["characters"] if "characters" in c.keys() else {}, - restricted="characters" in c.keys(), - val_type=str, - ) - check_argument( - "phonemes", - c["characters"] if "characters" in c.keys() else {}, - restricted="characters" in c.keys() and c["use_phonemes"], - val_type=str, - ) - check_argument( - "punctuations", - c["characters"] if "characters" in c.keys() else {}, - restricted="characters" in c.keys(), - val_type=str, - ) +# # vocabulary parameters +# check_argument('characters', c, restricted=False, val_type=dict) +# check_argument('pad', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) +# check_argument('eos', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) +# check_argument('bos', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) +# check_argument('characters', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) +# check_argument('phonemes', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys() and c['use_phonemes'], val_type=str) +# check_argument('punctuations', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) - # normalization parameters - check_argument("signal_norm", c["audio"], restricted=True, val_type=bool) - check_argument("symmetric_norm", c["audio"], restricted=True, val_type=bool) - check_argument("max_norm", c["audio"], restricted=True, val_type=float, min_val=0.1, max_val=1000) - check_argument("clip_norm", c["audio"], restricted=True, val_type=bool) - check_argument("mel_fmin", c["audio"], restricted=True, val_type=float, min_val=0.0, max_val=1000) - check_argument("mel_fmax", c["audio"], restricted=True, val_type=float, min_val=500.0) - check_argument("spec_gain", c["audio"], restricted=True, val_type=[int, float], min_val=1, max_val=100) - check_argument("do_trim_silence", c["audio"], restricted=True, val_type=bool) - check_argument("trim_db", c["audio"], restricted=True, val_type=int) +# # normalization parameters +# # check_argument('signal_norm', c['audio'], restricted=True, val_type=bool) +# # check_argument('symmetric_norm', c['audio'], restricted=True, val_type=bool) +# # check_argument('max_norm', c['audio'], restricted=True, val_type=float, min_val=0.1, max_val=1000) +# # check_argument('clip_norm', c['audio'], restricted=True, val_type=bool) +# # check_argument('mel_fmin', c['audio'], restricted=True, val_type=float, min_val=0.0, max_val=1000) +# # check_argument('mel_fmax', c['audio'], restricted=True, val_type=float, min_val=500.0) +# # check_argument('spec_gain', c['audio'], restricted=True, val_type=[int, float], min_val=1, max_val=100) +# # check_argument('do_trim_silence', c['audio'], restricted=True, val_type=bool) +# # check_argument('trim_db', c['audio'], restricted=True, val_type=int) - # training parameters - check_argument("batch_size", c, restricted=True, val_type=int, min_val=1) - check_argument("eval_batch_size", c, restricted=True, val_type=int, min_val=1) - check_argument("r", c, restricted=True, val_type=int, min_val=1) - check_argument("gradual_training", c, restricted=False, val_type=list) - check_argument("mixed_precision", c, restricted=False, val_type=bool) - # check_argument('grad_accum', c, restricted=True, val_type=int, min_val=1, max_val=100) +# # training parameters +# # check_argument('batch_size', c, restricted=True, val_type=int, min_val=1) +# # check_argument('eval_batch_size', c, restricted=True, val_type=int, min_val=1) +# # check_argument('r', c, restricted=True, val_type=int, min_val=1) +# # check_argument('gradual_training', c, restricted=False, val_type=list) +# # check_argument('mixed_precision', c, restricted=False, val_type=bool) +# # check_argument('grad_accum', c, restricted=True, val_type=int, min_val=1, max_val=100) - # loss parameters - check_argument("loss_masking", c, restricted=True, val_type=bool) - if c["model"].lower() in ["tacotron", "tacotron2"]: - check_argument("decoder_loss_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("postnet_loss_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("postnet_diff_spec_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("decoder_diff_spec_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("decoder_ssim_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("postnet_ssim_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("ga_alpha", c, restricted=True, val_type=float, min_val=0) - if c["model"].lower in ["speedy_speech", "align_tts"]: - check_argument("ssim_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("l1_alpha", c, restricted=True, val_type=float, min_val=0) - check_argument("huber_alpha", c, restricted=True, val_type=float, min_val=0) +# # loss parameters +# # check_argument('loss_masking', c, restricted=True, val_type=bool) +# # if c['model'].lower() in ['tacotron', 'tacotron2']: +# # check_argument('decoder_loss_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('postnet_loss_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('postnet_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('decoder_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('decoder_ssim_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('postnet_ssim_alpha', c, restricted=True, val_type=float, min_val=0) +# # check_argument('ga_alpha', c, restricted=True, val_type=float, min_val=0) +# if c['model'].lower in ["speedy_speech", "align_tts"]: +# check_argument('ssim_alpha', c, restricted=True, val_type=float, min_val=0) +# check_argument('l1_alpha', c, restricted=True, val_type=float, min_val=0) +# check_argument('huber_alpha', c, restricted=True, val_type=float, min_val=0) - # validation parameters - check_argument("run_eval", c, restricted=True, val_type=bool) - check_argument("test_delay_epochs", c, restricted=True, val_type=int, min_val=0) - check_argument("test_sentences_file", c, restricted=False, val_type=str) +# # validation parameters +# # check_argument('run_eval', c, restricted=True, val_type=bool) +# # check_argument('test_delay_epochs', c, restricted=True, val_type=int, min_val=0) +# # check_argument('test_sentences_file', c, restricted=False, val_type=str) - # optimizer - check_argument("noam_schedule", c, restricted=False, val_type=bool) - check_argument("grad_clip", c, restricted=True, val_type=float, min_val=0.0) - check_argument("epochs", c, restricted=True, val_type=int, min_val=1) - check_argument("lr", c, restricted=True, val_type=float, min_val=0) - check_argument("wd", c, restricted=is_tacotron(c), val_type=float, min_val=0) - check_argument("warmup_steps", c, restricted=True, val_type=int, min_val=0) - check_argument("seq_len_norm", c, restricted=is_tacotron(c), val_type=bool) +# # optimizer +# check_argument('noam_schedule', c, restricted=False, val_type=bool) +# check_argument('grad_clip', c, restricted=True, val_type=float, min_val=0.0) +# check_argument('epochs', c, restricted=True, val_type=int, min_val=1) +# check_argument('lr', c, restricted=True, val_type=float, min_val=0) +# check_argument('wd', c, restricted=is_tacotron(c), val_type=float, min_val=0) +# check_argument('warmup_steps', c, restricted=True, val_type=int, min_val=0) +# check_argument('seq_len_norm', c, restricted=is_tacotron(c), val_type=bool) - # tacotron prenet - check_argument("memory_size", c, restricted=is_tacotron(c), val_type=int, min_val=-1) - check_argument("prenet_type", c, restricted=is_tacotron(c), val_type=str, enum_list=["original", "bn"]) - check_argument("prenet_dropout", c, restricted=is_tacotron(c), val_type=bool) +# # tacotron prenet +# # check_argument('memory_size', c, restricted=is_tacotron(c), val_type=int, min_val=-1) +# # check_argument('prenet_type', c, restricted=is_tacotron(c), val_type=str, enum_list=['original', 'bn']) +# # check_argument('prenet_dropout', c, restricted=is_tacotron(c), val_type=bool) - # attention - check_argument( - "attention_type", - c, - restricted=is_tacotron(c), - val_type=str, - enum_list=["graves", "original", "dynamic_convolution"], - ) - check_argument("attention_heads", c, restricted=is_tacotron(c), val_type=int) - check_argument("attention_norm", c, restricted=is_tacotron(c), val_type=str, enum_list=["sigmoid", "softmax"]) - check_argument("windowing", c, restricted=is_tacotron(c), val_type=bool) - check_argument("use_forward_attn", c, restricted=is_tacotron(c), val_type=bool) - check_argument("forward_attn_mask", c, restricted=is_tacotron(c), val_type=bool) - check_argument("transition_agent", c, restricted=is_tacotron(c), val_type=bool) - check_argument("transition_agent", c, restricted=is_tacotron(c), val_type=bool) - check_argument("location_attn", c, restricted=is_tacotron(c), val_type=bool) - check_argument("bidirectional_decoder", c, restricted=is_tacotron(c), val_type=bool) - check_argument("double_decoder_consistency", c, restricted=is_tacotron(c), val_type=bool) - check_argument("ddc_r", c, restricted="double_decoder_consistency" in c.keys(), min_val=1, max_val=7, val_type=int) +# # attention +# check_argument('attention_type', c, restricted=is_tacotron(c), val_type=str, enum_list=['graves', 'original', 'dynamic_convolution']) +# check_argument('attention_heads', c, restricted=is_tacotron(c), val_type=int) +# check_argument('attention_norm', c, restricted=is_tacotron(c), val_type=str, enum_list=['sigmoid', 'softmax']) +# check_argument('windowing', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('use_forward_attn', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('forward_attn_mask', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('transition_agent', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('transition_agent', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('location_attn', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('bidirectional_decoder', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('double_decoder_consistency', c, restricted=is_tacotron(c), val_type=bool) +# check_argument('ddc_r', c, restricted='double_decoder_consistency' in c.keys(), min_val=1, max_val=7, val_type=int) - if c["model"].lower() in ["tacotron", "tacotron2"]: - # stopnet - check_argument("stopnet", c, restricted=is_tacotron(c), val_type=bool) - check_argument("separate_stopnet", c, restricted=is_tacotron(c), val_type=bool) +# if c['model'].lower() in ['tacotron', 'tacotron2']: +# # stopnet +# # check_argument('stopnet', c, restricted=is_tacotron(c), val_type=bool) +# # check_argument('separate_stopnet', c, restricted=is_tacotron(c), val_type=bool) - # Model Parameters for non-tacotron models - if c["model"].lower in ["speedy_speech", "align_tts"]: - check_argument("positional_encoding", c, restricted=True, val_type=type) - check_argument("encoder_type", c, restricted=True, val_type=str) - check_argument("encoder_params", c, restricted=True, val_type=dict) - check_argument("decoder_residual_conv_bn_params", c, restricted=True, val_type=dict) +# # Model Parameters for non-tacotron models +# if c['model'].lower in ["speedy_speech", "align_tts"]: +# check_argument('positional_encoding', c, restricted=True, val_type=type) +# check_argument('encoder_type', c, restricted=True, val_type=str) +# check_argument('encoder_params', c, restricted=True, val_type=dict) +# check_argument('decoder_residual_conv_bn_params', c, restricted=True, val_type=dict) - # GlowTTS parameters - check_argument("encoder_type", c, restricted=not is_tacotron(c), val_type=str) +# # GlowTTS parameters +# check_argument('encoder_type', c, restricted=not is_tacotron(c), val_type=str) - # tensorboard - check_argument("print_step", c, restricted=True, val_type=int, min_val=1) - check_argument("tb_plot_step", c, restricted=True, val_type=int, min_val=1) - check_argument("save_step", c, restricted=True, val_type=int, min_val=1) - check_argument("checkpoint", c, restricted=True, val_type=bool) - check_argument("tb_model_param_stats", c, restricted=True, val_type=bool) +# # tensorboard +# # check_argument('print_step', c, restricted=True, val_type=int, min_val=1) +# # check_argument('tb_plot_step', c, restricted=True, val_type=int, min_val=1) +# # check_argument('save_step', c, restricted=True, val_type=int, min_val=1) +# # check_argument('checkpoint', c, restricted=True, val_type=bool) +# # check_argument('tb_model_param_stats', c, restricted=True, val_type=bool) - # dataloading - # pylint: disable=import-outside-toplevel - from TTS.tts.utils.text import cleaners +# # dataloading +# # pylint: disable=import-outside-toplevel +# from TTS.tts.utils.text import cleaners +# # check_argument('text_cleaner', c, restricted=True, val_type=str, enum_list=dir(cleaners)) +# # check_argument('enable_eos_bos_chars', c, restricted=True, val_type=bool) +# # check_argument('num_loader_workers', c, restricted=True, val_type=int, min_val=0) +# # check_argument('num_val_loader_workers', c, restricted=True, val_type=int, min_val=0) +# # check_argument('batch_group_size', c, restricted=True, val_type=int, min_val=0) +# # check_argument('min_seq_len', c, restricted=True, val_type=int, min_val=0) +# # check_argument('max_seq_len', c, restricted=True, val_type=int, min_val=10) +# # check_argument('compute_input_seq_cache', c, restricted=True, val_type=bool) - check_argument("text_cleaner", c, restricted=True, val_type=str, enum_list=dir(cleaners)) - check_argument("enable_eos_bos_chars", c, restricted=True, val_type=bool) - check_argument("num_loader_workers", c, restricted=True, val_type=int, min_val=0) - check_argument("num_val_loader_workers", c, restricted=True, val_type=int, min_val=0) - check_argument("batch_group_size", c, restricted=True, val_type=int, min_val=0) - check_argument("min_seq_len", c, restricted=True, val_type=int, min_val=0) - check_argument("max_seq_len", c, restricted=True, val_type=int, min_val=10) - check_argument("compute_input_seq_cache", c, restricted=True, val_type=bool) +# # paths +# # check_argument('output_path', c, restricted=True, val_type=str) - # paths - check_argument("output_path", c, restricted=True, val_type=str) +# # multi-speaker and gst +# # check_argument('use_speaker_embedding', c, restricted=True, val_type=bool) +# # check_argument('use_external_speaker_embedding_file', c, restricted=c['use_speaker_embedding'], val_type=bool) +# # check_argument('external_speaker_embedding_file', c, restricted=c['use_external_speaker_embedding_file'], val_type=str) +# if c['model'].lower() in ['tacotron', 'tacotron2'] and c['use_gst']: +# # check_argument('use_gst', c, restricted=is_tacotron(c), val_type=bool) +# # check_argument('gst', c, restricted=is_tacotron(c), val_type=dict) +# # check_argument('gst_style_input', c['gst'], restricted=is_tacotron(c), val_type=[str, dict]) +# # check_argument('gst_embedding_dim', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=0, max_val=1000) +# # check_argument('gst_use_speaker_embedding', c['gst'], restricted=is_tacotron(c), val_type=bool) +# # check_argument('gst_num_heads', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=2, max_val=10) +# # check_argument('gst_num_style_tokens', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=1, max_val=1000) - # multi-speaker and gst - check_argument("use_speaker_embedding", c, restricted=True, val_type=bool) - check_argument("use_external_speaker_embedding_file", c, restricted=c["use_speaker_embedding"], val_type=bool) - check_argument( - "external_speaker_embedding_file", c, restricted=c["use_external_speaker_embedding_file"], val_type=str - ) - if c["model"].lower() in ["tacotron", "tacotron2"] and c["use_gst"]: - check_argument("use_gst", c, restricted=is_tacotron(c), val_type=bool) - check_argument("gst", c, restricted=is_tacotron(c), val_type=dict) - check_argument("gst_style_input", c["gst"], restricted=is_tacotron(c), val_type=[str, dict]) - check_argument("gst_embedding_dim", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=0, max_val=1000) - check_argument("gst_use_speaker_embedding", c["gst"], restricted=is_tacotron(c), val_type=bool) - check_argument("gst_num_heads", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=2, max_val=10) - check_argument("gst_style_tokens", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=1, max_val=1000) - - # datasets - checking only the first entry - check_argument("datasets", c, restricted=True, val_type=list) - for dataset_entry in c["datasets"]: - check_argument("name", dataset_entry, restricted=True, val_type=str) - check_argument("path", dataset_entry, restricted=True, val_type=str) - check_argument("meta_file_train", dataset_entry, restricted=True, val_type=[str, list]) - check_argument("meta_file_val", dataset_entry, restricted=True, val_type=str) +# # datasets - checking only the first entry +# # check_argument('datasets', c, restricted=True, val_type=list) +# # for dataset_entry in c['datasets']: +# # check_argument('name', dataset_entry, restricted=True, val_type=str) +# # check_argument('path', dataset_entry, restricted=True, val_type=str) +# # check_argument('meta_file_train', dataset_entry, restricted=True, val_type=[str, list]) +# # check_argument('meta_file_val', dataset_entry, restricted=True, val_type=str) diff --git a/TTS/tts/utils/speakers.py b/TTS/tts/utils/speakers.py old mode 100644 new mode 100755 index 583b46a8..84da1f72 --- a/TTS/tts/utils/speakers.py +++ b/TTS/tts/utils/speakers.py @@ -6,9 +6,9 @@ from typing import Union import numpy as np import torch +from TTS.config import load_config from TTS.speaker_encoder.utils.generic_utils import setup_model from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config def make_speakers_json_path(out_path): @@ -28,9 +28,10 @@ def load_speaker_mapping(out_path): def save_speaker_mapping(out_path, speaker_mapping): """Saves speaker mapping if not yet present.""" - speakers_json_path = make_speakers_json_path(out_path) - with open(speakers_json_path, "w") as f: - json.dump(speaker_mapping, f, indent=4) + if out_path is not None: + speakers_json_path = make_speakers_json_path(out_path) + with open(speakers_json_path, "w") as f: + json.dump(speaker_mapping, f, indent=4) def get_speakers(items): diff --git a/TTS/tts/utils/synthesis.py b/TTS/tts/utils/synthesis.py index f2cfbd43..9f417a1d 100644 --- a/TTS/tts/utils/synthesis.py +++ b/TTS/tts/utils/synthesis.py @@ -23,8 +23,8 @@ def text_to_seqvec(text, CONFIG): text_cleaner, CONFIG.phoneme_language, CONFIG.enable_eos_bos_chars, - tp=CONFIG.characters if "characters" in CONFIG.keys() else None, - add_blank=CONFIG["add_blank"] if "add_blank" in CONFIG.keys() else False, + tp=CONFIG.characters, + add_blank=CONFIG.add_blank, ), dtype=np.int32, ) @@ -33,8 +33,8 @@ def text_to_seqvec(text, CONFIG): text_to_sequence( text, text_cleaner, - tp=CONFIG.characters if "characters" in CONFIG.keys() else None, - add_blank=CONFIG["add_blank"] if "add_blank" in CONFIG.keys() else False, + tp=CONFIG.characters, + add_blank=CONFIG.add_blank, ), dtype=np.int32, ) @@ -65,28 +65,31 @@ def compute_style_mel(style_wav, ap, cuda=False): def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None, speaker_embeddings=None): - speaker_embedding_g = speaker_id if speaker_id is not None else speaker_embeddings if "tacotron" in CONFIG.model.lower(): - if not CONFIG.use_gst: - style_mel = None - - if truncated: - decoder_output, postnet_output, alignments, stop_tokens = model.inference_truncated( - inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings - ) - else: + if CONFIG.gst: decoder_output, postnet_output, alignments, stop_tokens = model.inference( inputs, style_mel=style_mel, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings ) + else: + if truncated: + decoder_output, postnet_output, alignments, stop_tokens = model.inference_truncated( + inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings + ) + else: + decoder_output, postnet_output, alignments, stop_tokens = model.inference( + inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings + ) elif "glow" in CONFIG.model.lower(): inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) # pylint: disable=not-callable if hasattr(model, "module"): # distributed model postnet_output, _, _, _, alignments, _, _ = model.module.inference( - inputs, inputs_lengths, g=speaker_embedding_g + inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings ) else: - postnet_output, _, _, _, alignments, _, _ = model.inference(inputs, inputs_lengths, g=speaker_embedding_g) + postnet_output, _, _, _, alignments, _, _ = model.inference( + inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings + ) postnet_output = postnet_output.permute(0, 2, 1) # these only belong to tacotron models. decoder_output = None @@ -95,9 +98,13 @@ def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) # pylint: disable=not-callable if hasattr(model, "module"): # distributed model - postnet_output, alignments = model.module.inference(inputs, inputs_lengths, g=speaker_embedding_g) + postnet_output, alignments = model.module.inference( + inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings + ) else: - postnet_output, alignments = model.inference(inputs, inputs_lengths, g=speaker_embedding_g) + postnet_output, alignments = model.inference( + inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings + ) postnet_output = postnet_output.permute(0, 2, 1) # these only belong to tacotron models. decoder_output = None @@ -108,7 +115,7 @@ def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel def run_model_tf(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None): - if CONFIG.use_gst and style_mel is not None: + if CONFIG.gst and style_mel is not None: raise NotImplementedError(" [!] GST inference not implemented for TF") if truncated: raise NotImplementedError(" [!] Truncated inference not implemented for TF") @@ -120,7 +127,7 @@ def run_model_tf(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=No def run_model_tflite(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None): - if CONFIG.use_gst and style_mel is not None: + if CONFIG.gst and style_mel is not None: raise NotImplementedError(" [!] GST inference not implemented for TfLite") if truncated: raise NotImplementedError(" [!] Truncated inference not implemented for TfLite") @@ -249,7 +256,7 @@ def synthesis( """ # GST processing style_mel = None - if "use_gst" in CONFIG.keys() and CONFIG.use_gst and style_wav is not None: + if CONFIG.has("gst") and CONFIG.gst and style_wav is not None: if isinstance(style_wav, dict): style_mel = style_wav else: diff --git a/TTS/tts/utils/text/__init__.py b/TTS/tts/utils/text/__init__.py index 1000b191..9367e6e2 100644 --- a/TTS/tts/utils/text/__init__.py +++ b/TTS/tts/utils/text/__init__.py @@ -2,12 +2,10 @@ import re -import phonemizer from packaging import version -from phonemizer.phonemize import phonemize -from TTS.tts.utils.chinese_mandarin.phonemizer import chinese_text_to_phonemes from TTS.tts.utils.text import cleaners +from TTS.tts.utils.text.chinese_mandarin.phonemizer import chinese_text_to_phonemes from TTS.tts.utils.text.symbols import _bos, _eos, _punctuations, make_symbols, phonemes, symbols # pylint: disable=unnecessary-comprehension @@ -28,62 +26,20 @@ PHONEME_PUNCTUATION_PATTERN = r"[" + _punctuations.replace(" ", "") + "]+" def text2phone(text, language): - """Convert graphemes to phonemes. For most of the languages, it calls - the phonemizer python library that calls espeak/espeak-ng. For chinese - mandarin, it calls pypinyin + custom function for phonemizing - Parameters: - text (str): text to phonemize - language (str): language of the text - Returns: - ph (str): phonemes as a string seperated by "|" - ph = "ɪ|g|ˈ|z|æ|m|p|ə|l" + """Convert graphemes to phonemes. + Parameters: + text (str): text to phonemize + language (str): language of the text + Returns: + ph (str): phonemes as a string seperated by "|" + ph = "ɪ|g|ˈ|z|æ|m|p|ə|l" """ # TO REVIEW : How to have a good implementation for this? if language == "zh-CN": ph = chinese_text_to_phonemes(text) return ph - - seperator = phonemizer.separator.Separator(" |", "", "|") - # try: - punctuations = re.findall(PHONEME_PUNCTUATION_PATTERN, text) - if version.parse(phonemizer.__version__) < version.parse("2.1"): - ph = phonemize(text, separator=seperator, strip=False, njobs=1, backend="espeak", language=language) - ph = ph[:-1].strip() # skip the last empty character - # phonemizer does not tackle punctuations. Here we do. - # Replace \n with matching punctuations. - if punctuations: - # if text ends with a punctuation. - if text[-1] == punctuations[-1]: - for punct in punctuations[:-1]: - ph = ph.replace("| |\n", "|" + punct + "| |", 1) - ph = ph + punctuations[-1] - else: - for punct in punctuations: - ph = ph.replace("| |\n", "|" + punct + "| |", 1) - elif version.parse(phonemizer.__version__) >= version.parse("2.1"): - ph = phonemize( - text, - separator=seperator, - strip=False, - njobs=1, - backend="espeak", - language=language, - preserve_punctuation=True, - language_switch="remove-flags", - ) - # this is a simple fix for phonemizer. - # https://github.com/bootphon/phonemizer/issues/32 - if punctuations: - for punctuation in punctuations: - ph = ph.replace(f"| |{punctuation} ", f"|{punctuation}| |").replace( - f"| |{punctuation}", f"|{punctuation}| |" - ) - ph = ph[:-3] - else: - raise RuntimeError(" [!] Use 'phonemizer' version 2.1 or older.") - - return ph + raise ValueError(f" [!] Language {language} is not supported for phonemization.") def intersperse(sequence, token): diff --git a/TTS/tts/utils/chinese_mandarin/__init__.py b/TTS/tts/utils/text/chinese_mandarin/__init__.py similarity index 100% rename from TTS/tts/utils/chinese_mandarin/__init__.py rename to TTS/tts/utils/text/chinese_mandarin/__init__.py diff --git a/TTS/tts/utils/chinese_mandarin/numbers.py b/TTS/tts/utils/text/chinese_mandarin/numbers.py similarity index 100% rename from TTS/tts/utils/chinese_mandarin/numbers.py rename to TTS/tts/utils/text/chinese_mandarin/numbers.py diff --git a/TTS/tts/utils/chinese_mandarin/phonemizer.py b/TTS/tts/utils/text/chinese_mandarin/phonemizer.py similarity index 100% rename from TTS/tts/utils/chinese_mandarin/phonemizer.py rename to TTS/tts/utils/text/chinese_mandarin/phonemizer.py diff --git a/TTS/tts/utils/chinese_mandarin/pinyinToPhonemes.py b/TTS/tts/utils/text/chinese_mandarin/pinyinToPhonemes.py similarity index 100% rename from TTS/tts/utils/chinese_mandarin/pinyinToPhonemes.py rename to TTS/tts/utils/text/chinese_mandarin/pinyinToPhonemes.py diff --git a/TTS/tts/utils/text/cleaners.py b/TTS/tts/utils/text/cleaners.py index d61738a6..2eddcdb8 100644 --- a/TTS/tts/utils/text/cleaners.py +++ b/TTS/tts/utils/text/cleaners.py @@ -14,7 +14,7 @@ import re from unidecode import unidecode -from TTS.tts.utils.chinese_mandarin.numbers import replace_numbers_to_characters_in_text +from TTS.tts.utils.text.chinese_mandarin.numbers import replace_numbers_to_characters_in_text from .abbreviations import abbreviations_en, abbreviations_fr from .number_norm import normalize_numbers diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index af0a1598..1b5a424b 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -4,20 +4,20 @@ import argparse import glob -import json import os import re import torch +from TTS.config import load_config from TTS.tts.utils.text.symbols import parse_symbols from TTS.utils.console_logger import ConsoleLogger from TTS.utils.generic_utils import create_experiment_folder, get_git_branch -from TTS.utils.io import copy_model_files, load_config +from TTS.utils.io import copy_model_files from TTS.utils.tensorboard_logger import TensorboardLogger -def parse_arguments(argv): +def init_arguments(argv): """Parse command line arguments of training scripts. Args: @@ -56,7 +56,7 @@ def parse_arguments(argv): parser.add_argument("--rank", type=int, default=0, help="DISTRIBUTED: process rank for distributed training.") parser.add_argument("--group_id", type=str, default="", help="DISTRIBUTED: process group id.") - return parser.parse_args() + return parser def get_last_checkpoint(path): @@ -117,16 +117,11 @@ def get_last_checkpoint(path): return last_models["checkpoint"], last_models["best_model"] -def process_args(args, model_class): - """Process parsed comand line arguments based on model class (tts or vocoder). +def process_args(args): + """Process parsed comand line arguments. Args: args (argparse.Namespace or dict like): Parsed input arguments. - model_type (str): Model type used to check config parameters and setup - the TensorBoard logger. One of ['tts', 'vocoder']. - - Raises: - ValueError: If `model_type` is not one of implemented choices. Returns: c (TTS.utils.io.AttrDict): Config paramaters. @@ -137,29 +132,26 @@ def process_args(args, model_class): tb_logger (TTS.utils.tensorboard.TensorboardLogger): Class that does the TensorBoard loggind. """ + if isinstance(args, tuple): + args, coqpit_overrides = args if args.continue_path: - args.output_path = args.continue_path + # continue a previous training from its output folder + experiment_path = args.continue_path args.config_path = os.path.join(args.continue_path, "config.json") args.restore_path, best_model = get_last_checkpoint(args.continue_path) if not args.best_path: args.best_path = best_model - # setup output paths and read configs - c = load_config(args.config_path) - _ = os.path.dirname(os.path.realpath(__file__)) - - if "mixed_precision" in c and c.mixed_precision: + config = load_config(args.config_path) + # override values from command-line args + config.parse_known_args(coqpit_overrides, relaxed_parser=True) + if config.mixed_precision: print(" > Mixed precision mode is ON") - - out_path = args.continue_path - if not out_path: - out_path = create_experiment_folder(c.output_path, c.run_name, args.debug) - - audio_path = os.path.join(out_path, "test_audios") - - c_logger = ConsoleLogger() - tb_logger = None - + experiment_path = args.continue_path + if not experiment_path: + experiment_path = create_experiment_folder(config.output_path, config.run_name, args.debug) + audio_path = os.path.join(experiment_path, "test_audios") + # setup rank 0 process in distributed training if args.rank == 0: os.makedirs(audio_path, exist_ok=True) new_fields = {} @@ -169,18 +161,22 @@ def process_args(args, model_class): # if model characters are not set in the config file # save the default set to the config file for future # compatibility. - if model_class == "tts" and "characters" not in c: + if config.has("characters_config"): used_characters = parse_symbols() new_fields["characters"] = used_characters - copy_model_files(c, args.config_path, out_path, new_fields) + copy_model_files(config, experiment_path, new_fields) os.chmod(audio_path, 0o775) - os.chmod(out_path, 0o775) + os.chmod(experiment_path, 0o775) + tb_logger = TensorboardLogger(experiment_path, model_name=config.model) + # write model desc to tensorboard + tb_logger.tb_add_text("model-config", f"
{config.to_json()}
", 0) + c_logger = ConsoleLogger() + return config, experiment_path, audio_path, c_logger, tb_logger - log_path = out_path - tb_logger = TensorboardLogger(log_path, model_name=model_class.upper()) - - # write model config to tensorboard - tb_logger.tb_add_text("model-config", f"
{json.dumps(c, indent=4)}
", 0) - - return c, out_path, audio_path, c_logger, tb_logger +def init_training(argv): + """Initialization of a training run.""" + parser = init_arguments(argv) + args = parser.parse_known_args() + config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args) + return args[0], config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger diff --git a/TTS/utils/audio.py b/TTS/utils/audio.py index cb1341f4..222b4c74 100644 --- a/TTS/utils/audio.py +++ b/TTS/utils/audio.py @@ -21,6 +21,7 @@ class AudioProcessor(object): sample_rate (int, optional): target audio sampling rate. Defaults to None. resample (bool, optional): enable/disable resampling of the audio clips when the target sampling rate does not match the original sampling rate. Defaults to False. num_mels (int, optional): number of melspectrogram dimensions. Defaults to None. + log_func (int, optional): log exponent used for converting spectrogram aplitude to DB. min_level_db (int, optional): minimum db threshold for the computed melspectrograms. Defaults to None. frame_shift_ms (int, optional): milliseconds of frames between STFT columns. Defaults to None. frame_length_ms (int, optional): milliseconds of STFT window length. Defaults to None. diff --git a/TTS/utils/generic_utils.py b/TTS/utils/generic_utils.py index 22287d14..a562e86f 100644 --- a/TTS/utils/generic_utils.py +++ b/TTS/utils/generic_utils.py @@ -1,11 +1,22 @@ +# -*- coding: utf-8 -*- import datetime import glob +import importlib import os +import re import shutil import subprocess import sys from pathlib import Path +import torch + + +def get_cuda(): + use_cuda = torch.cuda.is_available() + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + return use_cuda, device + def get_git_branch(): try: @@ -66,6 +77,20 @@ def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad) +def to_camel(text): + text = text.capitalize() + text = re.sub(r"(?!^)_([a-zA-Z])", lambda m: m.group(1).upper(), text) + text = text.replace("Tts", "TTS") + return text + + +def find_module(module_path: str, module_name: str) -> object: + module_name = module_name.lower() + module = importlib.import_module(module_path + "." + module_name) + class_name = to_camel(module_name) + return getattr(module, class_name) + + def get_user_data_dir(appname): if sys.platform == "win32": import winreg # pylint: disable=import-outside-toplevel @@ -92,7 +117,7 @@ def set_init_dict(model_dict, checkpoint_state, c): # 2. filter out different size layers pretrained_dict = {k: v for k, v in pretrained_dict.items() if v.numel() == model_dict[k].numel()} # 3. skip reinit layers - if c.reinit_layers is not None: + if c.has("reinit_layers") and c.reinit_layers is not None: for reinit_layer_name in c.reinit_layers: pretrained_dict = {k: v for k, v in pretrained_dict.items() if reinit_layer_name not in k} # 4. overwrite entries in the existing state dict @@ -137,29 +162,3 @@ class KeepAverage: def update_values(self, value_dict): for key, value in value_dict.items(): self.update_value(key, value) - - -def check_argument( - name, c, enum_list=None, max_val=None, min_val=None, restricted=False, val_type=None, alternative=None -): - if alternative in c.keys() and c[alternative] is not None: - return - if restricted: - assert name in c.keys(), f" [!] {name} not defined in config.json" - if name in c.keys(): - if max_val: - assert c[name] <= max_val, f" [!] {name} is larger than max value {max_val}" - if min_val: - assert c[name] >= min_val, f" [!] {name} is smaller than min value {min_val}" - if enum_list: - assert c[name].lower() in enum_list, f" [!] {name} is not a valid value" - if isinstance(val_type, list): - is_valid = False - for typ in val_type: - if isinstance(c[name], typ): - is_valid = True - assert is_valid or c[name] is None, f" [!] {name} has wrong type - {type(c[name])} vs {val_type}" - elif val_type: - assert ( - isinstance(c[name], val_type) or c[name] is None - ), f" [!] {name} has wrong type - {type(c[name])} vs {val_type}" diff --git a/TTS/utils/io.py b/TTS/utils/io.py index 84493e07..62d972f1 100644 --- a/TTS/utils/io.py +++ b/TTS/utils/io.py @@ -1,11 +1,7 @@ -import json import os import pickle as pickle_tts -import re from shutil import copyfile -import yaml - class RenamingUnpickler(pickle_tts.Unpickler): """Overload default pickler to solve module renaming problem""" @@ -23,64 +19,25 @@ class AttrDict(dict): self.__dict__ = self -def read_json_with_comments(json_path): - # fallback to json - with open(json_path, "r", encoding="utf-8") as f: - input_str = f.read() - # handle comments - input_str = re.sub(r"\\\n", "", input_str) - input_str = re.sub(r"//.*\n", "\n", input_str) - data = json.loads(input_str) - return data - - -def load_config(config_path: str) -> AttrDict: - """Load config files and discard comments - - Args: - config_path (str): path to config file. - """ - config = AttrDict() - - ext = os.path.splitext(config_path)[1] - if ext in (".yml", ".yaml"): - with open(config_path, "r", encoding="utf-8") as f: - data = yaml.safe_load(f) - else: - data = read_json_with_comments(config_path) - config.update(data) - return config - - -def copy_model_files(c, config_file, out_path, new_fields): +def copy_model_files(config, out_path, new_fields): """Copy config.json and other model files to training folder and add new fields. Args: - c (dict): model config from config.json. - config_file (str): path to config file. + config (Coqpit): Coqpit config defining the training run. out_path (str): output path to copy the file. new_fields (dict): new fileds to be added or edited in the config file. """ - # copy config.json copy_config_path = os.path.join(out_path, "config.json") - config_lines = open(config_file, "r", encoding="utf-8").readlines() # add extra information fields - for key, value in new_fields.items(): - if isinstance(value, str): - new_line = '"{}":"{}",\n'.format(key, value) - else: - new_line = '"{}":{},\n'.format(key, json.dumps(value, ensure_ascii=False)) - config_lines.insert(1, new_line) - config_out_file = open(copy_config_path, "w", encoding="utf-8") - config_out_file.writelines(config_lines) - config_out_file.close() + config.update(new_fields, allow_new=True) + config.save_json(copy_config_path) # copy model stats file if available - if c.audio["stats_path"] is not None: + if config.audio.stats_path is not None: copy_stats_path = os.path.join(out_path, "scale_stats.npy") if not os.path.exists(copy_stats_path): copyfile( - c.audio["stats_path"], + config.audio.stats_path, copy_stats_path, ) diff --git a/TTS/utils/manage.py b/TTS/utils/manage.py index 0cf69706..2e3caa81 100644 --- a/TTS/utils/manage.py +++ b/TTS/utils/manage.py @@ -8,8 +8,8 @@ from shutil import copyfile import gdown import requests +from TTS.config import load_config from TTS.utils.generic_utils import get_user_data_dir -from TTS.utils.io import load_config class ModelManager(object): @@ -101,6 +101,11 @@ class ModelManager(object): output_path = os.path.join(self.output_prefix, model_full_name) output_model_path = os.path.join(output_path, "model_file.pth.tar") output_config_path = os.path.join(output_path, "config.json") + # NOTE : band-aid for removing phoneme support + if "needs_phonemizer" in model_item and model_item["needs_phonemizer"]: + raise RuntimeError( + " [!] Use 🐸TTS <= v0.0.13 for this model. Current version does not support phoneme based models." + ) if os.path.exists(output_path): print(f" > {model_name} is already downloaded.") else: @@ -125,17 +130,15 @@ class ModelManager(object): # set scale stats path in config.json config_path = output_config_path config = load_config(config_path) - config["audio"]["stats_path"] = output_stats_path - with open(config_path, "w") as jf: - json.dump(config, jf) + config.audio.stats_path = output_stats_path + config.save_json(config_path) # update the speakers.json file path in the model config.json to the current path if os.path.exists(output_speakers_path): # set scale stats path in config.json config_path = output_config_path config = load_config(config_path) - config["external_speaker_embedding_file"] = output_speakers_path - with open(config_path, "w") as jf: - json.dump(config, jf) + config.external_speaker_embedding_file = output_speakers_path + config.save_json(config_path) return output_model_path, output_config_path, model_item def _download_gdrive_file(self, gdrive_idx, output): diff --git a/TTS/utils/synthesizer.py b/TTS/utils/synthesizer.py index 323231bf..bca3df31 100644 --- a/TTS/utils/synthesizer.py +++ b/TTS/utils/synthesizer.py @@ -5,6 +5,7 @@ import numpy as np import pysbd import torch +from TTS.config import load_config from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.speakers import SpeakerManager @@ -13,7 +14,6 @@ from TTS.tts.utils.speakers import SpeakerManager from TTS.tts.utils.synthesis import synthesis, trim_silence from TTS.tts.utils.text import make_symbols, phonemes, symbols from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config from TTS.vocoder.utils.generic_utils import interpolate_vocoder_input, setup_generator @@ -113,12 +113,11 @@ class Synthesizer(object): # pylint: disable=global-statement global symbols, phonemes - self.tts_config = load_config(tts_config_path) self.use_phonemes = self.tts_config.use_phonemes self.ap = AudioProcessor(verbose=False, **self.tts_config.audio) - if "characters" in self.tts_config.keys(): + if self.tts_config.has("characters") and self.tts_config.characters: symbols, phonemes = make_symbols(**self.tts_config.characters) if self.use_phonemes: @@ -151,7 +150,7 @@ class Synthesizer(object): use_cuda (bool): enable/disable CUDA use. """ self.vocoder_config = load_config(model_config) - self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config["audio"]) + self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config.audio) self.vocoder_model = setup_generator(self.vocoder_config) self.vocoder_model.load_checkpoint(self.vocoder_config, model_file, eval=True) if use_cuda: diff --git a/TTS/vocoder/configs/__init__.py b/TTS/vocoder/configs/__init__.py new file mode 100644 index 00000000..b5e11b99 --- /dev/null +++ b/TTS/vocoder/configs/__init__.py @@ -0,0 +1,17 @@ +import importlib +import os +from inspect import isclass + +# import all files under configs/ +configs_dir = os.path.dirname(__file__) +for file in os.listdir(configs_dir): + path = os.path.join(configs_dir, file) + if not file.startswith("_") and not file.startswith(".") and (file.endswith(".py") or os.path.isdir(path)): + config_name = file[: file.find(".py")] if file.endswith(".py") else file + module = importlib.import_module("TTS.vocoder.configs." + config_name) + for attribute_name in dir(module): + attribute = getattr(module, attribute_name) + + if isclass(attribute): + # Add the class to this package's variables + globals()[attribute_name] = attribute diff --git a/TTS/vocoder/configs/fullband_melgan_config.py b/TTS/vocoder/configs/fullband_melgan_config.py new file mode 100644 index 00000000..53444214 --- /dev/null +++ b/TTS/vocoder/configs/fullband_melgan_config.py @@ -0,0 +1,106 @@ +from dataclasses import dataclass, field + +from .shared_configs import BaseGANVocoderConfig + + +@dataclass +class FullbandMelganConfig(BaseGANVocoderConfig): + """Defines parameters for FullBand MelGAN vocoder. + + Example: + + >>> from TTS.vocoder.configs import FullbandMelganConfig + >>> config = FullbandMelganConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `melgan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'melgan_multiscale_discriminator`. + discriminator_model_params (dict): The discriminator model parameters. Defaults to + '{"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]}` + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `melgan_generator`. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + use_stft_loss (bool): + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): STFT loss parameters. Default to + `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + """ + + model: str = "melgan" + + # Model specific params + discriminator_model: str = "melgan_multiscale_discriminator" + discriminator_model_params: dict = field( + default_factory=lambda: {"base_channels": 16, "max_channels": 512, "downsample_factors": [4, 4, 4]} + ) + generator_model: str = "melgan_generator" + generator_model_params: dict = field( + default_factory=lambda: {"upsample_factors": [8, 8, 2, 2], "num_res_blocks": 4} + ) + + # Training - overrides + batch_size: int = 16 + seq_len: int = 8192 + pad_short: int = 2000 + use_noise_augment: bool = True + use_cache: bool = True + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = True + use_subband_stft_loss: bool = False + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = True # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = False + + stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240], + } + ) + + # loss weights - overrides + stft_loss_weight: float = 0.5 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 2.5 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 108 + l1_spec_loss_weight: float = 0.0 diff --git a/TTS/vocoder/configs/hifigan_config.py b/TTS/vocoder/configs/hifigan_config.py new file mode 100644 index 00000000..f76bb14c --- /dev/null +++ b/TTS/vocoder/configs/hifigan_config.py @@ -0,0 +1,138 @@ +from dataclasses import dataclass, field + +from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig + + +@dataclass +class HifiganConfig(BaseGANVocoderConfig): + """Defines parameters for FullBand MelGAN vocoder. + + Example: + + >>> from TTS.vocoder.configs import HifiganConfig + >>> config = HifiganConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `hifigan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'hifigan_discriminator`. + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `hifigan_generator`. + generator_model_params (dict): Parameters of the generator model. Defaults to + ` + { + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None, + } + ` + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + use_stft_loss (bool): + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): + STFT loss parameters. Default to + `{ + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }` + l1_spec_loss_params (dict): + L1 spectrogram loss parameters. Default to + `{ + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None, + }` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + """ + + model: str = "hifigan" + # model specific params + discriminator_model: str = "hifigan_discriminator" + generator_model: str = "hifigan_generator" + generator_model_params: dict = field( + default_factory=lambda: { + "upsample_factors": [8, 8, 2, 2], + "upsample_kernel_sizes": [16, 16, 4, 4], + "upsample_initial_channel": 512, + "resblock_kernel_sizes": [3, 7, 11], + "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]], + "resblock_type": "1", + } + ) + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = False + use_subband_stft_loss: bool = False + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = True # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = True + + # loss weights - overrides + stft_loss_weight: float = 0 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 1 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 108 + l1_spec_loss_weight: float = 45 + l1_spec_loss_params: dict = field( + default_factory=lambda: { + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None, + } + ) + + # optimizer parameters + lr: float = 1e-4 + wd: float = 1e-6 diff --git a/TTS/vocoder/configs/hifigan_ljspeech.json b/TTS/vocoder/configs/hifigan_ljspeech.json deleted file mode 100644 index 23cbf3f8..00000000 --- a/TTS/vocoder/configs/hifigan_ljspeech.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - "run_name": "hifigan", - "run_description": "hifigan mean-var scaling", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "log_func": "np.log10", - "do_sound_norm": true, - - // Silence trimming - "do_trim_silence": false,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/.local/share/tts/tts_models--en--ljspeech--speedy-speech-wn/scale_stats.npy" - }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54324" - }, - - // MODEL PARAMETERS - "use_pqmf": false, - - // LOSS PARAMETERS - "use_stft_loss": false, - "use_subband_stft_loss": false, - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": true, // use only with melgan discriminators - "use_l1_spec_loss": true, - - // loss weights - "stft_loss_weight": 0, - "subband_stft_loss_weight": 0, - "mse_G_loss_weight": 1, - "hinge_G_loss_weight": 0, - "feat_match_loss_weight": 10, - "l1_spec_loss_weight": 45, - - // multiscale stft loss parameters - // "stft_loss_params": { - // "n_ffts": [1024, 2048, 512], - // "hop_lengths": [120, 240, 50], - // "win_lengths": [600, 1200, 240] - // }, - - "l1_spec_loss_params": { - "use_mel": true, - "sample_rate": 22050, - "n_fft": 1024, - "hop_length": 256, - "win_length": 1024, - "n_mels": 80, - "mel_fmin": 0.0, - "mel_fmax": null - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "hifigan_discriminator", - //"discriminator_model_params":{ - // "peroids": [2, 3, 5, 7, 11], - // "base_channels": 16, - // "max_channels":512, - // "downsample_factors":[4, 4, 4] - //}, - "steps_to_start_discriminator": 0, // steps required to start GAN trainining.1 - "diff_samples_for_G_and_D": false, // draw a new sample from the dataset for the D pass. - - // GENERATOR - "generator_model": "hifigan_generator", - "generator_model_params": { - "upsample_factors":[8,8,2,2], - "upsample_kernel_sizes": [16,16,4,4], - "upsample_initial_channel": 512, - "resblock_kernel_sizes": [3,7,11], - "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], - "resblock_type": "1" - }, - - // DATASET - "data_path": "/home/erogol/gdrive/Datasets/LJSpeech-1.1/wavs/", - "feature_path": null, - // "feature_path": "/home/erogol/gdrive/Datasets/non-binary-voice-files/tacotron-DCA/", - "seq_len": 8192, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 16, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/gdrive/Trainings/LJSpeech/" -} - - diff --git a/TTS/vocoder/configs/melgan_config.py b/TTS/vocoder/configs/melgan_config.py new file mode 100644 index 00000000..dc35b6f8 --- /dev/null +++ b/TTS/vocoder/configs/melgan_config.py @@ -0,0 +1,106 @@ +from dataclasses import dataclass, field + +from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig + + +@dataclass +class MelganConfig(BaseGANVocoderConfig): + """Defines parameters for MelGAN vocoder. + + Example: + + >>> from TTS.vocoder.configs import MelganConfig + >>> config = MelganConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `melgan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'melgan_multiscale_discriminator`. + discriminator_model_params (dict): The discriminator model parameters. Defaults to + '{"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]}` + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `melgan_generator`. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + use_stft_loss (bool): + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): STFT loss parameters. Default to + `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + """ + + model: str = "melgan" + + # Model specific params + discriminator_model: str = "melgan_multiscale_discriminator" + discriminator_model_params: dict = field( + default_factory=lambda: {"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]} + ) + generator_model: str = "melgan_generator" + generator_model_params: dict = field( + default_factory=lambda: {"upsample_factors": [8, 8, 2, 2], "num_res_blocks": 3} + ) + + # Training - overrides + batch_size: int = 16 + seq_len: int = 8192 + pad_short: int = 2000 + use_noise_augment: bool = True + use_cache: bool = True + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = True + use_subband_stft_loss: bool = False + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = True # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = False + + stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240], + } + ) + + # loss weights - overrides + stft_loss_weight: float = 0.5 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 2.5 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 108 + l1_spec_loss_weight: float = 0 diff --git a/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json b/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json deleted file mode 100644 index d52893f0..00000000 --- a/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "run_name": "multiband-melgan-rwd", - "run_description": "multiband melgan with random window discriminator from https://arxiv.org/pdf/1909.11646.pdf", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "num_freq": 513, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - - // MODEL PARAMETERS - "use_pqmf": true, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": true, - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - // loss weights - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - // multiscale stft loss parameters - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - - // subband multiscale stft loss parameters - "subband_stft_loss_params":{ - "n_ffts": [384, 683, 171], - "hop_lengths": [30, 60, 10], - "win_lengths": [150, 300, 60] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "random_window_discriminator", - "discriminator_model_params":{ - "uncond_disc_donwsample_factors": [8, 4], - "cond_disc_downsample_factors": [[8, 4, 2, 2, 2], [8, 4, 2, 2], [8, 4, 2], [8, 4], [4, 2, 2]], - "cond_disc_out_channels": [[128, 128, 256, 256], [128, 256, 256], [128, 256], [256], [128, 256]], - "window_sizes": [512, 1024, 2048, 4096, 8192] - }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - - // GENERATOR - "generator_model": "multiband_melgan_generator", - "generator_model_params": { - "upsample_factors":[8, 4, 2], - "num_res_blocks": 4 - }, - - // DATASET - "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", - "seq_len": 16384, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "warmup_steps_disc": 4000, - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/TTS/vocoder/configs/multiband_melgan_config.json b/TTS/vocoder/configs/multiband_melgan_config.json deleted file mode 100644 index 5aea4a61..00000000 --- a/TTS/vocoder/configs/multiband_melgan_config.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "run_name": "multiband-melgan", - "run_description": "multiband melgan mean-var scaling", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": true, // use only with multi-band models. - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - // loss weights - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - // multiscale stft loss parameters - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - - // subband multiscale stft loss parameters - "subband_stft_loss_params":{ - "n_ffts": [384, 683, 171], - "hop_lengths": [30, 60, 10], - "win_lengths": [150, 300, 60] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "melgan_multiscale_discriminator", - "discriminator_model_params":{ - "base_channels": 16, - "max_channels":512, - "downsample_factors":[4, 4, 4] - }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - - // GENERATOR - "generator_model": "multiband_melgan_generator", - "generator_model_params": { - "upsample_factors":[8, 4, 2], - "num_res_blocks": 4 - }, - - // DATASET - "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", - "feature_path": null, - "seq_len": 16384, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/TTS/vocoder/configs/multiband_melgan_config.py b/TTS/vocoder/configs/multiband_melgan_config.py new file mode 100644 index 00000000..81fd7904 --- /dev/null +++ b/TTS/vocoder/configs/multiband_melgan_config.py @@ -0,0 +1,144 @@ +from dataclasses import dataclass, field + +from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig + + +@dataclass +class MultibandMelganConfig(BaseGANVocoderConfig): + """Defines parameters for MultiBandMelGAN vocoder. + + Example: + + >>> from TTS.vocoder.configs import MultibandMelganConfig + >>> config = MultibandMelganConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `melgan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'melgan_multiscale_discriminator`. + discriminator_model_params (dict): The discriminator model parameters. Defaults to + '{ + "base_channels": 16, + "max_channels": 512, + "downsample_factors": [4, 4, 4] + }` + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `melgan_generator`. + generator_model_param (dict): + The generator model parameters. Defaults to `{"upsample_factors": [8, 4, 2], "num_res_blocks": 4}`. + use_pqmf (bool): + enable / disable PQMF modulation for multi-band training. Defaults to True. + lr_gen (float): + Initial learning rate for the generator model. Defaults to 0.0001. + lr_disc (float): + Initial learning rate for the discriminator model. Defaults to 0.0001. + optimizer (torch.optim.Optimizer): + Optimizer used for the training. Defaults to `AdamW`. + optimizer_params (dict): + Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}` + lr_scheduler_gen (torch.optim.Scheduler): + Learning rate scheduler for the generator. Defaults to `MultiStepLR`. + lr_scheduler_gen_params (dict): + Parameters for the generator learning rate scheduler. Defaults to + `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}`. + lr_scheduler_disc (torch.optim.Scheduler): + Learning rate scheduler for the discriminator. Defaults to `MultiStepLR`. + lr_scheduler_dict_params (dict): + Parameters for the discriminator learning rate scheduler. Defaults to + `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}`. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + steps_to_start_discriminator (int): + Number of steps required to start training the discriminator. Defaults to 0. + use_stft_loss (bool):` + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): STFT loss parameters. Default to + `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + """ + + model: str = "multiband_melgan" + + # Model specific params + discriminator_model: str = "melgan_multiscale_discriminator" + discriminator_model_params: dict = field( + default_factory=lambda: {"base_channels": 16, "max_channels": 512, "downsample_factors": [4, 4, 4]} + ) + generator_model: str = "multiband_melgan_generator" + generator_model_params: dict = field(default_factory=lambda: {"upsample_factors": [8, 4, 2], "num_res_blocks": 4}) + use_pqmf: bool = True + + # optimizer - overrides + lr_gen: float = 0.0001 # Initial learning rate. + lr_disc: float = 0.0001 # Initial learning rate. + optimizer: str = "AdamW" + optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0}) + lr_scheduler_gen: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_gen_params: dict = field( + default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]} + ) + lr_scheduler_disc: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_disc_params: dict = field( + default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]} + ) + + # Training - overrides + batch_size: int = 64 + seq_len: int = 16384 + pad_short: int = 2000 + use_noise_augment: bool = False + use_cache: bool = True + steps_to_start_discriminator: bool = 200000 + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = True + use_subband_stft_loss: bool = True + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = False # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = False + + subband_stft_loss_params: dict = field( + default_factory=lambda: {"n_ffts": [384, 683, 171], "hop_lengths": [30, 60, 10], "win_lengths": [150, 300, 60]} + ) + + # loss weights - overrides + stft_loss_weight: float = 0.5 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 2.5 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 108 + l1_spec_loss_weight: float = 0 diff --git a/TTS/vocoder/configs/parallel_wavegan_config.json b/TTS/vocoder/configs/parallel_wavegan_config.json deleted file mode 100644 index 5ea7dbcd..00000000 --- a/TTS/vocoder/configs/parallel_wavegan_config.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "run_name": "pwgan", - "run_description": "parallel-wavegan training", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - - // MODEL PARAMETERS - "use_pqmf": true, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": false, // USE ONLY WITH MULTIBAND MODELS - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - // loss weights - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - // multiscale stft loss parameters - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - - // subband multiscale stft loss parameters - "subband_stft_loss_params":{ - "n_ffts": [384, 683, 171], - "hop_lengths": [30, 60, 10], - "win_lengths": [150, 300, 60] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "parallel_wavegan_discriminator", - "discriminator_model_params":{ - "num_layers": 10 - }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - - // GENERATOR - "generator_model": "parallel_wavegan_generator", - "generator_model_params": { - "upsample_factors":[4, 4, 4, 4], - "stacks": 3, - "num_res_blocks": 30 - }, - - // DATASET - "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", - "feature_path": null, - "seq_len": 25600, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 6, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/TTS/vocoder/configs/parallel_wavegan_config.py b/TTS/vocoder/configs/parallel_wavegan_config.py new file mode 100644 index 00000000..d132d2e1 --- /dev/null +++ b/TTS/vocoder/configs/parallel_wavegan_config.py @@ -0,0 +1,130 @@ +from dataclasses import dataclass, field + +from .shared_configs import BaseGANVocoderConfig + + +@dataclass +class ParallelWaveganConfig(BaseGANVocoderConfig): + """Defines parameters for ParallelWavegan vocoder. + + Args: + model (str): + Model name used for selecting the right configuration at initialization. Defaults to `parallel_wavegan`. + discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to + 'parallel_wavegan_discriminator`. + discriminator_model_params (dict): The discriminator model kwargs. Defaults to + '{"num_layers": 10}` + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `parallel_wavegan_generator`. + generator_model_param (dict): + The generator model kwargs. Defaults to `{"upsample_factors": [4, 4, 4, 4], "stacks": 3, "num_res_blocks": 30}`. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 16. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 8192. + pad_short (int): + Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + steps_to_start_discriminator (int): + Number of steps required to start training the discriminator. Defaults to 0. + use_stft_loss (bool):` + enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True. + use_subband_stft (bool): + enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True. + use_mse_gan_loss (bool): + enable / disable using Mean Squeare Error GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models. + Defaults to False. + use_feat_match_loss (bool): + enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True. + use_l1_spec_loss (bool): + enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False. + stft_loss_params (dict): STFT loss parameters. Default to + `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}` + stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total + model loss. Defaults to 0.5. + subband_stft_loss_weight (float): + Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + mse_G_loss_weight (float): + MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5. + hinge_G_loss_weight (float): + Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + feat_match_loss_weight (float): + Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 0. + l1_spec_loss_weight (float): + L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0. + lr_gen (float): + Generator model initial learning rate. Defaults to 0.0002. + lr_disc (float): + Discriminator model initial learning rate. Defaults to 0.0002. + optimizer (torch.optim.Optimizer): + Optimizer used for the training. Defaults to `AdamW`. + optimizer_params (dict): + Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}` + lr_scheduler_gen (torch.optim.Scheduler): + Learning rate scheduler for the generator. Defaults to `ExponentialLR`. + lr_scheduler_gen_params (dict): + Parameters for the generator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`. + lr_scheduler_disc (torch.optim.Scheduler): + Learning rate scheduler for the discriminator. Defaults to `ExponentialLR`. + lr_scheduler_dict_params (dict): + Parameters for the discriminator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`. + """ + + model: str = "parallel_wavegan" + + # Model specific params + discriminator_model: str = "parallel_wavegan_discriminator" + discriminator_model_params: dict = field(default_factory=lambda: {"num_layers": 10}) + generator_model: str = "parallel_wavegan_generator" + generator_model_params: dict = field( + default_factory=lambda: {"upsample_factors": [4, 4, 4, 4], "stacks": 3, "num_res_blocks": 30} + ) + + # Training - overrides + batch_size: int = 6 + seq_len: int = 25600 + pad_short: int = 2000 + use_noise_augment: bool = False + use_cache: bool = True + steps_to_start_discriminator: int = 200000 + + # LOSS PARAMETERS - overrides + use_stft_loss: bool = True + use_subband_stft_loss: bool = False + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = False + use_feat_match_loss: bool = False # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = False + + stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240], + } + ) + + # loss weights - overrides + stft_loss_weight: float = 0.5 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 2.5 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 0 + l1_spec_loss_weight: float = 0 + + # optimizer overrides + lr_gen: float = 0.0002 # Initial learning rate. + lr_disc: float = 0.0002 # Initial learning rate. + optimizer: str = "AdamW" + optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0}) + lr_scheduler_gen: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_gen_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1}) + lr_scheduler_disc: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_disc_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1}) diff --git a/TTS/vocoder/configs/shared_configs.py b/TTS/vocoder/configs/shared_configs.py new file mode 100644 index 00000000..664032d2 --- /dev/null +++ b/TTS/vocoder/configs/shared_configs.py @@ -0,0 +1,183 @@ +from dataclasses import dataclass, field + +from coqpit import MISSING + +from TTS.config import BaseAudioConfig, BaseTrainingConfig + + +@dataclass +class BaseVocoderConfig(BaseTrainingConfig): + """Shared parameters among all the vocoder models. + Args: + audio (BaseAudioConfig): + Audio processor config instance. Defaultsto `BaseAudioConfig()`. + use_noise_augment (bool): + Augment the input audio with random noise. Defaults to False/ + eval_split_size (int): + Number of instances used for evaluation. Defaults to 10. + data_path (str): + Root path of the training data. All the audio files found recursively from this root path are used for + training. Defaults to MISSING. + feature_path (str): + Root path to the precomputed feature files. Defaults to None. + seq_len (int): + Length of the waveform segments used for training. Defaults to MISSING. + pad_short (int): + Extra padding for the waveforms shorter than `seq_len`. Defaults to 0. + conv_path (int): + Extra padding for the feature frames against convolution of the edge frames. Defaults to MISSING. + Defaults to 0. + use_cache (bool): + enable / disable in memory caching of the computed features. If the RAM is not enough, if may cause OOM. + Defaults to False. + epochs (int): + Number of training epochs to. Defaults to 10000. + wd (float): + Weight decay. + """ + + audio: BaseAudioConfig = field(default_factory=BaseAudioConfig) + # dataloading + use_noise_augment: bool = False # enable/disable random noise augmentation in spectrograms. + eval_split_size: int = 10 # number of samples used for evaluation. + # dataset + data_path: str = MISSING # root data path. It finds all wav files recursively from there. + feature_path: str = None # if you use precomputed features + seq_len: int = MISSING # signal length used in training. + pad_short: int = 0 # additional padding for short wavs + conv_pad: int = 0 # additional padding against convolutions applied to spectrograms + use_cache: bool = False # use in memory cache to keep the computed features. This might cause OOM. + # OPTIMIZER + epochs: int = 10000 # total number of epochs to train. + wd: float = 0.0 # Weight decay weight. + + +@dataclass +class BaseGANVocoderConfig(BaseVocoderConfig): + """Base config class used among all the GAN based vocoders. + Args: + use_stft_loss (bool): + enable / disable the use of STFT loss. Defaults to True. + use_subband_stft_loss (bool): + enable / disable the use of Subband STFT loss. Defaults to True. + use_mse_gan_loss (bool): + enable / disable the use of Mean Squared Error based GAN loss. Defaults to True. + use_hinge_gan_loss (bool): + enable / disable the use of Hinge GAN loss. Defaults to True. + use_feat_match_loss (bool): + enable / disable feature matching loss. Defaults to True. + use_l1_spec_loss (bool): + enable / disable L1 spectrogram loss. Defaults to True. + stft_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 0. + subband_stft_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 0. + mse_G_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 1. + hinge_G_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 0. + feat_match_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 100. + l1_spec_loss_weight (float): + Loss weight that multiplies the computed loss value. Defaults to 45. + stft_loss_params (dict): + Parameters for the STFT loss. Defaults to `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}`. + l1_spec_loss_params (dict): + Parameters for the L1 spectrogram loss. Defaults to + `{ + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None, + }` + target_loss (str): + Target loss name that defines the quality of the model. Defaults to `avg_G_loss`. + gen_clip_grad (float): + Gradient clipping threshold for the generator model. Any value less than 0 disables clipping. + Defaults to -1. + disc_clip_grad (float): + Gradient clipping threshold for the discriminator model. Any value less than 0 disables clipping. + Defaults to -1. + lr_gen (float): + Generator model initial learning rate. Defaults to 0.0002. + lr_disc (float): + Discriminator model initial learning rate. Defaults to 0.0002. + optimizer (torch.optim.Optimizer): + Optimizer used for the training. Defaults to `AdamW`. + optimizer_params (dict): + Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}` + lr_scheduler_gen (torch.optim.Scheduler): + Learning rate scheduler for the generator. Defaults to `ExponentialLR`. + lr_scheduler_gen_params (dict): + Parameters for the generator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`. + lr_scheduler_disc (torch.optim.Scheduler): + Learning rate scheduler for the discriminator. Defaults to `ExponentialLR`. + lr_scheduler_dict_params (dict): + Parameters for the discriminator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`. + use_pqmf (bool): + enable / disable PQMF for subband approximation at training. Defaults to False. + steps_to_start_discriminator (int): + Number of steps required to start training the discriminator. Defaults to 0. + diff_samples_for_G_and_D (bool): + enable / disable use of different training samples for the generator and the discriminator iterations. + Enabling it results in slower iterations but faster convergance in some cases. Defaults to False. + """ + + # LOSS PARAMETERS + use_stft_loss: bool = True + use_subband_stft_loss: bool = True + use_mse_gan_loss: bool = True + use_hinge_gan_loss: bool = True + use_feat_match_loss: bool = True # requires MelGAN Discriminators (MelGAN and HifiGAN) + use_l1_spec_loss: bool = True + + # loss weights + stft_loss_weight: float = 0 + subband_stft_loss_weight: float = 0 + mse_G_loss_weight: float = 1 + hinge_G_loss_weight: float = 0 + feat_match_loss_weight: float = 100 + l1_spec_loss_weight: float = 45 + + stft_loss_params: dict = field( + default_factory=lambda: { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240], + } + ) + + l1_spec_loss_params: dict = field( + default_factory=lambda: { + "use_mel": True, + "sample_rate": 22050, + "n_fft": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mels": 80, + "mel_fmin": 0.0, + "mel_fmax": None, + } + ) + + target_loss: str = "avg_G_loss" # loss value to pick the best model to save after each epoch + + # optimizer + gen_clip_grad: float = -1 # Generator gradient clipping threshold. Apply gradient clipping if > 0 + disc_clip_grad: float = -1 # Discriminator gradient clipping threshold. + lr_gen: float = 0.0002 # Initial learning rate. + lr_disc: float = 0.0002 # Initial learning rate. + optimizer: str = "AdamW" + optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0}) + lr_scheduler_gen: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_gen_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1}) + lr_scheduler_disc: str = "ExponentialLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_disc_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1}) + + use_pqmf: bool = False # enable/disable using pqmf for multi-band training. (Multi-band MelGAN) + steps_to_start_discriminator = 0 # start training the discriminator after this number of steps. + diff_samples_for_G_and_D: bool = False # use different samples for G and D training steps. diff --git a/TTS/vocoder/configs/universal_fullband_melgan.json b/TTS/vocoder/configs/universal_fullband_melgan.json deleted file mode 100644 index 245de2d3..00000000 --- a/TTS/vocoder/configs/universal_fullband_melgan.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "run_name": "fullband-melgan", - "run_description": "fullband melgan mean-var scaling", - - // AUDIO PARAMETERS - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 24000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/libritts/LibriTTS/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54324" - }, - - // MODEL PARAMETERS - "use_pqmf": false, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": false, - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - // loss weights - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - // multiscale stft loss parameters - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch - - // DISCRIMINATOR - "discriminator_model": "melgan_multiscale_discriminator", - "discriminator_model_params":{ - "base_channels": 16, - "max_channels":512, - "downsample_factors":[4, 4, 4] - }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - - // GENERATOR - "generator_model": "fullband_melgan_generator", - "generator_model_params": { - "upsample_factors":[8, 8, 4], - "num_res_blocks": 4 - }, - - // DATASET - "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/", - "feature_path": null, - "seq_len": 16384, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": false, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 48, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "wd": 0.0, // Weight decay weight. - "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "disc_clip_grad": -1, // Discriminator gradient clipping threshold. - "lr_gen": 0.0002, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0002, - "optimizer": "AdamW", - "optimizer_params":{ - "betas": [0.8, 0.99], - "weight_decay": 0.0 - }, - "lr_scheduler_gen": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_gen_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - "lr_scheduler_disc": "ExponentialLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_disc_params": { - "gamma": 0.999, - "last_epoch": -1 - }, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/" -} - - diff --git a/TTS/vocoder/configs/wavegrad_config.py b/TTS/vocoder/configs/wavegrad_config.py new file mode 100644 index 00000000..271422ee --- /dev/null +++ b/TTS/vocoder/configs/wavegrad_config.py @@ -0,0 +1,111 @@ +from dataclasses import dataclass, field + +from TTS.vocoder.configs.shared_configs import BaseVocoderConfig + + +@dataclass +class WavegradConfig(BaseVocoderConfig): + """Defines parameters for WaveGrad vocoder. + Example: + + >>> from TTS.vocoder.configs import WavegradConfig + >>> config = WavegradConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `wavegrad`. + generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `wavegrad`. + model_params (dict): + WaveGrad kwargs. Defaults to + ` + { + "use_weight_norm": True, + "y_conv_channels": 32, + "x_conv_channels": 768, + "ublock_out_channels": [512, 512, 256, 128, 128], + "dblock_out_channels": [128, 128, 256, 512], + "upsample_factors": [4, 4, 4, 2, 2], + "upsample_dilations": [[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8], [1, 2, 4, 8], [1, 2, 4, 8]], + } + ` + target_loss (str): + Target loss name that defines the quality of the model. Defaults to `avg_wavegrad_loss`. + epochs (int): + Number of epochs to traing the model. Defaults to 10000. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 96. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 6144. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + mixed_precision (bool): + enable / disable mixed precision training. Default is True. + eval_split_size (int): + Number of samples used for evalutaion. Defaults to 50. + train_noise_schedule (dict): + Training noise schedule. Defaults to + `{"min_val": 1e-6, "max_val": 1e-2, "num_steps": 1000}` + test_noise_schedule (dict): + Inference noise schedule. For a better performance, you may need to use `bin/tune_wavegrad.py` to find a + better schedule. Defaults to + ` + { + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 50, + } + ` + grad_clip (float): + Gradient clipping threshold. If <= 0.0, no clipping is applied. Defaults to 1.0 + lr (float): + Initila leraning rate. Defaults to 1e-4. + lr_scheduler (str): + One of the learning rate schedulers from `torch.optim.scheduler.*`. Defaults to `MultiStepLR`. + lr_scheduler_params (dict): + kwargs for the scheduler. Defaults to `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}` + """ + + model: str = "wavegrad" + # Model specific params + generator_model: str = "wavegrad" + model_params: dict = field( + default_factory=lambda: { + "use_weight_norm": True, + "y_conv_channels": 32, + "x_conv_channels": 768, + "ublock_out_channels": [512, 512, 256, 128, 128], + "dblock_out_channels": [128, 128, 256, 512], + "upsample_factors": [4, 4, 4, 2, 2], + "upsample_dilations": [[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8], [1, 2, 4, 8], [1, 2, 4, 8]], + } + ) + target_loss: str = "avg_wavegrad_loss" # loss value to pick the best model to save after each epoch + + # Training - overrides + epochs: int = 10000 + batch_size: int = 96 + seq_len: int = 6144 + use_cache: bool = True + mixed_precision: bool = True + eval_split_size: int = 50 + + # NOISE SCHEDULE PARAMS + train_noise_schedule: dict = field(default_factory=lambda: {"min_val": 1e-6, "max_val": 1e-2, "num_steps": 1000}) + + test_noise_schedule: dict = field( + default_factory=lambda: { # inference noise schedule. Try TTS/bin/tune_wavegrad.py to find the optimal values. + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 50, + } + ) + + # optimizer overrides + grad_clip: float = 1.0 + lr: float = 1e-4 # Initial learning rate. + lr_scheduler: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_params: dict = field( + default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]} + ) diff --git a/TTS/vocoder/configs/wavegrad_libritts.json b/TTS/vocoder/configs/wavegrad_libritts.json deleted file mode 100644 index ade20a8f..00000000 --- a/TTS/vocoder/configs/wavegrad_libritts.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "run_name": "wavegrad-libritts", - "run_description": "wavegrad libritts", - - "audio":{ - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 24000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/libritts/LibriTTS/scale_stats_wavegrad.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - "mixed_precision": true, // enable torch mixed precision training (true, false) - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54322" - }, - - "target_loss": "avg_wavegrad_loss", // loss value to pick the best model to save after each epoch - - // MODEL PARAMETERS - "generator_model": "wavegrad", - "model_params":{ - "use_weight_norm": true, - "y_conv_channels":32, - "x_conv_channels":768, - "ublock_out_channels": [512, 512, 256, 128, 128], - "dblock_out_channels": [128, 128, 256, 512], - "upsample_factors": [4, 4, 4, 2, 2], - "upsample_dilations": [ - [1, 2, 1, 2], - [1, 2, 1, 2], - [1, 2, 4, 8], - [1, 2, 4, 8], - [1, 2, 4, 8]] - }, - - // DATASET - "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/", // root data path. It finds all wav files recursively from there. - "feature_path": null, // if you use precomputed features - "seq_len": 6144, // 24 * hop_length - "pad_short": 0, // additional padding for short wavs - "conv_pad": 0, // additional padding against convolutions applied to spectrograms - "use_noise_augment": false, // add noise to the audio signal for augmentation - "use_cache": false, // use in memory cache to keep the computed features. This might cause OOM. - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 96, // Batch size for training. - - // NOISE SCHEDULE PARAMS - Only effective at training time. - "train_noise_schedule":{ - "min_val": 1e-6, - "max_val": 1e-2, - "num_steps": 1000 - }, - "test_noise_schedule":{ - "min_val": 1e-6, - "max_val": 1e-2, - "num_steps": 50 - }, - - // VALIDATION - "run_eval": true, // enable/disable evaluation run - - // OPTIMIZER - "epochs": 10000, // total number of epochs to train. - "clip_grad": 1.0, // Generator gradient clipping threshold. Apply gradient clipping if > 0 - "lr_scheduler": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_params": { - "gamma": 0.5, - "milestones": [100000, 200000, 300000, 400000, 500000, 600000] - }, - "lr": 1e-4, // Initial learning rate. If Noam decay is active, maximum learning rate. - - // TENSORBOARD and LOGGING - "print_step": 50, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 5000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 256, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/TTS/vocoder/configs/wavernn_config.json b/TTS/vocoder/configs/wavernn_config.json deleted file mode 100644 index aa2d7b9f..00000000 --- a/TTS/vocoder/configs/wavernn_config.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "run_name": "wavernn_librittts", - "run_description": "wavernn libritts training from LJSpeech model", - -// AUDIO PARAMETERS - "audio": { - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - // Audio processing parameters - "sample_rate": 24000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - // Silence trimming - "do_trim_silence": false, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 40.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 20.0, // scaler value appplied after log transform of spectrogram. - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - -// Generating / Synthesizing - "batched": true, - "target_samples": 11000, // target number of samples to be generated in each batch entry - "overlap_samples": 550, // number of samples for crossfading between batches - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - -// MODEL MODE - "mode": "mold", // mold [string], gauss [string], bits [int] - "mulaw": true, // apply mulaw if mode is bits - -// MODEL PARAMETERS - "wavernn_model_params": { - "rnn_dims": 512, - "fc_dims": 512, - "compute_dims": 128, - "res_out_dims": 128, - "num_res_blocks": 10, - "use_aux_net": true, - "use_upsample_net": true, - "upsample_factors": [4, 8, 8] // this needs to correctly factorise hop_length - }, - -// GENERATOR - for backward compatibility - "generator_model": "WaveRNN", - -// DATASET - //"use_gta": true, // use computed gta features from the tts model - "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/", // path containing training wav files - "feature_path": null, // path containing computed features from wav files if null compute them - "seq_len": 1280, // has to be devideable by hop_length - "padding": 2, // pad the input for resnet to see wider input length - -// TRAINING - "batch_size": 256, // Batch size for training. - "epochs": 10000, // total number of epochs to train. - "mixed_precision": true, // enable/ disable mixed precision training - -// VALIDATION - "run_eval": true, - "test_every_epochs": 10, // Test after set number of epochs (Test every 10 epochs for example) - -// OPTIMIZER - "grad_clip": 4, // apply gradient clipping if > 0 - "lr_scheduler": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate - "lr_scheduler_params": { - "gamma": 0.5, - "milestones": [200000, 400000, 600000] - }, - "lr": 1e-4, // initial learning rate - -// TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints loss values for each step in eval run. - "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_all_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - -// DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 50, // number of samples for testing - -// PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} diff --git a/TTS/vocoder/configs/wavernn_config.py b/TTS/vocoder/configs/wavernn_config.py new file mode 100644 index 00000000..95a3cfc4 --- /dev/null +++ b/TTS/vocoder/configs/wavernn_config.py @@ -0,0 +1,115 @@ +from dataclasses import dataclass, field + +from TTS.vocoder.configs.shared_configs import BaseVocoderConfig + + +@dataclass +class WavernnConfig(BaseVocoderConfig): + """Defines parameters for Wavernn vocoder. + Example: + + >>> from TTS.vocoder.configs import WavernnConfig + >>> config = WavernnConfig() + + Args: + model (str): + Model name used for selecting the right model at initialization. Defaults to `wavernn`. + mode (str): + Output mode of the WaveRNN vocoder. `mold` for Mixture of Logistic Distribution, `gauss` for a single + Gaussian Distribution and `bits` for quantized bits as the model's output. + mulaw (bool): + enable / disable the use of Mulaw quantization for training. Only applicable if `mode == 'bits'`. Defaults + to `True`. + generator_model (str): + One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is + considered as a generator too. Defaults to `WaveRNN`. + wavernn_model_params (dict): + kwargs for the WaveRNN model. Defaults to + `{ + "rnn_dims": 512, + "fc_dims": 512, + "compute_dims": 128, + "res_out_dims": 128, + "num_res_blocks": 10, + "use_aux_net": True, + "use_upsample_net": True, + "upsample_factors": [4, 8, 8] + }` + batched (bool): + enable / disable the batched inference. It speeds up the inference by splitting the input into segments and + processing the segments in a batch. Then it merges the outputs with a certain overlap and smoothing. If + you set it False, without CUDA, it is too slow to be practical. Defaults to True. + target_samples (int): + Size of the segments in batched mode. Defaults to 11000. + overlap_sampels (int): + Size of the overlap between consecutive segments. Defaults to 550. + batch_size (int): + Batch size used at training. Larger values use more memory. Defaults to 256. + seq_len (int): + Audio segment length used at training. Larger values use more memory. Defaults to 1280. + padding (int): + Padding applied to the input feature frames against the convolution layers of the feature network. + Defaults to 2. + use_noise_augment (bool): + enable / disable random noise added to the input waveform. The noise is added after computing the + features. Defaults to True. + use_cache (bool): + enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is + not large enough. Defaults to True. + mixed_precision (bool): + enable / disable mixed precision training. Default is True. + eval_split_size (int): + Number of samples used for evalutaion. Defaults to 50. + test_every_epoch (int): + Number of epochs waited to run the next evalution. Since inference takes some time, it is better to + wait some number of epochs not ot waste training time. Defaults to 10. + grad_clip (float): + Gradient clipping threshold. If <= 0.0, no clipping is applied. Defaults to 4.0 + lr (float): + Initila leraning rate. Defaults to 1e-4. + lr_scheduler (str): + One of the learning rate schedulers from `torch.optim.scheduler.*`. Defaults to `MultiStepLR`. + lr_scheduler_params (dict): + kwargs for the scheduler. Defaults to `{"gamma": 0.5, "milestones": [200000, 400000, 600000]}` + """ + + model: str = "wavernn" + + # Model specific params + mode: str = "mold" # mold [string], gauss [string], bits [int] + mulaw: bool = True # apply mulaw if mode is bits + generator_model: str = "WaveRNN" + wavernn_model_params: dict = field( + default_factory=lambda: { + "rnn_dims": 512, + "fc_dims": 512, + "compute_dims": 128, + "res_out_dims": 128, + "num_res_blocks": 10, + "use_aux_net": True, + "use_upsample_net": True, + "upsample_factors": [4, 8, 8], # this needs to correctly factorise hop_length + } + ) + + # Inference + batched: bool = True + target_samples: int = 11000 + overlap_samples: int = 550 + + # Training - overrides + epochs: int = 10000 + batch_size: int = 256 + seq_len: int = 1280 + padding: int = 2 + use_noise_augment: bool = False + use_cache: bool = True + mixed_precision: bool = True + eval_split_size: int = 50 + test_every_epochs: int = 10 # number of epochs to wait until the next test run (synthesizing a full audio clip). + + # optimizer overrides + grad_clip: float = 4.0 + lr: float = 1e-4 # Initial learning rate. + lr_scheduler: str = "MultiStepLR" # one of the schedulers from https:#pytorch.org/docs/stable/optim.html + lr_scheduler_params: dict = field(default_factory=lambda: {"gamma": 0.5, "milestones": [200000, 400000, 600000]}) diff --git a/hubconf.py b/hubconf.py index 152374c8..bcbd6fce 100644 --- a/hubconf.py +++ b/hubconf.py @@ -1,6 +1,6 @@ dependencies = [ 'torch', 'gdown', 'pysbd', 'phonemizer', 'unidecode', 'pypinyin' -] # apt install espeak-ng +] import torch from TTS.utils.manage import ModelManager diff --git a/notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb b/notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb deleted file mode 100644 index c39cc53e..00000000 --- a/notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb +++ /dev/null @@ -1,387 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6LWsNd3_M3MP" - }, - "source": [ - "# Mozilla TTS on CPU Real-Time Speech Synthesis with TFLite" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FAqrSIWgLyP0" - }, - "source": [ - "**These models are converted from released [PyTorch models](https://colab.research.google.com/drive/1u_16ZzHjKYFn1HNVuA4Qf_i2MMFB9olY?usp=sharing) using our TF utilities provided in Mozilla TTS.**\n", - "\n", - "#### **Notebook Details**\n", - "These TFLite models support TF 2.3rc0 and for different versions you might need to regenerate them. \n", - "\n", - "TFLite optimizations degrades the TTS model performance and we do not apply\n", - "any optimization for the vocoder model due to the same reason. If you like to\n", - "keep the quality, consider to regenerate TFLite model accordingly.\n", - "\n", - "Models optimized with TFLite can be slow on a regular CPU since it is optimized\n", - "specifically for lower-end systems.\n", - "\n", - "---\n", - "\n", - "\n", - "\n", - "#### **Model Details** \n", - "We use Tacotron2 and MultiBand-Melgan models and LJSpeech dataset.\n", - "\n", - "Tacotron2 is trained using [Double Decoder Consistency](https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency/) (DDC) only for 130K steps (3 days) with a single GPU.\n", - "\n", - "MultiBand-Melgan is trained 1.45M steps with real spectrograms.\n", - "\n", - "Note that both model performances can be improved with more training.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ku-dA4DKoeXk" - }, - "source": [ - "### Download TF Models and configs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 162 - }, - "colab_type": "code", - "id": "jGIgnWhGsxU1", - "outputId": "57af701e-77ec-400d-fee5-64aa7603d357" - }, - "outputs": [], - "source": [ - "!gdown --id 17PYXCmTe0el_SLTwznrt3vOArNGMGo5v -O tts_model.tflite\n", - "!gdown --id 18CQ6G6tBEOfvCHlPqP8EBI4xWbrr9dBc -O config.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 235 - }, - "colab_type": "code", - "id": "4dnpE0-kvTsu", - "outputId": "6aab0622-9add-4ee4-b9f8-177d6ddc0e86" - }, - "outputs": [], - "source": [ - "!gdown --id 1aXveT-NjOM1mUr6tM4JfWjshq67GvVIO -O vocoder_model.tflite\n", - "!gdown --id 1Rd0R_nRCrbjEdpOwq6XwZAktvugiBvmu -O config_vocoder.json\n", - "!gdown --id 11oY3Tv0kQtxK_JPgxrfesa99maVXHNxU -O scale_stats.npy" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ZuDrj_ioqHE" - }, - "source": [ - "### Setup Libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 964 - }, - "colab_type": "code", - "id": "X2axt5BYq7gv", - "outputId": "aa53986f-f218-4d17-8667-0d74bb90c927" - }, - "outputs": [], - "source": [ - "# need it for char to phoneme conversion\n", - "! sudo apt-get install espeak" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 144 - }, - "colab_type": "code", - "id": "ZduAf-qYYEIT", - "outputId": "c1fcac0d-b8f8-442c-d598-4f549c42b698" - }, - "outputs": [], - "source": [ - "!git clone https://github.com/mozilla/TTS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "colab_type": "code", - "id": "ofPCvPyjZEcT", - "outputId": "f3d3ea73-eae5-473c-db19-276bd0e721cc" - }, - "outputs": [], - "source": [ - "%cd TTS\n", - "!git checkout c7296b3\n", - "!pip install -r requirements.txt\n", - "!python setup.py install\n", - "!pip install tensorflow==2.3.0rc0\n", - "%cd .." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zlgi8fPdpRF0" - }, - "source": [ - "### Define TTS function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "f-Yc42nQZG5A" - }, - "outputs": [], - "source": [ - "def run_vocoder(mel_spec):\n", - " vocoder_inputs = mel_spec[None, :, :]\n", - " # get input and output details\n", - " input_details = vocoder_model.get_input_details()\n", - " # reshape input tensor for the new input shape\n", - " vocoder_model.resize_tensor_input(input_details[0]['index'], vocoder_inputs.shape)\n", - " vocoder_model.allocate_tensors()\n", - " detail = input_details[0]\n", - " vocoder_model.set_tensor(detail['index'], vocoder_inputs)\n", - " # run the model\n", - " vocoder_model.invoke()\n", - " # collect outputs\n", - " output_details = vocoder_model.get_output_details()\n", - " waveform = vocoder_model.get_tensor(output_details[0]['index'])\n", - " return waveform \n", - "\n", - "\n", - "def tts(model, text, CONFIG, p):\n", - " t_1 = time.time()\n", - " waveform, alignment, mel_spec, mel_postnet_spec, stop_tokens, inputs = synthesis(model, text, CONFIG, use_cuda, ap, speaker_id, style_wav=None,\n", - " truncated=False, enable_eos_bos_chars=CONFIG.enable_eos_bos_chars,\n", - " backend='tflite')\n", - " waveform = run_vocoder(mel_postnet_spec.T)\n", - " waveform = waveform[0, 0]\n", - " rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n", - " tps = (time.time() - t_1) / len(waveform)\n", - " print(waveform.shape)\n", - " print(\" > Run-time: {}\".format(time.time() - t_1))\n", - " print(\" > Real-time factor: {}\".format(rtf))\n", - " print(\" > Time per step: {}\".format(tps))\n", - " IPython.display.display(IPython.display.Audio(waveform, rate=CONFIG.audio['sample_rate'])) \n", - " return alignment, mel_postnet_spec, stop_tokens, waveform" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZksegYQepkFg" - }, - "source": [ - "### Load TF Models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "oVa0kOamprgj" - }, - "outputs": [], - "source": [ - "import os\n", - "import torch\n", - "import time\n", - "import IPython\n", - "\n", - "from TTS.tf.utils.tflite import load_tflite_model\n", - "from TTS.tf.utils.io import load_checkpoint\n", - "from TTS.utils.io import load_config\n", - "from TTS.utils.text.symbols import symbols, phonemes\n", - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.tts.utils.synthesis import synthesis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EY-sHVO8IFSH" - }, - "outputs": [], - "source": [ - "# runtime settings\n", - "use_cuda = False" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_1aIUp2FpxOQ" - }, - "outputs": [], - "source": [ - "# model paths\n", - "TTS_MODEL = \"tts_model.tflite\"\n", - "TTS_CONFIG = \"config.json\"\n", - "VOCODER_MODEL = \"vocoder_model.tflite\"\n", - "VOCODER_CONFIG = \"config_vocoder.json\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "CpgmdBVQplbv" - }, - "outputs": [], - "source": [ - "# load configs\n", - "TTS_CONFIG = load_config(TTS_CONFIG)\n", - "VOCODER_CONFIG = load_config(VOCODER_CONFIG)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 471 - }, - "colab_type": "code", - "id": "zmrQxiozIUVE", - "outputId": "ca7e9016-4c28-4cef-efe7-0613d399aa4c" - }, - "outputs": [], - "source": [ - "# load the audio processor\n", - "ap = AudioProcessor(**TTS_CONFIG.audio) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8fLoI4ipqMeS" - }, - "outputs": [], - "source": [ - "# LOAD TTS MODEL\n", - "# multi speaker \n", - "speaker_id = None\n", - "speakers = []\n", - "\n", - "# load the models\n", - "model = load_tflite_model(TTS_MODEL)\n", - "vocoder_model = load_tflite_model(VOCODER_MODEL)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ws_YkPKsLgo-" - }, - "source": [ - "## Run Inference" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 134 - }, - "colab_type": "code", - "id": "FuWxZ9Ey5Puj", - "outputId": "d1888ebd-3208-42a4-aaf9-78d0e3ec987d" - }, - "outputs": [], - "source": [ - "sentence = \"Bill got in the habit of asking himself “Is that thought true?” and if he wasn’t absolutely certain it was, he just let it go.\"\n", - "align, spec, stop_tokens, wav = tts(model, sentence, TTS_CONFIG, ap)" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "DDC-TTS_and_MultiBand-MelGAN_TFLite_Example.ipynb", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb b/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb deleted file mode 100644 index 82efdc2a..00000000 --- a/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb +++ /dev/null @@ -1,650 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yZK6UdwSFnOO" - }, - "source": [ - "# **Download and install Coqui TTS**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yvb0pX3WY6MN" - }, - "outputs": [], - "source": [ - "import os \n", - "!git clone https://github.com/Edresson/TTS -b dev-gst-embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iB9nl2UEG3SY" - }, - "outputs": [], - "source": [ - "!apt-get install espeak\n", - "os.chdir('TTS')\n", - "!pip install -r requirements.txt\n", - "!python setup.py develop\n", - "os.chdir('..')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w6Krn8k1inC_" - }, - "source": [ - "\n", - "\n", - "**Download Checkpoint**\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "PiYHf3lKhi9z" - }, - "outputs": [], - "source": [ - "!wget -c -q --show-progress -O ./TTS-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/Checkpoints-TTS-MultiSpeaker-Jia-et-al-2018.zip\n", - "!unzip ./TTS-checkpoint.zip\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MpYNgqrZcJKn" - }, - "source": [ - "**Utils Functions**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4KZA4b_CbMqx" - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "import argparse\n", - "import json\n", - "# pylint: disable=redefined-outer-name, unused-argument\n", - "import os\n", - "import string\n", - "import time\n", - "import sys\n", - "import numpy as np\n", - "\n", - "TTS_PATH = \"../content/TTS\"\n", - "# add libraries into environment\n", - "sys.path.append(TTS_PATH) # set this if TTS is not installed globally\n", - "\n", - "import torch\n", - "\n", - "from TTS.tts.utils.generic_utils import setup_model\n", - "from TTS.tts.utils.synthesis import synthesis\n", - "from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols\n", - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.utils.io import load_config\n", - "from TTS.vocoder.utils.generic_utils import setup_generator\n", - "\n", - "\n", - "def tts(model, vocoder_model, text, CONFIG, use_cuda, ap, use_gl, speaker_fileid, speaker_embedding=None):\n", - " t_1 = time.time()\n", - " waveform, _, _, mel_postnet_spec, _, _ = synthesis(model, text, CONFIG, use_cuda, ap, speaker_fileid, None, False, CONFIG.enable_eos_bos_chars, use_gl, speaker_embedding=speaker_embedding)\n", - " if CONFIG.model == \"Tacotron\" and not use_gl:\n", - " mel_postnet_spec = ap.out_linear_to_mel(mel_postnet_spec.T).T\n", - " if not use_gl:\n", - " waveform = vocoder_model.inference(torch.FloatTensor(mel_postnet_spec.T).unsqueeze(0))\n", - " if use_cuda and not use_gl:\n", - " waveform = waveform.cpu()\n", - " if not use_gl:\n", - " waveform = waveform.numpy()\n", - " waveform = waveform.squeeze()\n", - " rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n", - " tps = (time.time() - t_1) / len(waveform)\n", - " print(\" > Run-time: {}\".format(time.time() - t_1))\n", - " print(\" > Real-time factor: {}\".format(rtf))\n", - " print(\" > Time per step: {}\".format(tps))\n", - " return waveform\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ENA2OumIVeMA" - }, - "source": [ - "# **Vars definitions**\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jPD0d_XpVXmY" - }, - "outputs": [], - "source": [ - "TEXT = ''\n", - "OUT_PATH = 'tests-audios/'\n", - "# create output path\n", - "os.makedirs(OUT_PATH, exist_ok=True)\n", - "\n", - "SPEAKER_FILEID = None # if None use the first embedding from speakers.json\n", - "\n", - "# model vars \n", - "MODEL_PATH = 'best_model.pth.tar'\n", - "CONFIG_PATH = 'config.json'\n", - "SPEAKER_JSON = 'speakers.json'\n", - "\n", - "# vocoder vars\n", - "VOCODER_PATH = ''\n", - "VOCODER_CONFIG_PATH = ''\n", - "\n", - "USE_CUDA = True" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dV6cXXlfi72r" - }, - "source": [ - "# **Restore TTS Model**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "x1WgLFauWUPe" - }, - "outputs": [], - "source": [ - "# load the config\n", - "C = load_config(CONFIG_PATH)\n", - "C.forward_attn_mask = True\n", - "\n", - "# load the audio processor\n", - "ap = AudioProcessor(**C.audio)\n", - "\n", - "# if the vocabulary was passed, replace the default\n", - "if 'characters' in C.keys():\n", - " symbols, phonemes = make_symbols(**C.characters)\n", - "\n", - "speaker_embedding = None\n", - "speaker_embedding_dim = None\n", - "num_speakers = 0\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " num_speakers = len(speaker_mapping)\n", - " if C.use_external_speaker_embedding_file:\n", - " if SPEAKER_FILEID is not None:\n", - " speaker_embedding = speaker_mapping[SPEAKER_FILEID]['embedding']\n", - " else: # if speaker_fileid is not specificated use the first sample in speakers.json\n", - " choise_speaker = list(speaker_mapping.keys())[0]\n", - " print(\" Speaker: \",choise_speaker.split('_')[0],'was chosen automatically', \"(this speaker seen in training)\")\n", - " speaker_embedding = speaker_mapping[choise_speaker]['embedding']\n", - " speaker_embedding_dim = len(speaker_embedding)\n", - "\n", - "# load the model\n", - "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n", - "model = setup_model(num_chars, num_speakers, C, speaker_embedding_dim)\n", - "cp = torch.load(MODEL_PATH, map_location=torch.device('cpu'))\n", - "model.load_state_dict(cp['model'])\n", - "model.eval()\n", - "\n", - "if USE_CUDA:\n", - " model.cuda()\n", - "\n", - "model.decoder.set_r(cp['r'])\n", - "\n", - "# load vocoder model\n", - "if VOCODER_PATH!= \"\":\n", - " VC = load_config(VOCODER_CONFIG_PATH)\n", - " vocoder_model = setup_generator(VC)\n", - " vocoder_model.load_state_dict(torch.load(VOCODER_PATH, map_location=\"cpu\")[\"model\"])\n", - " vocoder_model.remove_weight_norm()\n", - " if USE_CUDA:\n", - " vocoder_model.cuda()\n", - " vocoder_model.eval()\n", - "else:\n", - " vocoder_model = None\n", - " VC = None\n", - "\n", - "# synthesize voice\n", - "use_griffin_lim = VOCODER_PATH== \"\"\n", - "\n", - "if not C.use_external_speaker_embedding_file:\n", - " if SPEAKER_FILEID.isdigit():\n", - " SPEAKER_FILEID = int(SPEAKER_FILEID)\n", - " else:\n", - " SPEAKER_FILEID = None\n", - "else:\n", - " SPEAKER_FILEID = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tNvVEoE30qY6" - }, - "source": [ - "Synthesize sentence with Speaker\n", - "\n", - "> Stop running the cell to leave!\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2o8fXkVSyXOa" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",choise_speaker.split('_')[0], \"(this speaker seen in training)\")\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vnV-FigfvsS2" - }, - "source": [ - "# **Select Speaker**\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RuCGOnJ_fgDV" - }, - "outputs": [], - "source": [ - "\n", - "# VCTK speakers not seen in training (new speakers)\n", - "VCTK_test_Speakers = [\"p225\", \"p234\", \"p238\", \"p245\", \"p248\", \"p261\", \"p294\", \"p302\", \"p326\", \"p335\", \"p347\"]\n", - "\n", - "# VCTK speakers seen in training\n", - "VCTK_train_Speakers = ['p244', 'p300', 'p303', 'p273', 'p292', 'p252', 'p254', 'p269', 'p345', 'p274', 'p363', 'p285', 'p351', 'p361', 'p295', 'p266', 'p307', 'p230', 'p339', 'p253', 'p310', 'p241', 'p256', 'p323', 'p237', 'p229', 'p298', 'p336', 'p276', 'p305', 'p255', 'p278', 'p299', 'p265', 'p267', 'p280', 'p260', 'p272', 'p262', 'p334', 'p283', 'p247', 'p246', 'p374', 'p297', 'p249', 'p250', 'p304', 'p240', 'p236', 'p312', 'p286', 'p263', 'p258', 'p313', 'p376', 'p279', 'p340', 'p362', 'p284', 'p231', 'p308', 'p277', 'p275', 'p333', 'p314', 'p330', 'p264', 'p226', 'p288', 'p343', 'p239', 'p232', 'p268', 'p270', 'p329', 'p227', 'p271', 'p228', 'p311', 'p301', 'p293', 'p364', 'p251', 'p317', 'p360', 'p281', 'p243', 'p287', 'p233', 'p259', 'p316', 'p257', 'p282', 'p306', 'p341', 'p318']\n", - "\n", - "\n", - "num_samples_speaker = 2 # In theory the more samples of the speaker the more similar to the real voice it will be!\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hkvv7gRcx4WV" - }, - "source": [ - "## **Example select a VCTK seen speaker in training**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BviNMI9UyCYz" - }, - "outputs": [], - "source": [ - "# get embedding\n", - "Speaker_choise = VCTK_train_Speakers[0] # choise one of training speakers\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " if C.use_external_speaker_embedding_file:\n", - " speaker_embeddings = []\n", - " for key in list(speaker_mapping.keys()):\n", - " if Speaker_choise in key:\n", - " if len(speaker_embeddings) < num_samples_speaker:\n", - " speaker_embeddings.append(speaker_mapping[key]['embedding'])\n", - " # takes the average of the embedings samples of the announcers\n", - " speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5e5_XnLsx3jg" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker seen in training)\")\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJ6VgT2a4vHW" - }, - "source": [ - "## **Example select a VCTK not seen speaker in training (new Speakers)**\n", - "\n", - "\n", - "> Fitting new Speakers :)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SZS57ZK-4vHa" - }, - "outputs": [], - "source": [ - "# get embedding\n", - "Speaker_choise = VCTK_test_Speakers[0] # choise one of training speakers\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " if C.use_external_speaker_embedding_file:\n", - " speaker_embeddings = []\n", - " for key in list(speaker_mapping.keys()):\n", - " if Speaker_choise in key:\n", - " if len(speaker_embeddings) < num_samples_speaker:\n", - " speaker_embeddings.append(speaker_mapping[key]['embedding'])\n", - " # takes the average of the embedings samples of the announcers\n", - " speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bbs85vzz4vHo" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LEE6mQLh5Who" - }, - "source": [ - "# **Example Synthesizing with your own voice :)**\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "La70gSB65nrs" - }, - "source": [ - " Download and load GE2E Speaker Encoder " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "r0IEFZ0B5vQg" - }, - "outputs": [], - "source": [ - "!wget -c -q --show-progress -O ./SpeakerEncoder-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/GE2E-SpeakerEncoder-iter25k.zip\n", - "!unzip ./SpeakerEncoder-checkpoint.zip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jEH8HCTh5mF6" - }, - "outputs": [], - "source": [ - "SE_MODEL_RUN_PATH = \"GE2E-SpeakerEncoder/\"\n", - "SE_MODEL_PATH = os.path.join(SE_MODEL_RUN_PATH, \"best_model.pth.tar\")\n", - "SE_CONFIG_PATH =os.path.join(SE_MODEL_RUN_PATH, \"config.json\")\n", - "USE_CUDA = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tOwkfQqT6-Qo" - }, - "outputs": [], - "source": [ - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.speaker_encoder.model import SpeakerEncoder\n", - "se_config = load_config(SE_CONFIG_PATH)\n", - "se_ap = AudioProcessor(**se_config['audio'])\n", - "\n", - "se_model = SpeakerEncoder(**se_config.model)\n", - "se_model.load_state_dict(torch.load(SE_MODEL_PATH)['model'])\n", - "se_model.eval()\n", - "if USE_CUDA:\n", - " se_model.cuda()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TLlbUFG8O36" - }, - "source": [ - "Upload a wav audio file in your voice.\n", - "\n", - "\n", - "> We recommend files longer than 3 seconds, the bigger the file the closer to your voice :)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_FWwHPjJ8NXl" - }, - "outputs": [], - "source": [ - "from google.colab import files\n", - "file_list = files.upload()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WWOf6sgbBbGY" - }, - "outputs": [], - "source": [ - "# extract embedding from wav files\n", - "speaker_embeddings = []\n", - "for name in file_list.keys():\n", - " if '.wav' in name:\n", - " mel_spec = se_ap.melspectrogram(se_ap.load_wav(name, sr=se_ap.sample_rate)).T\n", - " mel_spec = torch.FloatTensor(mel_spec[None, :, :])\n", - " if USE_CUDA:\n", - " mel_spec = mel_spec.cuda()\n", - " embedd = se_model.compute_embedding(mel_spec).cpu().detach().numpy().reshape(-1)\n", - " speaker_embeddings.append(embedd)\n", - " else:\n", - " print(\" You need upload Wav files, others files is not supported !!\")\n", - "\n", - "# takes the average of the embedings samples of the announcers\n", - "speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xmItcGac5WiG" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [ - "vnV-FigfvsS2", - "hkvv7gRcx4WV", - "QJ6VgT2a4vHW" - ], - "name": "Demo-Mozilla-TTS-MultiSpeaker-jia-et-al-2018.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018_With_GST.ipynb b/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018_With_GST.ipynb deleted file mode 100644 index f65d09a6..00000000 --- a/notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018_With_GST.ipynb +++ /dev/null @@ -1,847 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yZK6UdwSFnOO" - }, - "source": [ - "# **Download and install Coqui TTS**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yvb0pX3WY6MN" - }, - "outputs": [], - "source": [ - "import os \n", - "!git clone https://github.com/Edresson/TTS -b dev-gst-embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iB9nl2UEG3SY" - }, - "outputs": [], - "source": [ - "!apt-get install espeak\n", - "os.chdir('TTS')\n", - "!pip install -r requirements.txt\n", - "!python setup.py develop\n", - "os.chdir('..')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w6Krn8k1inC_" - }, - "source": [ - "\n", - "\n", - "**Download Checkpoint**\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "PiYHf3lKhi9z" - }, - "outputs": [], - "source": [ - "!wget -c -q --show-progress -O ./TTS-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/Checkpoints-TTS-MultiSpeaker-Jia-et-al-2018-with-GST.zip\n", - "!unzip ./TTS-checkpoint.zip\n", - "\n", - "# Download gst style example\n", - "!wget https://github.com/Edresson/TTS/releases/download/v1.0.0/gst-style-example.wav" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MpYNgqrZcJKn" - }, - "source": [ - "**Utils Functions**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4KZA4b_CbMqx" - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "import argparse\n", - "import json\n", - "# pylint: disable=redefined-outer-name, unused-argument\n", - "import os\n", - "import string\n", - "import time\n", - "import sys\n", - "import numpy as np\n", - "\n", - "TTS_PATH = \"../content/TTS\"\n", - "# add libraries into environment\n", - "sys.path.append(TTS_PATH) # set this if TTS is not installed globally\n", - "\n", - "import torch\n", - "\n", - "from TTS.tts.utils.generic_utils import setup_model\n", - "from TTS.tts.utils.synthesis import synthesis\n", - "from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols\n", - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.utils.io import load_config\n", - "from TTS.vocoder.utils.generic_utils import setup_generator\n", - "\n", - "\n", - "def tts(model, vocoder_model, text, CONFIG, use_cuda, ap, use_gl, speaker_fileid, speaker_embedding=None, gst_style=None):\n", - " t_1 = time.time()\n", - " waveform, _, _, mel_postnet_spec, _, _ = synthesis(model, text, CONFIG, use_cuda, ap, speaker_fileid, gst_style, False, CONFIG.enable_eos_bos_chars, use_gl, speaker_embedding=speaker_embedding)\n", - " if CONFIG.model == \"Tacotron\" and not use_gl:\n", - " mel_postnet_spec = ap.out_linear_to_mel(mel_postnet_spec.T).T\n", - " if not use_gl:\n", - " waveform = vocoder_model.inference(torch.FloatTensor(mel_postnet_spec.T).unsqueeze(0))\n", - " if use_cuda and not use_gl:\n", - " waveform = waveform.cpu()\n", - " if not use_gl:\n", - " waveform = waveform.numpy()\n", - " waveform = waveform.squeeze()\n", - " rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n", - " tps = (time.time() - t_1) / len(waveform)\n", - " print(\" > Run-time: {}\".format(time.time() - t_1))\n", - " print(\" > Real-time factor: {}\".format(rtf))\n", - " print(\" > Time per step: {}\".format(tps))\n", - " return waveform\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ENA2OumIVeMA" - }, - "source": [ - "# **Vars definitions**\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jPD0d_XpVXmY" - }, - "outputs": [], - "source": [ - "TEXT = ''\n", - "OUT_PATH = 'tests-audios/'\n", - "# create output path\n", - "os.makedirs(OUT_PATH, exist_ok=True)\n", - "\n", - "SPEAKER_FILEID = None # if None use the first embedding from speakers.json\n", - "\n", - "# model vars \n", - "MODEL_PATH = 'best_model.pth.tar'\n", - "CONFIG_PATH = 'config.json'\n", - "SPEAKER_JSON = 'speakers.json'\n", - "\n", - "# vocoder vars\n", - "VOCODER_PATH = ''\n", - "VOCODER_CONFIG_PATH = ''\n", - "\n", - "USE_CUDA = True" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dV6cXXlfi72r" - }, - "source": [ - "# **Restore TTS Model**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "x1WgLFauWUPe" - }, - "outputs": [], - "source": [ - "# load the config\n", - "C = load_config(CONFIG_PATH)\n", - "C.forward_attn_mask = True\n", - "\n", - "# load the audio processor\n", - "ap = AudioProcessor(**C.audio)\n", - "\n", - "# if the vocabulary was passed, replace the default\n", - "if 'characters' in C.keys():\n", - " symbols, phonemes = make_symbols(**C.characters)\n", - "\n", - "speaker_embedding = None\n", - "speaker_embedding_dim = None\n", - "num_speakers = 0\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " num_speakers = len(speaker_mapping)\n", - " if C.use_external_speaker_embedding_file:\n", - " if SPEAKER_FILEID is not None:\n", - " speaker_embedding = speaker_mapping[SPEAKER_FILEID]['embedding']\n", - " else: # if speaker_fileid is not specificated use the first sample in speakers.json\n", - " choise_speaker = list(speaker_mapping.keys())[0]\n", - " print(\" Speaker: \",choise_speaker.split('_')[0],'was chosen automatically', \"(this speaker seen in training)\")\n", - " speaker_embedding = speaker_mapping[choise_speaker]['embedding']\n", - " speaker_embedding_dim = len(speaker_embedding)\n", - "\n", - "# load the model\n", - "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n", - "model = setup_model(num_chars, num_speakers, C, speaker_embedding_dim)\n", - "cp = torch.load(MODEL_PATH, map_location=torch.device('cpu'))\n", - "model.load_state_dict(cp['model'])\n", - "model.eval()\n", - "\n", - "if USE_CUDA:\n", - " model.cuda()\n", - "\n", - "model.decoder.set_r(cp['r'])\n", - "\n", - "# load vocoder model\n", - "if VOCODER_PATH!= \"\":\n", - " VC = load_config(VOCODER_CONFIG_PATH)\n", - " vocoder_model = setup_generator(VC)\n", - " vocoder_model.load_state_dict(torch.load(VOCODER_PATH, map_location=\"cpu\")[\"model\"])\n", - " vocoder_model.remove_weight_norm()\n", - " if USE_CUDA:\n", - " vocoder_model.cuda()\n", - " vocoder_model.eval()\n", - "else:\n", - " vocoder_model = None\n", - " VC = None\n", - "\n", - "# synthesize voice\n", - "use_griffin_lim = VOCODER_PATH== \"\"\n", - "\n", - "if not C.use_external_speaker_embedding_file:\n", - " if SPEAKER_FILEID.isdigit():\n", - " SPEAKER_FILEID = int(SPEAKER_FILEID)\n", - " else:\n", - " SPEAKER_FILEID = None\n", - "else:\n", - " SPEAKER_FILEID = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tNvVEoE30qY6" - }, - "source": [ - "Synthesize sentence with Speaker\n", - "\n", - "> Stop running the cell to leave!\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2o8fXkVSyXOa" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",choise_speaker.split('_')[0], \"(this speaker seen in training)\")\n", - "gst_style = 'gst-style-example.wav'\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vnV-FigfvsS2" - }, - "source": [ - "# **Select Speaker**\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RuCGOnJ_fgDV" - }, - "outputs": [], - "source": [ - "\n", - "# VCTK speakers not seen in training (new speakers)\n", - "VCTK_test_Speakers = [\"p225\", \"p234\", \"p238\", \"p245\", \"p248\", \"p261\", \"p294\", \"p302\", \"p326\", \"p335\", \"p347\"]\n", - "\n", - "# VCTK speakers seen in training\n", - "VCTK_train_Speakers = ['p244', 'p300', 'p303', 'p273', 'p292', 'p252', 'p254', 'p269', 'p345', 'p274', 'p363', 'p285', 'p351', 'p361', 'p295', 'p266', 'p307', 'p230', 'p339', 'p253', 'p310', 'p241', 'p256', 'p323', 'p237', 'p229', 'p298', 'p336', 'p276', 'p305', 'p255', 'p278', 'p299', 'p265', 'p267', 'p280', 'p260', 'p272', 'p262', 'p334', 'p283', 'p247', 'p246', 'p374', 'p297', 'p249', 'p250', 'p304', 'p240', 'p236', 'p312', 'p286', 'p263', 'p258', 'p313', 'p376', 'p279', 'p340', 'p362', 'p284', 'p231', 'p308', 'p277', 'p275', 'p333', 'p314', 'p330', 'p264', 'p226', 'p288', 'p343', 'p239', 'p232', 'p268', 'p270', 'p329', 'p227', 'p271', 'p228', 'p311', 'p301', 'p293', 'p364', 'p251', 'p317', 'p360', 'p281', 'p243', 'p287', 'p233', 'p259', 'p316', 'p257', 'p282', 'p306', 'p341', 'p318']\n", - "\n", - "\n", - "num_samples_speaker = 2 # In theory the more samples of the speaker the more similar to the real voice it will be!\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hkvv7gRcx4WV" - }, - "source": [ - "## **Example select a VCTK seen speaker in training**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BviNMI9UyCYz" - }, - "outputs": [], - "source": [ - "# get embedding\n", - "Speaker_choise = VCTK_train_Speakers[0] # choise one of training speakers\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " if C.use_external_speaker_embedding_file:\n", - " speaker_embeddings = []\n", - " for key in list(speaker_mapping.keys()):\n", - " if Speaker_choise in key:\n", - " if len(speaker_embeddings) < num_samples_speaker:\n", - " speaker_embeddings.append(speaker_mapping[key]['embedding'])\n", - " # takes the average of the embedings samples of the announcers\n", - " speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5e5_XnLsx3jg" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker seen in training)\")\n", - "gst_style = 'gst-style-example.wav'\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJ6VgT2a4vHW" - }, - "source": [ - "## **Example select a VCTK not seen speaker in training (new Speakers)**\n", - "\n", - "\n", - "> Fitting new Speakers :)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SZS57ZK-4vHa" - }, - "outputs": [], - "source": [ - "# get embedding\n", - "Speaker_choise = VCTK_test_Speakers[0] # choise one of training speakers\n", - "# load speakers\n", - "if SPEAKER_JSON != '':\n", - " speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n", - " if C.use_external_speaker_embedding_file:\n", - " speaker_embeddings = []\n", - " for key in list(speaker_mapping.keys()):\n", - " if Speaker_choise in key:\n", - " if len(speaker_embeddings) < num_samples_speaker:\n", - " speaker_embeddings.append(speaker_mapping[key]['embedding'])\n", - " # takes the average of the embedings samples of the announcers\n", - " speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bbs85vzz4vHo" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "gst_style = 'gst-style-example.wav'\n", - "while True:\n", - " TEXT = input(\"Enter sentence: \")\n", - " print(\" > Text: {}\".format(TEXT))\n", - " wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - " IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - " # save the results\n", - " file_name = TEXT.replace(\" \", \"_\")\n", - " file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - " out_path = os.path.join(OUT_PATH, file_name)\n", - " print(\" > Saving output to {}\".format(out_path))\n", - " ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_G_HweN04W-" - }, - "source": [ - "# **Changing GST tokens manually (without wav reference)**" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jyFP5syW2bjt" - }, - "source": [ - "You can define tokens manually, this way you can increase/decrease the function of a given GST token. For example a token is responsible for the length of the speaker's pauses, if you increase the value of that token you will have longer pauses and if you decrease it you will have shorter pauses." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SpwjDjCM2a3Y" - }, - "outputs": [], - "source": [ - "# set gst tokens, in this model we have 5 tokens\n", - "gst_style = {\"0\": 0, \"1\": 0, \"3\": 0, \"4\": 0}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qWChMbI_0z5X" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uFjUi9xQ3mG3" - }, - "outputs": [], - "source": [ - "gst_style = {\"0\": 0.9, \"1\": 0, \"3\": 0, \"4\": 0}\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Uw0d6gWg4L27" - }, - "outputs": [], - "source": [ - "gst_style = {\"0\": -0.9, \"1\": 0, \"3\": 0, \"4\": 0}\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "V9izw4-54-Tl" - }, - "outputs": [], - "source": [ - "gst_style = {\"0\": 0, \"1\": 0.9, \"3\": 0, \"4\": 0}\n", - "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LEE6mQLh5Who" - }, - "source": [ - "# **Example Synthesizing with your own voice :)**\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "La70gSB65nrs" - }, - "source": [ - " Download and load GE2E Speaker Encoder " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "r0IEFZ0B5vQg" - }, - "outputs": [], - "source": [ - "!wget -c -q --show-progress -O ./SpeakerEncoder-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/GE2E-SpeakerEncoder-iter25k.zip\n", - "!unzip ./SpeakerEncoder-checkpoint.zip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jEH8HCTh5mF6" - }, - "outputs": [], - "source": [ - "SE_MODEL_RUN_PATH = \"GE2E-SpeakerEncoder/\"\n", - "SE_MODEL_PATH = os.path.join(SE_MODEL_RUN_PATH, \"best_model.pth.tar\")\n", - "SE_CONFIG_PATH =os.path.join(SE_MODEL_RUN_PATH, \"config.json\")\n", - "USE_CUDA = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tOwkfQqT6-Qo" - }, - "outputs": [], - "source": [ - "from TTS.utils.audio import AudioProcessor\n", - "from TTS.speaker_encoder.model import SpeakerEncoder\n", - "se_config = load_config(SE_CONFIG_PATH)\n", - "se_ap = AudioProcessor(**se_config['audio'])\n", - "\n", - "se_model = SpeakerEncoder(**se_config.model)\n", - "se_model.load_state_dict(torch.load(SE_MODEL_PATH)['model'])\n", - "se_model.eval()\n", - "if USE_CUDA:\n", - " se_model.cuda()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TLlbUFG8O36" - }, - "source": [ - "Upload one or more wav audio files in your voice.\n", - "\n", - "\n", - "> We recommend files longer than 3 seconds, the bigger the file the closer to your voice :)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_FWwHPjJ8NXl" - }, - "outputs": [], - "source": [ - "# select one or more wav files\n", - "from google.colab import files\n", - "file_list = files.upload()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WWOf6sgbBbGY" - }, - "outputs": [], - "source": [ - "# extract embedding from wav files\n", - "speaker_embeddings = []\n", - "for name in file_list.keys():\n", - " if '.wav' in name:\n", - " mel_spec = se_ap.melspectrogram(se_ap.load_wav(name, sr=se_ap.sample_rate)).T\n", - " mel_spec = torch.FloatTensor(mel_spec[None, :, :])\n", - " if USE_CUDA:\n", - " mel_spec = mel_spec.cuda()\n", - " embedd = se_model.compute_embedding(mel_spec).cpu().detach().numpy().reshape(-1)\n", - " speaker_embeddings.append(embedd)\n", - " else:\n", - " print(\"You need upload Wav files, others files is not supported !!\")\n", - "\n", - "# takes the average of the embedings samples of the announcers\n", - "speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AQ7eP31d9yzq" - }, - "outputs": [], - "source": [ - "import IPython\n", - "from IPython.display import Audio\n", - "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n", - "gst_style = {\"0\": 0, \"1\": 0.0, \"3\": 0, \"4\": 0}\n", - "gst_style = 'gst-style-example.wav'\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "11i10yE1-LMJ" - }, - "source": [ - "Uploading your own GST reference wav file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eKohSQG1-KkT" - }, - "outputs": [], - "source": [ - "# select one wav file for GST reference\n", - "from google.colab import files\n", - "file_list = files.upload()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xmItcGac5WiG" - }, - "outputs": [], - "source": [ - "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n", - "gst_style = list(file_list.keys())[0]\n", - "TEXT = input(\"Enter sentence: \")\n", - "print(\" > Text: {}\".format(TEXT))\n", - "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n", - "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n", - "# save the results\n", - "file_name = TEXT.replace(\" \", \"_\")\n", - "file_name = file_name.translate(\n", - " str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n", - "out_path = os.path.join(OUT_PATH, file_name)\n", - "print(\" > Saving output to {}\".format(out_path))\n", - "ap.save_wav(wav, out_path)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [ - "yZK6UdwSFnOO", - "ENA2OumIVeMA", - "dV6cXXlfi72r", - "vnV-FigfvsS2", - "g_G_HweN04W-", - "LEE6mQLh5Who" - ], - "name": "Demo-Mozilla-TTS-MultiSpeaker-jia-et-al-2018-With-GST.ipynb", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/recipes/README.md b/recipes/README.md new file mode 100644 index 00000000..041693a2 --- /dev/null +++ b/recipes/README.md @@ -0,0 +1,13 @@ +# 🐸💬 TTS Training Recipes + +TTS recipes intended to host bash scripts running all the necessary steps to train a TTS model with a particular dataset. + +Run each script from the root TTS folder as follows + +```console +$ bash ./recipes///run.sh +``` + +All the outputs are held under the recipe directory unless you change the paths in the bash script. + +If you train a new model using TTS, feel free to share your training to expand the list of recipes. \ No newline at end of file diff --git a/recipes/ljspeech/tacotron2-DDC/run.sh b/recipes/ljspeech/tacotron2-DDC/run.sh new file mode 100644 index 00000000..eaa05b60 --- /dev/null +++ b/recipes/ljspeech/tacotron2-DDC/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# take the scripts's parent's directory to prefix all the output paths. +RUN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +echo $RUN_DIR +# download LJSpeech dataset +wget http://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2 +# extract +tar -xjf LJSpeech-1.1.tar.bz2 +# create train-val splits +shuf LJSpeech-1.1/metadata.csv > LJSpeech-1.1/metadata_shuf.csv +head -n 12000 LJSpeech-1.1/metadata_shuf.csv > LJSpeech-1.1/metadata_train.csv +tail -n 1100 LJSpeech-1.1/metadata_shuf.csv > LJSpeech-1.1/metadata_val.csv +mv LJSpeech-1.1 $RUN_DIR/ +rm LJSpeech-1.1.tar.bz2 +# compute dataset mean and variance for normalization +python TTS/bin/compute_statistics.py $RUN_DIR/tacotron2-DDC.json $RUN_DIR/scale_stats.npy --data_path $RUN_DIR/LJSpeech-1.1/wavs/ +# training .... +# change the GPU id if needed +CUDA_VISIBLE_DEVICES="0" python TTS/bin/train_tacotron.py --config_path $RUN_DIR/tacotron2-DDC.json \ + --coqpit.output_path $RUN_DIR \ + --coqpit.datasets.0.path $RUN_DIR/LJSpeech-1.1/ \ + --coqpit.audio.stats_path $RUN_DIR/scale_stats.npy \ \ No newline at end of file diff --git a/recipes/ljspeech/tacotron2-DDC/tacotron2-DDC.json b/recipes/ljspeech/tacotron2-DDC/tacotron2-DDC.json new file mode 100644 index 00000000..9cdbbd3b --- /dev/null +++ b/recipes/ljspeech/tacotron2-DDC/tacotron2-DDC.json @@ -0,0 +1,91 @@ +{ + "datasets": [ + { + "name": "ljspeech", + "path": "DEFINE THIS", + "meta_file_train": "metadata.csv", + "meta_file_val": null + } + ], + "audio": { + "fft_size": 1024, + "win_length": 1024, + "hop_length": 256, + "frame_length_ms": null, + "frame_shift_ms": null, + "sample_rate": 22050, + "preemphasis": 0.0, + "ref_level_db": 20, + "do_trim_silence": true, + "trim_db": 60, + "power": 1.5, + "griffin_lim_iters": 60, + "num_mels": 80, + "mel_fmin": 50.0, + "mel_fmax": 7600.0, + "spec_gain": 1, + "signal_norm": true, + "min_level_db": -100, + "symmetric_norm": true, + "max_norm": 4.0, + "clip_norm": true, + "stats_path": "scale_stats.npy" + }, + "gst":{ + "gst_embedding_dim": 256, + "gst_num_heads": 4, + "gst_num_style_tokens": 10 + }, + "model": "Tacotron2", + "run_name": "ljspeech-ddc", + "run_description": "tacotron2 with double decoder consistency.", + "batch_size": 64, + "eval_batch_size": 16, + "mixed_precision": true, + "loss_masking": true, + "decoder_loss_alpha": 0.25, + "postnet_loss_alpha": 0.25, + "postnet_diff_spec_alpha": 0.25, + "decoder_diff_spec_alpha": 0.25, + "decoder_ssim_alpha": 0.25, + "postnet_ssim_alpha": 0.25, + "ga_alpha": 5.0, + "stopnet_pos_weight": 15.0, + "run_eval": true, + "test_delay_epochs": 10, + "test_sentences_file": null, + "noam_schedule": true, + "grad_clip": 0.05, + "epochs": 1000, + "lr": 0.001, + "wd": 1e-06, + "warmup_steps": 4000, + "memory_size": -1, + "prenet_type": "original", + "prenet_dropout": true, + "attention_type": "original", + "location_attn": true, + "double_decoder_consistency": true, + "ddc_r": 6, + "attention_norm": "sigmoid", + "r": 6, + "gradual_training": [[0, 6, 64], [10000, 4, 32], [50000, 3, 32], [100000, 2, 32]], + "stopnet": true, + "separate_stopnet": true, + "print_step": 25, + "tb_plot_step": 100, + "print_eval": false, + "save_step": 10000, + "checkpoint": true, + "text_cleaner": "phoneme_cleaners", + "num_loader_workers": 4, + "num_val_loader_workers": 4, + "batch_group_size": 4, + "min_seq_len": 6, + "max_seq_len": 180, + "compute_input_seq_cache": true, + "output_path": "DEFINE THIS", + "phoneme_cache_path": "DEFINE THIS", + "use_phonemes": false, + "phoneme_language": "en-us" +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 204d1191..c6ce7672 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ librosa==0.8.0 matplotlib numpy==1.18.5 pandas -phonemizer>=2.2.0 pypinyin pysbd pyyaml @@ -16,5 +15,7 @@ soundfile tensorboardX torch>=1.7 tqdm +numba==0.52 umap-learn==0.4.6 unidecode==0.4.20 +coqpit diff --git a/run_bash_tests.sh b/run_bash_tests.sh index 16381611..feb9082b 100755 --- a/run_bash_tests.sh +++ b/run_bash_tests.sh @@ -2,13 +2,7 @@ set -e TF_CPP_MIN_LOG_LEVEL=3 # runtime bash based tests +# TODO: move these to python ./tests/bash_tests/test_demo_server.sh && \ ./tests/bash_tests/test_resample.sh && \ -./tests/bash_tests/test_tacotron_train.sh && \ -./tests/bash_tests/test_glow-tts_train.sh && \ -./tests/bash_tests/test_vocoder_gan_train.sh && \ -./tests/bash_tests/test_vocoder_wavernn_train.sh && \ -./tests/bash_tests/test_vocoder_wavegrad_train.sh && \ -./tests/bash_tests/test_speedy_speech_train.sh && \ -./tests/bash_tests/test_aligntts_train.sh && \ ./tests/bash_tests/test_compute_statistics.sh diff --git a/setup.py b/setup.py index abed43a2..a68b09e0 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import os import subprocess import sys from distutils.version import LooseVersion +from TTS._version import __version__ import numpy import setuptools.command.build_py @@ -18,7 +19,7 @@ if LooseVersion(sys.version) < LooseVersion("3.6") or LooseVersion(sys.version) ) -version = '0.0.13.2' +version = __version__ cwd = os.path.dirname(os.path.abspath(__file__)) class build_py(setuptools.command.build_py.build_py): # pylint: disable=too-many-ancestors diff --git a/tests/__init__.py b/tests/__init__.py index f1445c92..c7930ef9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,16 @@ import os +from TTS.utils.generic_utils import get_cuda + + +def get_device_id(): + use_cuda, _ = get_cuda() + if use_cuda: + GPU_ID = "0" + else: + GPU_ID = "" + return GPU_ID + def get_tests_path(): """Returns the path to the test directory.""" diff --git a/tests/bash_tests/test_aligntts_train.sh b/tests/bash_tests/test_aligntts_train.sh deleted file mode 100755 index 38d46520..00000000 --- a/tests/bash_tests/test_aligntts_train.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_align_tts.py --config_path $BASEDIR/../inputs/test_align_tts.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_align_tts.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ diff --git a/tests/bash_tests/test_glow-tts_train.sh b/tests/bash_tests/test_glow-tts_train.sh deleted file mode 100755 index 04aef2ad..00000000 --- a/tests/bash_tests/test_glow-tts_train.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_glow_tts.py --config_path $BASEDIR/../inputs/test_glow_tts.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_glow_tts.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ diff --git a/tests/bash_tests/test_speedy_speech_train.sh b/tests/bash_tests/test_speedy_speech_train.sh deleted file mode 100755 index 2276034f..00000000 --- a/tests/bash_tests/test_speedy_speech_train.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_speedy_speech.py --config_path $BASEDIR/../inputs/test_speedy_speech.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_speedy_speech.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ diff --git a/tests/bash_tests/test_tacotron_train.sh b/tests/bash_tests/test_tacotron_train.sh deleted file mode 100755 index 4aacf69c..00000000 --- a/tests/bash_tests/test_tacotron_train.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" - -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ - -# run Tacotron bi-directional decoder -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron_bd_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ - -# Tacotron2 -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron2_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/ - diff --git a/tests/bash_tests/test_vocoder_gan_train.sh b/tests/bash_tests/test_vocoder_gan_train.sh deleted file mode 100755 index b2f43721..00000000 --- a/tests/bash_tests/test_vocoder_gan_train.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# create run dir -mkdir $BASEDIR/../train_outputs -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_gan.py --config_path $BASEDIR/../inputs/test_vocoder_multiband_melgan_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_gan.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER diff --git a/tests/bash_tests/test_vocoder_wavegrad_train.sh b/tests/bash_tests/test_vocoder_wavegrad_train.sh deleted file mode 100755 index 9626187f..00000000 --- a/tests/bash_tests/test_vocoder_wavegrad_train.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# create run dir -mkdir -p $BASEDIR/../train_outputs -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavegrad.py --config_path $BASEDIR/../inputs/test_vocoder_wavegrad.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavegrad.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER \ No newline at end of file diff --git a/tests/bash_tests/test_vocoder_wavernn_train.sh b/tests/bash_tests/test_vocoder_wavernn_train.sh deleted file mode 100755 index 7b554fc9..00000000 --- a/tests/bash_tests/test_vocoder_wavernn_train.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -xe -BASEDIR=$(dirname "$0") -echo "$BASEDIR" -# create run dir -mkdir -p $BASEDIR/../train_outputs -# run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavernn.py --config_path $BASEDIR/../inputs/test_vocoder_wavernn_config.json -# find the training folder -LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1) -echo $LATEST_FOLDER -# continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavernn.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER -# remove all the outputs -rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER \ No newline at end of file diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0001_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0001_phoneme.npy new file mode 100644 index 00000000..fc024a6d Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0001_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0002_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0002_phoneme.npy new file mode 100644 index 00000000..9ac355d9 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0002_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0003_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0003_phoneme.npy new file mode 100644 index 00000000..ea3944c1 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0003_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0004_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0004_phoneme.npy new file mode 100644 index 00000000..ff7f0e3e Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0004_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0005_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0005_phoneme.npy new file mode 100644 index 00000000..d7c6b2e3 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0005_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0006_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0006_phoneme.npy new file mode 100644 index 00000000..4e7fa19a Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0006_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0007_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0007_phoneme.npy new file mode 100644 index 00000000..b6db4460 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0007_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0008_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0008_phoneme.npy new file mode 100644 index 00000000..ff9fb8bc Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0008_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0009_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0009_phoneme.npy new file mode 100644 index 00000000..d273bbb2 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0009_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0010_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0010_phoneme.npy new file mode 100644 index 00000000..c53050da Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0010_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0011_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0011_phoneme.npy new file mode 100644 index 00000000..98921023 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0011_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0012_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0012_phoneme.npy new file mode 100644 index 00000000..bfdf8123 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0012_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0013_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0013_phoneme.npy new file mode 100644 index 00000000..f0e7c1a1 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0013_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0014_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0014_phoneme.npy new file mode 100644 index 00000000..f0ed86b0 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0014_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0015_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0015_phoneme.npy new file mode 100644 index 00000000..381b981a Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0015_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0016_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0016_phoneme.npy new file mode 100644 index 00000000..5667d54b Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0016_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0017_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0017_phoneme.npy new file mode 100644 index 00000000..7f6c7fc7 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0017_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0018_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0018_phoneme.npy new file mode 100644 index 00000000..b060181d Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0018_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0019_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0019_phoneme.npy new file mode 100644 index 00000000..b0e818cc Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0019_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0020_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0020_phoneme.npy new file mode 100644 index 00000000..472ac10b Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0020_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0021_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0021_phoneme.npy new file mode 100644 index 00000000..8fd4d5ab Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0021_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0022_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0022_phoneme.npy new file mode 100644 index 00000000..25273f68 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0022_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0023_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0023_phoneme.npy new file mode 100644 index 00000000..94b639df Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0023_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0024_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0024_phoneme.npy new file mode 100644 index 00000000..1178fc90 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0024_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0025_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0025_phoneme.npy new file mode 100644 index 00000000..b00b1750 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0025_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0026_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0026_phoneme.npy new file mode 100644 index 00000000..da78e6af Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0026_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0027_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0027_phoneme.npy new file mode 100644 index 00000000..313f2b41 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0027_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0028_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0028_phoneme.npy new file mode 100644 index 00000000..d6cf441c Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0028_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0029_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0029_phoneme.npy new file mode 100644 index 00000000..0c0d379d Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0029_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0030_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0030_phoneme.npy new file mode 100644 index 00000000..a8bcc0ae Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0030_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0031_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0031_phoneme.npy new file mode 100644 index 00000000..835ea796 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0031_phoneme.npy differ diff --git a/tests/data/ljspeech/phoneme_cache/LJ001-0032_phoneme.npy b/tests/data/ljspeech/phoneme_cache/LJ001-0032_phoneme.npy new file mode 100644 index 00000000..60a7b433 Binary files /dev/null and b/tests/data/ljspeech/phoneme_cache/LJ001-0032_phoneme.npy differ diff --git a/tests/inputs/test_config.json b/tests/inputs/test_config.json index b28bec64..8f8810d1 100644 --- a/tests/inputs/test_config.json +++ b/tests/inputs/test_config.json @@ -1,24 +1,24 @@ { "audio":{ - "audio_processor": "audio", // to use dictate different audio processors, if available. - "num_mels": 80, // size of the mel spec frame. - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 22050, // wav sample-rate. If different than the original data, it is resampled. - "frame_length_ms": null, // stft window length in ms. - "frame_shift_ms": null, // stft window hop-lengh in ms. + "audio_processor": "audio", + "num_mels": 80, + "fft_size": 1024, + "sample_rate": 22050, + "frame_length_ms": null, + "frame_shift_ms": null, "hop_length": 256, "win_length": 1024, - "preemphasis": 0.97, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "min_level_db": -100, // normalization range - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 30,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - "signal_norm": true, // normalize the spec values in range [0, 1] - "symmetric_norm": true, // move normalization to range [-1, 1] - "clip_norm": true, // clip normalized values into the range. - "max_norm": 4, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "mel_fmin": 0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000, // maximum freq level for mel-spec. Tune for dataset!! + "preemphasis": 0.97, + "min_level_db": -100, + "ref_level_db": 20, + "power": 1.5, + "griffin_lim_iters": 30, + "signal_norm": true, + "symmetric_norm": true, + "clip_norm": true, + "max_norm": 4, + "mel_fmin": 0, + "mel_fmax": 8000, "do_trim_silence": false, "spec_gain": 20 }, @@ -53,17 +53,17 @@ "max_seq_len": 300, "log_dir": "tests/outputs/", - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": true, // use global style tokens - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_style_tokens). - "gst_use_speaker_embedding": true, // if true pass speaker embedding in attention input GST. + + "use_speaker_embedding": false, + "use_gst": true, + "gst": { + "gst_style_input": null, + + + + "gst_use_speaker_embedding": true, "gst_embedding_dim": 512, "gst_num_heads": 4, - "gst_style_tokens": 10 + "gst_num_style_tokens": 10 } } diff --git a/tests/inputs/test_glow_tts.json b/tests/inputs/test_glow_tts.json index 0ee9395b..64cc3822 100644 --- a/tests/inputs/test_glow_tts.json +++ b/tests/inputs/test_glow_tts.json @@ -128,7 +128,7 @@ // PHONEMES "phoneme_cache_path": "tests/outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST diff --git a/tests/inputs/test_speaker_encoder_config.json b/tests/inputs/test_speaker_encoder_config.json index f1174e76..4f3678e1 100644 --- a/tests/inputs/test_speaker_encoder_config.json +++ b/tests/inputs/test_speaker_encoder_config.json @@ -1,5 +1,6 @@ { + "model": "speaker_encoder", "run_name": "test_speaker_encoder", "run_description": "test speaker encoder.", "audio":{ @@ -42,8 +43,9 @@ "checkpoint": true, // If true, it saves checkpoints per "save_step" "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. "print_step": 20, // Number of steps to log traning on console. + "batch_size": 32, "output_path": "", // DATASET-RELATED: output path for all training outputs. - "model": { + "model_params": { "input_dim": 40, "proj_dim": 256, "lstm_dim": 768, diff --git a/tests/inputs/test_speedy_speech.json b/tests/inputs/test_speedy_speech.json index c4e27737..a29fc992 100644 --- a/tests/inputs/test_speedy_speech.json +++ b/tests/inputs/test_speedy_speech.json @@ -132,7 +132,7 @@ // PHONEMES "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronoun[ciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronoun[ciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST diff --git a/tests/inputs/test_tacotron2_config.json b/tests/inputs/test_tacotron2_config.json index 14449867..cc2c1bb5 100644 --- a/tests/inputs/test_tacotron2_config.json +++ b/tests/inputs/test_tacotron2_config.json @@ -64,7 +64,7 @@ "batch_size": 1, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. "eval_batch_size":1, "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 4]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. + "gradual_training": [[0, 7, 4], [1, 5, 2]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. "loss_masking": true, // enable / disable loss masking against the sequence padding. "ga_alpha": 10.0, // weight for guided attention loss. If > 0, guided attention is enabled. "mixed_precision": false, @@ -141,7 +141,7 @@ // PHONEMES "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST @@ -153,11 +153,11 @@ "gst_style_input": null, // Condition the style input either on a // -> wave file [path to wave] or // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) == len(gst_style_tokens). + // with the dictionary being len(dict) == len(gst_num_style_tokens). "gst_use_speaker_embedding": true, // if true pass speaker embedding in attention input GST. "gst_embedding_dim": 512, "gst_num_heads": 4, - "gst_style_tokens": 10 + "gst_num_style_tokens": 10 }, // DATASETS diff --git a/tests/inputs/test_tacotron_bd_config.json b/tests/inputs/test_tacotron_bd_config.json index b6092f4f..9d2935aa 100644 --- a/tests/inputs/test_tacotron_bd_config.json +++ b/tests/inputs/test_tacotron_bd_config.json @@ -141,7 +141,7 @@ // PHONEMES "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST diff --git a/tests/inputs/test_tacotron_config.json b/tests/inputs/test_tacotron_config.json index a2fdd690..c8fae623 100644 --- a/tests/inputs/test_tacotron_config.json +++ b/tests/inputs/test_tacotron_config.json @@ -64,7 +64,7 @@ "batch_size": 1, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. "eval_batch_size":1, "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 4]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. + "gradual_training": [[0, 7, 4], [1, 5, 2]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. "loss_masking": true, // enable / disable loss masking against the sequence padding. "ga_alpha": 10.0, // weight for guided attention loss. If > 0, guided attention is enabled. "mixed_precision": false, @@ -141,7 +141,7 @@ // PHONEMES "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST diff --git a/tests/outputs/dummy_model_config.json b/tests/outputs/dummy_model_config.json index 3996e09a..b51bb3a8 100644 --- a/tests/outputs/dummy_model_config.json +++ b/tests/outputs/dummy_model_config.json @@ -87,7 +87,6 @@ // MULTI-SPEAKER and GST "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": true, // use global style tokens "gst": { // gst parameter if gst is enabled "gst_style_input": null, // Condition the style input either on a // -> wave file [path to wave] or diff --git a/tests/test_audio.py b/tests/test_audio.py index 527defa8..22e965f0 100644 --- a/tests/test_audio.py +++ b/tests/test_audio.py @@ -2,22 +2,22 @@ import os import unittest from tests import get_tests_input_path, get_tests_output_path, get_tests_path +from TTS.config import BaseAudioConfig from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config TESTS_PATH = get_tests_path() OUT_PATH = os.path.join(get_tests_output_path(), "audio_tests") WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") os.makedirs(OUT_PATH, exist_ok=True) -conf = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +conf = BaseAudioConfig(mel_fmax=8000) # pylint: disable=protected-access class TestAudio(unittest.TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.ap = AudioProcessor(**conf.audio) + self.ap = AudioProcessor(**conf) def test_audio_synthesis(self): """1. load wav @@ -163,12 +163,12 @@ class TestAudio(unittest.TestCase): def test_scaler(self): scaler_stats_path = os.path.join(get_tests_input_path(), "scale_stats.npy") - conf.audio["stats_path"] = scaler_stats_path - conf.audio["preemphasis"] = 0.0 - conf.audio["do_trim_silence"] = True - conf.audio["signal_norm"] = True + conf.stats_path = scaler_stats_path + conf.preemphasis = 0.0 + conf.do_trim_silence = True + conf.signal_norm = True - ap = AudioProcessor(**conf.audio) + ap = AudioProcessor(**conf) mel_mean, mel_std, linear_mean, linear_std, _ = ap.load_stats(scaler_stats_path) ap.setup_scaler(mel_mean, mel_std, linear_mean, linear_std) diff --git a/tests/test_extract_tts_spectrograms.py b/tests/test_extract_tts_spectrograms.py new file mode 100644 index 00000000..38cee473 --- /dev/null +++ b/tests/test_extract_tts_spectrograms.py @@ -0,0 +1,70 @@ +import os +import unittest + +import torch + +from tests import get_tests_input_path, get_tests_output_path, run_cli +from TTS.config import load_config +from TTS.tts.utils.generic_utils import setup_model +from TTS.tts.utils.text.symbols import phonemes, symbols + +torch.manual_seed(1) + +# pylint: disable=protected-access +class TestExtractTTSSpectrograms(unittest.TestCase): + @staticmethod + def test_GlowTTS(): + # set paths + config_path = os.path.join(get_tests_input_path(), "test_glow_tts.json") + checkpoint_path = os.path.join(get_tests_output_path(), "checkpoint_test.pth.tar") + output_path = os.path.join(get_tests_output_path(), "output_extract_tts_spectrograms/") + # load config + c = load_config(config_path) + # create model + num_chars = len(phonemes if c.use_phonemes else symbols) + model = setup_model(num_chars, 1, c, speaker_embedding_dim=None) + # save model + torch.save({"model": model.state_dict()}, checkpoint_path) + # run test + run_cli( + f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"' + ) + run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') + + @staticmethod + def test_Tacotron2(): + # set paths + config_path = os.path.join(get_tests_input_path(), "test_tacotron2_config.json") + checkpoint_path = os.path.join(get_tests_output_path(), "checkpoint_test.pth.tar") + output_path = os.path.join(get_tests_output_path(), "output_extract_tts_spectrograms/") + # load config + c = load_config(config_path) + # create model + num_chars = len(phonemes if c.use_phonemes else symbols) + model = setup_model(num_chars, 1, c, speaker_embedding_dim=None) + # save model + torch.save({"model": model.state_dict()}, checkpoint_path) + # run test + run_cli( + f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"' + ) + run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') + + @staticmethod + def test_Tacotron(): + # set paths + config_path = os.path.join(get_tests_input_path(), "test_tacotron_config.json") + checkpoint_path = os.path.join(get_tests_output_path(), "checkpoint_test.pth.tar") + output_path = os.path.join(get_tests_output_path(), "output_extract_tts_spectrograms/") + # load config + c = load_config(config_path) + # create model + num_chars = len(phonemes if c.use_phonemes else symbols) + model = setup_model(num_chars, 1, c, speaker_embedding_dim=None) + # save model + torch.save({"model": model.state_dict()}, checkpoint_path) + # run test + run_cli( + f'CUDA_VISIBLE_DEVICES="" python TTS/bin/extract_tts_spectrograms.py --config_path "{config_path}" --checkpoint_path "{checkpoint_path}" --output_path "{output_path}"' + ) + run_cli(f'rm -rf "{output_path}" "{checkpoint_path}"') diff --git a/tests/test_glow_tts.py b/tests/test_glow_tts.py index 8e699faf..486de274 100644 --- a/tests/test_glow_tts.py +++ b/tests/test_glow_tts.py @@ -6,10 +6,10 @@ import torch from torch import optim from tests import get_tests_input_path +from TTS.tts.configs import GlowTTSConfig from TTS.tts.layers.losses import GlowTTSLoss from TTS.tts.models.glow_tts import GlowTTS from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config # pylint: disable=unused-variable @@ -17,7 +17,7 @@ torch.manual_seed(1) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +c = GlowTTSConfig() ap = AudioProcessor(**c.audio) WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") @@ -129,3 +129,58 @@ class GlowTTSTrainTest(unittest.TestCase): count, param.shape, param, param_ref ) count += 1 + + +class GlowTTSInferenceTest(unittest.TestCase): + @staticmethod + def test_inference(): + input_dummy = torch.randint(0, 24, (8, 128)).long().to(device) + input_lengths = torch.randint(100, 129, (8,)).long().to(device) + input_lengths[-1] = 128 + mel_spec = torch.rand(8, c.audio["num_mels"], 30).to(device) + mel_lengths = torch.randint(20, 30, (8,)).long().to(device) + speaker_ids = torch.randint(0, 5, (8,)).long().to(device) + + # create model + model = GlowTTS( + num_chars=32, + hidden_channels_enc=48, + hidden_channels_dec=48, + hidden_channels_dp=32, + out_channels=80, + encoder_type="rel_pos_transformer", + encoder_params={ + "kernel_size": 3, + "dropout_p": 0.1, + "num_layers": 6, + "num_heads": 2, + "hidden_channels_ffn": 16, # 4 times the hidden_channels + "input_length": None, + }, + use_encoder_prenet=True, + num_flow_blocks_dec=12, + kernel_size_dec=5, + dilation_rate=1, + num_block_layers=4, + dropout_p_dec=0.0, + num_speakers=0, + c_in_channels=0, + num_splits=4, + num_squeeze=1, + sigmoid_scale=False, + mean_only=False, + ).to(device) + + model.eval() + print(" > Num parameters for GlowTTS model:%s" % (count_parameters(model))) + + # inference encoder and decoder with MAS + y, *_ = model.inference_with_MAS(input_dummy, input_lengths, mel_spec, mel_lengths, None) + + y_dec, _ = model.decoder_inference(mel_spec, mel_lengths) + + assert ( + y_dec.shape == y.shape + ), "Difference between the shapes of the glowTTS inference with MAS ({}) and the inference using only the decoder ({}) !!".format( + y.shape, y_dec.shape + ) diff --git a/tests/test_loader.py b/tests/test_loader.py index 6174865b..e2dba37a 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -6,17 +6,21 @@ import numpy as np import torch from torch.utils.data import DataLoader -from tests import get_tests_input_path, get_tests_output_path +from tests import get_tests_output_path +from TTS.tts.configs import BaseTTSConfig from TTS.tts.datasets import TTSDataset from TTS.tts.datasets.preprocess import ljspeech from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config # pylint: disable=unused-variable OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") os.makedirs(OUTPATH, exist_ok=True) -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) + +# create a dummy config for testing data loaders. +c = BaseTTSConfig(text_cleaner="english_cleaners", num_loader_workers=0, batch_size=2) +c.r = 5 +c.data_path = "tests/data/ljspeech/" ok_ljspeech = os.path.exists(c.data_path) DATA_EXIST = True @@ -40,7 +44,7 @@ class TestTTSDataset(unittest.TestCase): compute_linear_spec=True, ap=self.ap, meta_data=items, - tp=c.characters if "characters" in c.keys() else None, + tp=c.characters, batch_group_size=bgs, min_seq_len=c.min_seq_len, max_seq_len=float("inf"), diff --git a/tests/test_speaker_encoder.py b/tests/test_speaker_encoder.py index 32ba2924..8939ccf6 100644 --- a/tests/test_speaker_encoder.py +++ b/tests/test_speaker_encoder.py @@ -1,4 +1,3 @@ -import os import unittest import torch as T @@ -6,10 +5,8 @@ import torch as T from tests import get_tests_input_path from TTS.speaker_encoder.losses import AngleProtoLoss, GE2ELoss from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.utils.io import load_config file_path = get_tests_input_path() -c = load_config(os.path.join(file_path, "test_config.json")) class SpeakerEncoderTests(unittest.TestCase): diff --git a/tests/test_speaker_encoder_train.py b/tests/test_speaker_encoder_train.py new file mode 100644 index 00000000..525730f2 --- /dev/null +++ b/tests/test_speaker_encoder_train.py @@ -0,0 +1,47 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.config.shared_configs import BaseAudioConfig +from TTS.speaker_encoder.speaker_encoder_config import SpeakerEncoderConfig + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = SpeakerEncoderConfig( + batch_size=4, + num_speakers_in_batch=1, + num_utters_per_speaker=10, + num_loader_workers=0, + max_train_step=2, + print_step=1, + save_step=1, + print_eval=True, + audio=BaseAudioConfig(num_mels=40), +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_encoder.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_encoder.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/test_speaker_manager.py b/tests/test_speaker_manager.py index b176e353..ffb98ed7 100644 --- a/tests/test_speaker_manager.py +++ b/tests/test_speaker_manager.py @@ -5,11 +5,11 @@ import numpy as np import torch from tests import get_tests_input_path +from TTS.config import load_config from TTS.speaker_encoder.model import SpeakerEncoder -from TTS.speaker_encoder.utils.generic_utils import save_checkpoint +from TTS.speaker_encoder.utils.io import save_checkpoint from TTS.tts.utils.speakers import SpeakerManager from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config encoder_config_path = os.path.join(get_tests_input_path(), "test_speaker_encoder_config.json") encoder_model_path = os.path.join(get_tests_input_path(), "checkpoint_0.pth.tar") @@ -25,11 +25,11 @@ class SpeakerManagerTest(unittest.TestCase): def test_speaker_embedding(): # load config config = load_config(encoder_config_path) - config["audio"]["resample"] = True + config.audio.resample = True # create a dummy speaker encoder - model = SpeakerEncoder(**config.model) - save_checkpoint(model, None, None, get_tests_input_path(), 0, 0) + model = SpeakerEncoder(**config.model_params) + save_checkpoint(model, None, None, get_tests_input_path(), 0) # load audio processor and speaker encoder ap = AudioProcessor(**config.audio) diff --git a/tests/test_synthesize.py b/tests/test_synthesize.py index 526f7dc8..62eb6dbe 100644 --- a/tests/test_synthesize.py +++ b/tests/test_synthesize.py @@ -10,19 +10,19 @@ def test_synthesize(): # single speaker model run_cli(f'tts --text "This is an example." --out_path "{output_path}"') - run_cli( - "tts --model_name tts_models/en/ljspeech/speedy-speech-wn " - f'--text "This is an example." --out_path "{output_path}"' - ) - run_cli( - "tts --model_name tts_models/en/ljspeech/speedy-speech-wn " - "--vocoder_name vocoder_models/en/ljspeech/multiband-melgan " - f'--text "This is an example." --out_path "{output_path}"' - ) + # run_cli( + # "tts --model_name tts_models/en/ljspeech/speedy-speech-wn " + # f'--text "This is an example." --out_path "{output_path}"' + # ) + # run_cli( + # "tts --model_name tts_models/en/ljspeech/speedy-speech-wn " + # "--vocoder_name vocoder_models/en/ljspeech/multiband-melgan " + # f'--text "This is an example." --out_path "{output_path}"' + # ) - # multi-speaker model - run_cli("tts --model_name tts_models/en/vctk/sc-glow-tts --list_speaker_idxs") - run_cli( - f'tts --model_name tts_models/en/vctk/sc-glow-tts --speaker_idx "p304" ' - f'--text "This is an example." --out_path "{output_path}"' - ) + # # multi-speaker model + # run_cli("tts --model_name tts_models/en/vctk/sc-glow-tts --list_speaker_idxs") + # run_cli( + # f'tts --model_name tts_models/en/vctk/sc-glow-tts --speaker_idx "p304" ' + # f'--text "This is an example." --out_path "{output_path}"' + # ) diff --git a/tests/test_synthesizer.py b/tests/test_synthesizer.py index aa06a6f0..a1cd4de5 100644 --- a/tests/test_synthesizer.py +++ b/tests/test_synthesizer.py @@ -1,11 +1,11 @@ import os import unittest -from tests import get_tests_input_path, get_tests_output_path +from tests import get_tests_output_path +from TTS.config import load_config from TTS.tts.utils.generic_utils import setup_model from TTS.tts.utils.io import save_checkpoint from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols -from TTS.utils.io import load_config from TTS.utils.synthesizer import Synthesizer @@ -15,8 +15,8 @@ class SynthesizerTest(unittest.TestCase): # pylint: disable=global-statement global symbols, phonemes config = load_config(os.path.join(get_tests_output_path(), "dummy_model_config.json")) - if "characters" in config.keys(): - symbols, phonemes = make_symbols(**config.characters) + if config.has("characters") and config.characters: + symbols, phonemes = make_symbols(**config.characters.to_dict()) num_chars = len(phonemes) if config.use_phonemes else len(symbols) model = setup_model(num_chars, 0, config) @@ -25,11 +25,10 @@ class SynthesizerTest(unittest.TestCase): def test_in_out(self): self._create_random_model() - config = load_config(os.path.join(get_tests_input_path(), "server_config.json")) tts_root_path = get_tests_output_path() - config["tts_checkpoint"] = os.path.join(tts_root_path, config["tts_checkpoint"]) - config["tts_config"] = os.path.join(tts_root_path, config["tts_config"]) - synthesizer = Synthesizer(config["tts_checkpoint"], config["tts_config"], None, None) + tts_checkpoint = os.path.join(tts_root_path, "checkpoint_10.pth.tar") + tts_config = os.path.join(tts_root_path, "dummy_model_config.json") + synthesizer = Synthesizer(tts_checkpoint, tts_config, None, None) synthesizer.tts("Better this test works!!") def test_split_into_sentences(self): diff --git a/tests/test_tacotron2_model.py b/tests/test_tacotron2_model.py index 0e35605f..4d711700 100644 --- a/tests/test_tacotron2_model.py +++ b/tests/test_tacotron2_model.py @@ -6,10 +6,10 @@ import torch from torch import nn, optim from tests import get_tests_input_path +from TTS.tts.configs import Tacotron2Config from TTS.tts.layers.losses import MSELossMasked from TTS.tts.models.tacotron2 import Tacotron2 from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config # pylint: disable=unused-variable @@ -17,7 +17,7 @@ torch.manual_seed(1) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +c = Tacotron2Config() ap = AudioProcessor(**c.audio) WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") @@ -148,15 +148,7 @@ class TacotronGSTTrainTest(unittest.TestCase): criterion = MSELossMasked(seq_len_norm=False).to(device) criterion_st = nn.BCEWithLogitsLoss().to(device) - model = Tacotron2( - num_chars=24, - r=c.r, - num_speakers=5, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - ).to(device) + model = Tacotron2(num_chars=24, r=c.r, num_speakers=5, use_gst=True, gst=c.gst).to(device) model.train() model_ref = copy.deepcopy(model) count = 0 @@ -212,15 +204,7 @@ class TacotronGSTTrainTest(unittest.TestCase): criterion = MSELossMasked(seq_len_norm=False).to(device) criterion_st = nn.BCEWithLogitsLoss().to(device) - model = Tacotron2( - num_chars=24, - r=c.r, - num_speakers=5, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - ).to(device) + model = Tacotron2(num_chars=24, r=c.r, num_speakers=5, use_gst=True, gst=c.gst).to(device) model.train() model_ref = copy.deepcopy(model) count = 0 @@ -275,17 +259,9 @@ class SCGSTMultiSpeakeTacotronTrainTest(unittest.TestCase): stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float().squeeze() criterion = MSELossMasked(seq_len_norm=False).to(device) criterion_st = nn.BCEWithLogitsLoss().to(device) - model = Tacotron2( - num_chars=24, - r=c.r, - num_speakers=5, - speaker_embedding_dim=55, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"], - ).to(device) + model = Tacotron2(num_chars=24, r=c.r, num_speakers=5, speaker_embedding_dim=55, use_gst=True, gst=c.gst).to( + device + ) model.train() model_ref = copy.deepcopy(model) count = 0 diff --git a/tests/test_tacotron2_tf_model.py b/tests/test_tacotron2_tf_model.py index 2c4b4884..ee7f720b 100644 --- a/tests/test_tacotron2_tf_model.py +++ b/tests/test_tacotron2_tf_model.py @@ -5,10 +5,9 @@ import numpy as np import tensorflow as tf import torch -from tests import get_tests_input_path +from TTS.tts.configs import Tacotron2Config from TTS.tts.tf.models.tacotron2 import Tacotron2 from TTS.tts.tf.utils.tflite import convert_tacotron2_to_tflite, load_tflite_model -from TTS.utils.io import load_config tf.get_logger().setLevel("INFO") @@ -19,7 +18,7 @@ torch.manual_seed(1) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +c = Tacotron2Config() class TacotronTFTrainTest(unittest.TestCase): diff --git a/tests/test_tacotron_model.py b/tests/test_tacotron_model.py index e3ed8ae2..fcbac0f7 100644 --- a/tests/test_tacotron_model.py +++ b/tests/test_tacotron_model.py @@ -6,10 +6,10 @@ import torch from torch import nn, optim from tests import get_tests_input_path +from TTS.tts.configs import TacotronConfig from TTS.tts.layers.losses import L1LossMasked from TTS.tts.models.tacotron import Tacotron from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config # pylint: disable=unused-variable @@ -17,7 +17,7 @@ torch.manual_seed(1) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -c = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +c = TacotronConfig() ap = AudioProcessor(**c.audio) WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") @@ -37,6 +37,7 @@ class TacotronTrainTest(unittest.TestCase): mel_spec = torch.rand(8, 30, c.audio["num_mels"]).to(device) linear_spec = torch.rand(8, 30, c.audio["fft_size"]).to(device) mel_lengths = torch.randint(20, 30, (8,)).long().to(device) + mel_lengths[-1] = mel_spec.size(1) stop_targets = torch.zeros(8, 30, 1).float().to(device) speaker_ids = torch.randint(0, 5, (8,)).long().to(device) @@ -96,6 +97,7 @@ class MultiSpeakeTacotronTrainTest(unittest.TestCase): mel_spec = torch.rand(8, 30, c.audio["num_mels"]).to(device) linear_spec = torch.rand(8, 30, c.audio["fft_size"]).to(device) mel_lengths = torch.randint(20, 30, (8,)).long().to(device) + mel_lengths[-1] = mel_spec.size(1) stop_targets = torch.zeros(8, 30, 1).float().to(device) speaker_embeddings = torch.rand(8, 55).to(device) @@ -172,10 +174,8 @@ class TacotronGSTTrainTest(unittest.TestCase): model = Tacotron( num_chars=32, num_speakers=5, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], + use_gst=True, + gst=c.gst, postnet_output_dim=c.audio["fft_size"], decoder_output_dim=c.audio["num_mels"], r=c.r, @@ -237,10 +237,8 @@ class TacotronGSTTrainTest(unittest.TestCase): model = Tacotron( num_chars=32, num_speakers=5, - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], + use_gst=True, + gst=c.gst, postnet_output_dim=c.audio["fft_size"], decoder_output_dim=c.audio["num_mels"], r=c.r, @@ -303,11 +301,8 @@ class SCGSTMultiSpeakeTacotronTrainTest(unittest.TestCase): num_speakers=5, postnet_output_dim=c.audio["fft_size"], decoder_output_dim=c.audio["num_mels"], - gst=True, - gst_embedding_dim=c.gst["gst_embedding_dim"], - gst_num_heads=c.gst["gst_num_heads"], - gst_style_tokens=c.gst["gst_style_tokens"], - gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"], + use_gst=True, + gst=c.gst, r=c.r, memory_size=c.memory_size, speaker_embedding_dim=55, diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py deleted file mode 100644 index f70056b1..00000000 --- a/tests/test_text_processing.py +++ /dev/null @@ -1,176 +0,0 @@ -import os - -# pylint: disable=unused-wildcard-import -# pylint: disable=wildcard-import -# pylint: disable=unused-import -from tests import get_tests_input_path, get_tests_path -from TTS.tts.utils.text import * -from TTS.utils.io import load_config - -conf = load_config(os.path.join(get_tests_input_path(), "test_config.json")) - - -def test_phoneme_to_sequence(): - - text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" - text_cleaner = ["phoneme_cleaners"] - lang = "en-us" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "ɹiːsənt ɹᵻsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪŋkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹᵻspɑːnsᵻbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjʊleɪʃən ænd lɜːnɪŋ!" - assert text_hat == text_hat_with_params == gt - - # multiple punctuations - text = "Be a voice, not an! echo?" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ?" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # not ending with punctuation - text = "Be a voice, not an! echo" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # original - text = "Be a voice, not an echo!" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # extra space after the sentence - text = "Be a voice, not an! echo. " - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ." - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # extra space after the sentence - text = "Be a voice, not an! echo. " - sequence = phoneme_to_sequence(text, text_cleaner, lang, True) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "^biː ɐ vɔɪs, nɑːt æn! ɛkoʊ.~" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # padding char - text = "_Be a _voice, not an! echo_" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - -def test_phoneme_to_sequence_with_blank_token(): - - text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" - text_cleaner = ["phoneme_cleaners"] - lang = "en-us" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "ɹiːsənt ɹᵻsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪŋkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹᵻspɑːnsᵻbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjʊleɪʃən ænd lɜːnɪŋ!" - assert text_hat == text_hat_with_params == gt - - # multiple punctuations - text = "Be a voice, not an! echo?" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ?" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # not ending with punctuation - text = "Be a voice, not an! echo" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # original - text = "Be a voice, not an echo!" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # extra space after the sentence - text = "Be a voice, not an! echo. " - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ." - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # extra space after the sentence - text = "Be a voice, not an! echo. " - sequence = phoneme_to_sequence(text, text_cleaner, lang, True) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "^biː ɐ vɔɪs, nɑːt æn! ɛkoʊ.~" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - # padding char - text = "_Be a _voice, not an! echo_" - sequence = phoneme_to_sequence(text, text_cleaner, lang) - text_hat = sequence_to_phoneme(sequence) - _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) - text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) - gt = "biː ɐ vɔɪs, nɑːt æn! ɛkoʊ" - print(text_hat) - print(len(sequence)) - assert text_hat == text_hat_with_params == gt - - -def test_text2phone(): - text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" - gt = "ɹ|iː|s|ə|n|t| |ɹ|ᵻ|s|ɜː|tʃ| |æ|t| |h|ɑːɹ|v|ɚ|d| |h|ɐ|z| |ʃ|oʊ|n| |m|ɛ|d|ᵻ|t|eɪ|ɾ|ɪ|ŋ| |f|ɔː|ɹ| |æ|z| |l|ɪ|ɾ|əl| |æ|z| |eɪ|t| |w|iː|k|s| |k|æ|n| |æ|k|tʃ|uː|əl|i| |ɪ|ŋ|k|ɹ|iː|s|,| |ð|ə| |ɡ|ɹ|eɪ| |m|æ|ɾ|ɚ|ɹ| |ɪ|n|ð|ə| |p|ɑːɹ|t|s| |ʌ|v|ð|ə| |b|ɹ|eɪ|n| |ɹ|ᵻ|s|p|ɑː|n|s|ᵻ|b|əl| |f|ɔː|ɹ| |ɪ|m|oʊ|ʃ|ə|n|əl| |ɹ|ɛ|ɡ|j|ʊ|l|eɪ|ʃ|ə|n| |æ|n|d| |l|ɜː|n|ɪ|ŋ|!" - lang = "en-us" - ph = text2phone(text, lang) - assert gt == ph diff --git a/TTS/tts/configs/aligntts_transformers.json b/tests/tts_tests/__init__.py similarity index 100% rename from TTS/tts/configs/aligntts_transformers.json rename to tests/tts_tests/__init__.py diff --git a/tests/tts_tests/test_align_tts_train.py b/tests/tts_tests/test_align_tts_train.py new file mode 100644 index 00000000..848f46c1 --- /dev/null +++ b/tests/tts_tests/test_align_tts_train.py @@ -0,0 +1,50 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.tts.configs import AlignTTSConfig + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = AlignTTSConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=False, + phoneme_language="en-us", + phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_align_tts.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_align_tts.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_glow_tts_train.py b/tests/tts_tests/test_glow_tts_train.py new file mode 100644 index 00000000..00c7e852 --- /dev/null +++ b/tests/tts_tests/test_glow_tts_train.py @@ -0,0 +1,51 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.tts.configs import GlowTTSConfig + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = GlowTTSConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=True, + phoneme_language="zh-CN", + phoneme_cache_path='tests/data/ljspeech/phoneme_cache/', + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_glow_tts.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " + "--coqpit.datasets.0.meta_file_attn_mask tests/data/ljspeech/metadata_attn_mask.txt" +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_glow_tts.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_speedy_speech_train.py b/tests/tts_tests/test_speedy_speech_train.py new file mode 100644 index 00000000..cc2845c2 --- /dev/null +++ b/tests/tts_tests/test_speedy_speech_train.py @@ -0,0 +1,51 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.tts.configs import SpeedySpeechConfig + +config_path = os.path.join(get_tests_output_path(), "test_speedy_speech_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = SpeedySpeechConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=True, + phoneme_language="zh-CN", + phoneme_cache_path='tests/data/ljspeech/phoneme_cache/', + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_speedy_speech.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " + "--coqpit.datasets.0.meta_file_attn_mask tests/data/ljspeech/metadata_attn_mask.txt" +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_speedy_speech.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_tacotron2_train.py b/tests/tts_tests/test_tacotron2_train.py new file mode 100644 index 00000000..dbec309b --- /dev/null +++ b/tests/tts_tests/test_tacotron2_train.py @@ -0,0 +1,51 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.tts.configs import Tacotron2Config + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = Tacotron2Config( + r=5, + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=False, + phoneme_language="en-us", + phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/tts_tests/test_tacotron_train.py b/tests/tts_tests/test_tacotron_train.py new file mode 100644 index 00000000..34ee6e06 --- /dev/null +++ b/tests/tts_tests/test_tacotron_train.py @@ -0,0 +1,50 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.tts.configs import TacotronConfig + +config_path = os.path.join(get_tests_output_path(), "test_model_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = TacotronConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + text_cleaner="english_cleaners", + use_phonemes=False, + phoneme_language="en-us", + phoneme_cache_path=os.path.join(get_tests_output_path(), "train_outputs/phoneme_cache/"), + run_eval=True, + test_delay_epochs=-1, + epochs=1, + print_step=1, + print_eval=True, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --config_path {config_path} " + f"--coqpit.output_path {output_path} " + "--coqpit.datasets.0.name ljspeech " + "--coqpit.datasets.0.meta_file_train metadata.csv " + "--coqpit.datasets.0.meta_file_val metadata.csv " + "--coqpit.datasets.0.path tests/data/ljspeech " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_tacotron.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/__init__.py b/tests/vocoder_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/vocoder_tests/test_fullband_melgan_train.py b/tests/vocoder_tests/test_fullband_melgan_train.py new file mode 100644 index 00000000..d9bc51ac --- /dev/null +++ b/tests/vocoder_tests/test_fullband_melgan_train.py @@ -0,0 +1,44 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.vocoder.configs import FullbandMelganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = FullbandMelganConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_hifigan_train.py b/tests/vocoder_tests/test_hifigan_train.py new file mode 100644 index 00000000..11057570 --- /dev/null +++ b/tests/vocoder_tests/test_hifigan_train.py @@ -0,0 +1,45 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.vocoder.configs import HifiganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + + +config = HifiganConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=1024, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_melgan_train.py b/tests/vocoder_tests/test_melgan_train.py new file mode 100644 index 00000000..de48ca24 --- /dev/null +++ b/tests/vocoder_tests/test_melgan_train.py @@ -0,0 +1,44 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.vocoder.configs import MelganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = MelganConfig( + batch_size=4, + eval_batch_size=4, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=2048, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_multiband_melgan_train.py b/tests/vocoder_tests/test_multiband_melgan_train.py new file mode 100644 index 00000000..081fb40e --- /dev/null +++ b/tests/vocoder_tests/test_multiband_melgan_train.py @@ -0,0 +1,44 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.vocoder.configs import MultibandMelganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = MultibandMelganConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_parallel_wavegan_train.py b/tests/vocoder_tests/test_parallel_wavegan_train.py new file mode 100644 index 00000000..fb6ea87c --- /dev/null +++ b/tests/vocoder_tests/test_parallel_wavegan_train.py @@ -0,0 +1,44 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.vocoder.configs import ParallelWaveganConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = ParallelWaveganConfig( + batch_size=4, + eval_batch_size=4, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=2048, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_gan.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/test_vocoder_gan_datasets.py b/tests/vocoder_tests/test_vocoder_gan_datasets.py similarity index 94% rename from tests/test_vocoder_gan_datasets.py rename to tests/vocoder_tests/test_vocoder_gan_datasets.py index 7202d06e..c39d70e9 100644 --- a/tests/test_vocoder_gan_datasets.py +++ b/tests/vocoder_tests/test_vocoder_gan_datasets.py @@ -3,9 +3,9 @@ import os import numpy as np from torch.utils.data import DataLoader -from tests import get_tests_input_path, get_tests_output_path, get_tests_path +from tests import get_tests_output_path, get_tests_path from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.vocoder.configs import BaseGANVocoderConfig from TTS.vocoder.datasets.gan_dataset import GANDataset from TTS.vocoder.datasets.preprocess import load_wav_data @@ -13,7 +13,7 @@ file_path = os.path.dirname(os.path.realpath(__file__)) OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") os.makedirs(OUTPATH, exist_ok=True) -C = load_config(os.path.join(get_tests_input_path(), "test_config.json")) +C = BaseGANVocoderConfig() test_data_path = os.path.join(get_tests_path(), "data/ljspeech/") ok_ljspeech = os.path.exists(test_data_path) @@ -46,6 +46,8 @@ def gan_dataset_case( def check_item(feat, wav): """Pass a single pair of features and waveform""" + feat = feat.numpy() + wav = wav.numpy() expected_feat_shape = (batch_size, ap.num_mels, seq_len // hop_len + conv_pad * 2) # check shapes @@ -61,7 +63,7 @@ def gan_dataset_case( # the first 2 and the last 2 frames are skipped due to the padding # differences in stft max_diff = abs((feat - mel[:, : feat.shape[-1]])[:, 2:-2]).max() - assert max_diff <= 0, f" [!] {max_diff}" + assert max_diff <= 1e-6, f" [!] {max_diff}" # return random segments or return the whole audio if return_segments: @@ -69,7 +71,6 @@ def gan_dataset_case( for item1, item2 in loader: feat1, wav1 = item1 feat2, wav2 = item2 - check_item(feat1, wav1) check_item(feat2, wav2) count_iter += 1 diff --git a/tests/test_vocoder_losses.py b/tests/vocoder_tests/test_vocoder_losses.py similarity index 95% rename from tests/test_vocoder_losses.py rename to tests/vocoder_tests/test_vocoder_losses.py index 87151a05..2a35aa2e 100644 --- a/tests/test_vocoder_losses.py +++ b/tests/vocoder_tests/test_vocoder_losses.py @@ -3,8 +3,8 @@ import os import torch from tests import get_tests_input_path, get_tests_output_path, get_tests_path +from TTS.config import BaseAudioConfig from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config from TTS.vocoder.layers.losses import MelganFeatureLoss, MultiScaleSTFTLoss, STFTLoss, TorchSTFT TESTS_PATH = get_tests_path() @@ -14,8 +14,7 @@ os.makedirs(OUT_PATH, exist_ok=True) WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") -C = load_config(os.path.join(get_tests_input_path(), "test_config.json")) -ap = AudioProcessor(**C.audio) +ap = AudioProcessor(**BaseAudioConfig().to_dict()) def test_torch_stft(): diff --git a/tests/test_vocoder_melgan_discriminator.py b/tests/vocoder_tests/test_vocoder_melgan_discriminator.py similarity index 100% rename from tests/test_vocoder_melgan_discriminator.py rename to tests/vocoder_tests/test_vocoder_melgan_discriminator.py diff --git a/tests/test_vocoder_melgan_generator.py b/tests/vocoder_tests/test_vocoder_melgan_generator.py similarity index 100% rename from tests/test_vocoder_melgan_generator.py rename to tests/vocoder_tests/test_vocoder_melgan_generator.py diff --git a/tests/test_vocoder_parallel_wavegan_discriminator.py b/tests/vocoder_tests/test_vocoder_parallel_wavegan_discriminator.py similarity index 100% rename from tests/test_vocoder_parallel_wavegan_discriminator.py rename to tests/vocoder_tests/test_vocoder_parallel_wavegan_discriminator.py diff --git a/tests/test_vocoder_parallel_wavegan_generator.py b/tests/vocoder_tests/test_vocoder_parallel_wavegan_generator.py similarity index 100% rename from tests/test_vocoder_parallel_wavegan_generator.py rename to tests/vocoder_tests/test_vocoder_parallel_wavegan_generator.py diff --git a/tests/test_vocoder_pqmf.py b/tests/vocoder_tests/test_vocoder_pqmf.py similarity index 100% rename from tests/test_vocoder_pqmf.py rename to tests/vocoder_tests/test_vocoder_pqmf.py diff --git a/tests/test_vocoder_rwd.py b/tests/vocoder_tests/test_vocoder_rwd.py similarity index 100% rename from tests/test_vocoder_rwd.py rename to tests/vocoder_tests/test_vocoder_rwd.py diff --git a/tests/test_vocoder_tf_melgan_generator.py b/tests/vocoder_tests/test_vocoder_tf_melgan_generator.py similarity index 100% rename from tests/test_vocoder_tf_melgan_generator.py rename to tests/vocoder_tests/test_vocoder_tf_melgan_generator.py diff --git a/tests/test_vocoder_tf_pqmf.py b/tests/vocoder_tests/test_vocoder_tf_pqmf.py similarity index 100% rename from tests/test_vocoder_tf_pqmf.py rename to tests/vocoder_tests/test_vocoder_tf_pqmf.py diff --git a/tests/test_vocoder_wavernn.py b/tests/vocoder_tests/test_vocoder_wavernn.py similarity index 100% rename from tests/test_vocoder_wavernn.py rename to tests/vocoder_tests/test_vocoder_wavernn.py diff --git a/tests/test_vocoder_wavernn_datasets.py b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py similarity index 93% rename from tests/test_vocoder_wavernn_datasets.py rename to tests/vocoder_tests/test_vocoder_wavernn_datasets.py index 7adc1dea..503b4e24 100644 --- a/tests/test_vocoder_wavernn_datasets.py +++ b/tests/vocoder_tests/test_vocoder_wavernn_datasets.py @@ -4,9 +4,9 @@ import shutil import numpy as np from torch.utils.data import DataLoader -from tests import get_tests_input_path, get_tests_output_path, get_tests_path +from tests import get_tests_output_path, get_tests_path from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.vocoder.configs import WavernnConfig from TTS.vocoder.datasets.preprocess import load_wav_feat_data, preprocess_wav_files from TTS.vocoder.datasets.wavernn_dataset import WaveRNNDataset @@ -14,7 +14,7 @@ file_path = os.path.dirname(os.path.realpath(__file__)) OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") os.makedirs(OUTPATH, exist_ok=True) -C = load_config(os.path.join(get_tests_input_path(), "test_vocoder_wavernn_config.json")) +C = WavernnConfig() test_data_path = os.path.join(get_tests_path(), "data/ljspeech/") test_mel_feat_path = os.path.join(test_data_path, "mel") diff --git a/tests/test_wavegrad_train.py b/tests/vocoder_tests/test_wavegrad.py similarity index 100% rename from tests/test_wavegrad_train.py rename to tests/vocoder_tests/test_wavegrad.py diff --git a/tests/test_wavegrad_layers.py b/tests/vocoder_tests/test_wavegrad_layers.py similarity index 100% rename from tests/test_wavegrad_layers.py rename to tests/vocoder_tests/test_wavegrad_layers.py diff --git a/tests/vocoder_tests/test_wavegrad_train.py b/tests/vocoder_tests/test_wavegrad_train.py new file mode 100644 index 00000000..e222de3a --- /dev/null +++ b/tests/vocoder_tests/test_wavegrad_train.py @@ -0,0 +1,43 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.vocoder.configs import WavegradConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = WavegradConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=8192, + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, + test_noise_schedule={"min_val": 1e-6, "max_val": 1e-2, "num_steps": 2}, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavegrad.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavegrad.py --continue_path {continue_path} " +run_cli(command_train) +shutil.rmtree(continue_path) diff --git a/tests/vocoder_tests/test_wavernn_train.py b/tests/vocoder_tests/test_wavernn_train.py new file mode 100644 index 00000000..414ed719 --- /dev/null +++ b/tests/vocoder_tests/test_wavernn_train.py @@ -0,0 +1,44 @@ +import glob +import os +import shutil + +from tests import get_device_id, get_tests_output_path, run_cli +from TTS.vocoder.configs import WavernnConfig + +config_path = os.path.join(get_tests_output_path(), "test_vocoder_config.json") +output_path = os.path.join(get_tests_output_path(), "train_outputs") + +config = WavernnConfig( + batch_size=8, + eval_batch_size=8, + num_loader_workers=0, + num_val_loader_workers=0, + run_eval=True, + test_delay_epochs=-1, + epochs=1, + seq_len=256, # for shorter test time + eval_split_size=1, + print_step=1, + print_eval=True, + data_path="tests/data/ljspeech", + output_path=output_path, +) +config.audio.do_trim_silence = True +config.audio.trim_db = 60 +config.save_json(config_path) + +# train the model for one epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavernn.py --config_path {config_path} " +) +run_cli(command_train) + +# Find latest folder +continue_path = max(glob.glob(os.path.join(output_path, "*/")), key=os.path.getmtime) + +# restore the model and continue training for one more epoch +command_train = ( + f"CUDA_VISIBLE_DEVICES='{get_device_id()}' python TTS/bin/train_vocoder_wavernn.py --continue_path {continue_path} " +) +run_cli(command_train) +shutil.rmtree(continue_path)