Merge pull request #487 from coqui-ai/dev

v0.0.14

.compute

@@ -1,7 +1,6 @@
 #!/bin/bash
 yes | apt-get install sox
 yes | apt-get install ffmpeg
-yes | apt-get install espeak
 yes | apt-get install tmux
 yes | apt-get install zsh
 sh -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)"

.github/workflows/main.yml

@@ -30,6 +30,7 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+          architecture: x64
       - name: check OS
         run: cat /etc/os-release
       - name: Install dependencies

.gitignore

@@ -132,4 +132,3 @@ notebooks/data/*
 TTS/tts/layers/glow_tts/monotonic_align/core.c
 .vscode-upload.json
 temp_build/*
-recipes/*

.pylintrc

@@ -563,7 +563,7 @@ max-branches=12
 max-locals=15
 
 # Maximum number of parents for a class (see R0901).
-max-parents=7
+max-parents=15
 
 # Maximum number of public methods for a class (see R0904).
 max-public-methods=20

Makefile

@@ -7,7 +7,6 @@ help:
 target_dirs := tests TTS notebooks
 
 system-deps:	## install linux system deps
-	sudo apt-get install -y espeak-ng
 	sudo apt-get install -y libsndfile1-dev
 
 dev-deps:  ## install development deps
@@ -18,9 +17,12 @@ deps:	## install 🐸 requirements.
 	pip install -r requirements.txt
 
 test:	## run tests.
-	nosetests -x --with-cov -cov  --cover-erase --cover-package TTS tests --nologcapture
+	nosetests -x --with-cov -cov  --cover-erase --cover-package TTS tests --nologcapture --with-id
 	./run_bash_tests.sh
 
+test_failed:  ## only run tests failed the last time.
+	nosetests -x --with-cov -cov  --cover-erase --cover-package TTS tests --nologcapture --failed
+
 style:	## update code style.
 	black ${target_dirs}
 	isort ${target_dirs}

TTS/.models.json

@@ -6,7 +6,8 @@
                     "description": "EK1 en-rp tacotron2 by NMStoker",
                     "github_rls_url": "https://github.com/coqui-ai/TTS/releases/download/v0.0.10/tts_models--en--ek1--tacotron2.zip",
                     "default_vocoder": "vocoder_models/en/ek1/wavegrad",
-                    "commit": "c802255"
+                    "commit": "c802255",
+                    "needs_phonemizer": true
                 }
             },
             "ljspeech":{
@@ -17,7 +18,8 @@
                     "commit": "bae2ad0f",
                     "author": "Eren Gölge @erogol",
                     "license": "",
-                    "contact":"egolge@coqui.com"
+                    "contact":"egolge@coqui.com",
+                    "needs_phonemizer": false
                 },
                 "glow-tts":{
                     "description": "",
@@ -27,7 +29,8 @@
                     "commit": "",
                     "author": "Eren Gölge @erogol",
                     "license": "MPL",
-                    "contact":"egolge@coqui.com"
+                    "contact":"egolge@coqui.com",
+                    "needs_phonemizer": true
                 },
                 "tacotron2-DCA": {
                     "description": "",
@@ -36,7 +39,8 @@
                     "commit": "",
                     "author": "Eren Gölge @erogol",
                     "license": "MPL",
-                    "contact":"egolge@coqui.com"
+                    "contact":"egolge@coqui.com",
+                    "needs_phonemizer": true
                 },
                 "speedy-speech-wn":{
                     "description": "Speedy Speech model with wavenet decoder.",
@@ -45,7 +49,8 @@
                     "commit": "77b6145",
                     "author": "Eren Gölge @erogol",
                     "license": "MPL",
-                    "contact":"egolge@coqui.com"
+                    "contact":"egolge@coqui.com",
+                    "needs_phonemizer": true
                 }
             },
             "vctk":{
@@ -56,7 +61,9 @@
                     "commit": "b531fa69",
                     "author": "Edresson Casanova",
                     "license": "",
-                    "contact":""
+                    "contact":"",
+                    "needs_phonemizer": true
+
 
                 }
             },
@@ -68,7 +75,8 @@
                     "commit": "bae2ad0f",
                     "author": "Eren Gölge @erogol",
                     "license": "",
-                    "contact":"egolge@coqui.com"
+                    "contact":"egolge@coqui.com",
+                    "needs_phonemizer": true
                 }
             }
         },
@@ -80,7 +88,8 @@
                     "commit": "",
                     "author": "Eren Gölge @erogol",
                     "license": "MPL",
-                    "contact":"egolge@coqui.com"
+                    "contact":"egolge@coqui.com",
+                    "needs_phonemizer": true
                 }
             }
         },
@@ -92,7 +101,8 @@
                     "commit": "",
                     "author": "Eren Gölge @erogol",
                     "license": "MPL",
-                    "contact":"egolge@coqui.com"
+                    "contact":"egolge@coqui.com",
+                    "needs_phonemizer": true
                 }
             }
         },
@@ -112,7 +122,8 @@
                     "author": "@r-dh",
                     "default_vocoder": "vocoder_models/nl/mai/parallel-wavegan",
                     "stats_file": null,
-                    "commit": "540d811"
+                    "commit": "540d811",
+                    "needs_phonemizer": true
                 }
             }
         },
@@ -123,7 +134,8 @@
                     "author": "@erogol",
                     "default_vocoder": "vocoder_models/universal/libri-tts/fullband-melgan",
                     "license":"",
-                    "contact": "egolge@coqui.com"
+                    "contact": "egolge@coqui.com",
+                    "needs_phonemizer": true
                 }
             }
         },
@@ -133,7 +145,8 @@
                     "github_rls_url": "https://github.com/coqui-ai/TTS/releases/download/v0.0.11/tts_models--de--thorsten--tacotron2-DCA.zip",
                     "default_vocoder": "vocoder_models/de/thorsten/wavegrad",
                     "author": "@thorstenMueller",
-                    "commit": "unknown"
+                    "commit": "unknown",
+                    "needs_phonemizer": true
                 }
             }
         }

TTS/__init__.py

@@ -0,0 +1 @@
+from ._version import __version__

TTS/_version.py

@@ -0,0 +1 @@
+__version__ = "0.0.14"

TTS/bin/compute_statistics.py

@@ -8,31 +8,38 @@ import os
 import numpy as np
 from tqdm import tqdm
 
+# from TTS.utils.io import load_config
+from TTS.config import load_config
 from TTS.tts.datasets.preprocess import load_meta_data
 from TTS.utils.audio import AudioProcessor
-from TTS.utils.io import load_config
 
 
 def main():
     """Run preprocessing process."""
     parser = argparse.ArgumentParser(description="Compute mean and variance of spectrogtram features.")
+    parser.add_argument("config_path", type=str, help="TTS config file path to define audio processin parameters.")
+    parser.add_argument("out_path", type=str, help="save path (directory and filename).")
     parser.add_argument(
-        "--config_path", type=str, required=True, help="TTS config file path to define audio processin parameters."
+        "--data_path",
+        type=str,
+        required=False,
+        help="folder including the target set of wavs overriding dataset config.",
     )
-    parser.add_argument("--out_path", type=str, required=True, help="save path (directory and filename).")
-    args = parser.parse_args()
+    args, overrides = parser.parse_known_args()
+
+    CONFIG = load_config(args.config_path)
+    CONFIG.parse_known_args(overrides, relaxed_parser=True)
 
     # load config
-    CONFIG = load_config(args.config_path)
-    CONFIG.audio["signal_norm"] = False  # do not apply earlier normalization
-    CONFIG.audio["stats_path"] = None  # discard pre-defined stats
+    CONFIG.audio.signal_norm = False  # do not apply earlier normalization
+    CONFIG.audio.stats_path = None  # discard pre-defined stats
 
     # load audio processor
-    ap = AudioProcessor(**CONFIG.audio)
+    ap = AudioProcessor(**CONFIG.audio.to_dict())
 
     # load the meta data of target dataset
-    if "data_path" in CONFIG.keys():
-        dataset_items = glob.glob(os.path.join(CONFIG.data_path, "**", "*.wav"), recursive=True)
+    if args.data_path:
+        dataset_items = glob.glob(os.path.join(args.data_path, "**", "*.wav"), recursive=True)
     else:
         dataset_items = load_meta_data(CONFIG.datasets)[0]  # take only train data
     print(f" > There are {len(dataset_items)} files.")
@@ -73,14 +80,14 @@ def main():
     print(f" > Avg lienar spec scale: {linear_scale.mean()}")
 
     # set default config values for mean-var scaling
-    CONFIG.audio["stats_path"] = output_file_path
-    CONFIG.audio["signal_norm"] = True
+    CONFIG.audio.stats_path = output_file_path
+    CONFIG.audio.signal_norm = True
     # remove redundant values
-    del CONFIG.audio["max_norm"]
-    del CONFIG.audio["min_level_db"]
-    del CONFIG.audio["symmetric_norm"]
-    del CONFIG.audio["clip_norm"]
-    stats["audio_config"] = CONFIG.audio
+    del CONFIG.audio.max_norm
+    del CONFIG.audio.min_level_db
+    del CONFIG.audio.symmetric_norm
+    del CONFIG.audio.clip_norm
+    stats["audio_config"] = CONFIG.audio.to_dict()
     np.save(output_file_path, stats, allow_pickle=True)
     print(f" > stats saved to {output_file_path}")
 

TTS/bin/extract_tts_spectrograms.py

@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+"""Extract Mel spectrograms with teacher forcing."""
+
+import argparse
+import os
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+
+from TTS.config import load_config
+from TTS.tts.datasets.preprocess import load_meta_data
+from TTS.tts.datasets.TTSDataset import MyDataset
+from TTS.tts.utils.generic_utils import setup_model
+from TTS.tts.utils.speakers import parse_speakers
+from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols
+from TTS.utils.audio import AudioProcessor
+from TTS.utils.generic_utils import count_parameters
+
+use_cuda = torch.cuda.is_available()
+
+
+def setup_loader(ap, r, verbose=False):
+    dataset = MyDataset(
+        r,
+        c.text_cleaner,
+        compute_linear_spec=False,
+        meta_data=meta_data,
+        ap=ap,
+        tp=c.characters if "characters" in c.keys() else None,
+        add_blank=c["add_blank"] if "add_blank" in c.keys() else False,
+        batch_group_size=0,
+        min_seq_len=c.min_seq_len,
+        max_seq_len=c.max_seq_len,
+        phoneme_cache_path=c.phoneme_cache_path,
+        use_phonemes=c.use_phonemes,
+        phoneme_language=c.phoneme_language,
+        enable_eos_bos=c.enable_eos_bos_chars,
+        use_noise_augment=False,
+        verbose=verbose,
+        speaker_mapping=speaker_mapping if c.use_speaker_embedding and c.use_external_speaker_embedding_file else None,
+    )
+
+    if c.use_phonemes and c.compute_input_seq_cache:
+        # precompute phonemes to have a better estimate of sequence lengths.
+        dataset.compute_input_seq(c.num_loader_workers)
+    dataset.sort_items()
+
+    loader = DataLoader(
+        dataset,
+        batch_size=c.batch_size,
+        shuffle=False,
+        collate_fn=dataset.collate_fn,
+        drop_last=False,
+        sampler=None,
+        num_workers=c.num_loader_workers,
+        pin_memory=False,
+    )
+    return loader
+
+
+def set_filename(wav_path, out_path):
+    wav_file = os.path.basename(wav_path)
+    file_name = wav_file.split(".")[0]
+    os.makedirs(os.path.join(out_path, "quant"), exist_ok=True)
+    os.makedirs(os.path.join(out_path, "mel"), exist_ok=True)
+    os.makedirs(os.path.join(out_path, "wav_gl"), exist_ok=True)
+    os.makedirs(os.path.join(out_path, "wav"), exist_ok=True)
+    wavq_path = os.path.join(out_path, "quant", file_name)
+    mel_path = os.path.join(out_path, "mel", file_name)
+    wav_gl_path = os.path.join(out_path, "wav_gl", file_name + ".wav")
+    wav_path = os.path.join(out_path, "wav", file_name + ".wav")
+    return file_name, wavq_path, mel_path, wav_gl_path, wav_path
+
+
+def format_data(data):
+    # setup input data
+    text_input = data[0]
+    text_lengths = data[1]
+    speaker_names = data[2]
+    mel_input = data[4]
+    mel_lengths = data[5]
+    item_idx = data[7]
+    attn_mask = data[9]
+    avg_text_length = torch.mean(text_lengths.float())
+    avg_spec_length = torch.mean(mel_lengths.float())
+
+    if c.use_speaker_embedding:
+        if c.use_external_speaker_embedding_file:
+            speaker_embeddings = data[8]
+            speaker_ids = None
+        else:
+            speaker_ids = [speaker_mapping[speaker_name] for speaker_name in speaker_names]
+            speaker_ids = torch.LongTensor(speaker_ids)
+            speaker_embeddings = None
+    else:
+        speaker_embeddings = None
+        speaker_ids = None
+
+    # dispatch data to GPU
+    if use_cuda:
+        text_input = text_input.cuda(non_blocking=True)
+        text_lengths = text_lengths.cuda(non_blocking=True)
+        mel_input = mel_input.cuda(non_blocking=True)
+        mel_lengths = mel_lengths.cuda(non_blocking=True)
+        if speaker_ids is not None:
+            speaker_ids = speaker_ids.cuda(non_blocking=True)
+        if speaker_embeddings is not None:
+            speaker_embeddings = speaker_embeddings.cuda(non_blocking=True)
+
+        if attn_mask is not None:
+            attn_mask = attn_mask.cuda(non_blocking=True)
+    return (
+        text_input,
+        text_lengths,
+        mel_input,
+        mel_lengths,
+        speaker_ids,
+        speaker_embeddings,
+        avg_text_length,
+        avg_spec_length,
+        attn_mask,
+        item_idx,
+    )
+
+
+@torch.no_grad()
+def inference(
+    model_name,
+    model,
+    ap,
+    text_input,
+    text_lengths,
+    mel_input,
+    mel_lengths,
+    attn_mask=None,
+    speaker_ids=None,
+    speaker_embeddings=None,
+):
+    if model_name == "glow_tts":
+        mel_input = mel_input.permute(0, 2, 1)  # B x D x T
+        speaker_c = None
+        if speaker_ids is not None:
+            speaker_c = speaker_ids
+        elif speaker_embeddings is not None:
+            speaker_c = speaker_embeddings
+
+        model_output, *_ = model.inference_with_MAS(
+            text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c
+        )
+        model_output = model_output.transpose(1, 2).detach().cpu().numpy()
+
+    elif "tacotron" in model_name:
+        _, postnet_outputs, *_ = model(
+            text_input,
+            text_lengths,
+            mel_input,
+            mel_lengths,
+            speaker_ids=speaker_ids,
+            speaker_embeddings=speaker_embeddings,
+        )
+        # normalize tacotron output
+        if model_name == "tacotron":
+            mel_specs = []
+            postnet_outputs = postnet_outputs.data.cpu().numpy()
+            for b in range(postnet_outputs.shape[0]):
+                postnet_output = postnet_outputs[b]
+                mel_specs.append(torch.FloatTensor(ap.out_linear_to_mel(postnet_output.T).T))
+            model_output = torch.stack(mel_specs).cpu().numpy()
+
+        elif model_name == "tacotron2":
+            model_output = postnet_outputs.detach().cpu().numpy()
+    return model_output
+
+
+def extract_spectrograms(
+    data_loader, model, ap, output_path, quantized_wav=False, save_audio=False, debug=False, metada_name="metada.txt"
+):
+    model.eval()
+    export_metadata = []
+    for _, data in tqdm(enumerate(data_loader), total=len(data_loader)):
+
+        # format data
+        (
+            text_input,
+            text_lengths,
+            mel_input,
+            mel_lengths,
+            speaker_ids,
+            speaker_embeddings,
+            _,
+            _,
+            attn_mask,
+            item_idx,
+        ) = format_data(data)
+
+        model_output = inference(
+            c.model.lower(),
+            model,
+            ap,
+            text_input,
+            text_lengths,
+            mel_input,
+            mel_lengths,
+            attn_mask,
+            speaker_ids,
+            speaker_embeddings,
+        )
+
+        for idx in range(text_input.shape[0]):
+            wav_file_path = item_idx[idx]
+            wav = ap.load_wav(wav_file_path)
+            _, wavq_path, mel_path, wav_gl_path, wav_path = set_filename(wav_file_path, output_path)
+
+            # quantize and save wav
+            if quantized_wav:
+                wavq = ap.quantize(wav)
+                np.save(wavq_path, wavq)
+
+            # save TTS mel
+            mel = model_output[idx]
+            mel_length = mel_lengths[idx]
+            mel = mel[:mel_length, :].T
+            np.save(mel_path, mel)
+
+            export_metadata.append([wav_file_path, mel_path])
+            if save_audio:
+                ap.save_wav(wav, wav_path)
+
+            if debug:
+                print("Audio for debug saved at:", wav_gl_path)
+                wav = ap.inv_melspectrogram(mel)
+                ap.save_wav(wav, wav_gl_path)
+
+    with open(os.path.join(output_path, metada_name), "w") as f:
+        for data in export_metadata:
+            f.write(f"{data[0]}|{data[1]+'.npy'}\n")
+
+
+def main(args):  # pylint: disable=redefined-outer-name
+    # pylint: disable=global-variable-undefined
+    global meta_data, symbols, phonemes, model_characters, speaker_mapping
+
+    # Audio processor
+    ap = AudioProcessor(**c.audio)
+    if "characters" in c.keys() and c["characters"]:
+        symbols, phonemes = make_symbols(**c.characters)
+
+    # set model characters
+    model_characters = phonemes if c.use_phonemes else symbols
+    num_chars = len(model_characters)
+
+    # load data instances
+    meta_data_train, meta_data_eval = load_meta_data(c.datasets)
+
+    # use eval and training partitions
+    meta_data = meta_data_train + meta_data_eval
+
+    # parse speakers
+    num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, None)
+
+    # setup model
+    model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim)
+
+    # restore model
+    checkpoint = torch.load(args.checkpoint_path, map_location="cpu")
+    model.load_state_dict(checkpoint["model"])
+
+    if use_cuda:
+        model.cuda()
+
+    num_params = count_parameters(model)
+    print("\n > Model has {} parameters".format(num_params), flush=True)
+    # set r
+    r = 1 if c.model.lower() == "glow_tts" else model.decoder.r
+    own_loader = setup_loader(ap, r, verbose=True)
+
+    extract_spectrograms(
+        own_loader,
+        model,
+        ap,
+        args.output_path,
+        quantized_wav=args.quantized,
+        save_audio=args.save_audio,
+        debug=args.debug,
+        metada_name="metada.txt",
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config_path", type=str, help="Path to config file for training.", required=True)
+    parser.add_argument("--checkpoint_path", type=str, help="Model file to be restored.", required=True)
+    parser.add_argument("--output_path", type=str, help="Path to save mel specs", required=True)
+    parser.add_argument("--debug", default=False, action="store_true", help="Save audio files for debug")
+    parser.add_argument("--save_audio", default=False, action="store_true", help="Save audio files")
+    parser.add_argument("--quantized", action="store_true", help="Save quantized audio files")
+    args = parser.parse_args()
+
+    c = load_config(args.config_path)
+    main(args)

TTS/bin/train_align_tts.py

@@ -23,169 +23,304 @@ from TTS.tts.utils.speakers import parse_speakers
 from TTS.tts.utils.synthesis import synthesis
 from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols
 from TTS.tts.utils.visual import plot_alignment, plot_spectrogram
-from TTS.utils.arguments import parse_arguments, process_args
+from TTS.utils.arguments import init_training
 from TTS.utils.audio import AudioProcessor
 from TTS.utils.distribute import init_distributed, reduce_tensor
 from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict
 from TTS.utils.radam import RAdam
 from TTS.utils.training import NoamLR, setup_torch_training_env
 
-if __name__ == "__main__":
-    use_cuda, num_gpus = setup_torch_training_env(True, False)
-    # torch.autograd.set_detect_anomaly(True)
+use_cuda, num_gpus = setup_torch_training_env(True, False)
+# torch.autograd.set_detect_anomaly(True)
 
-    def setup_loader(ap, r, is_val=False, verbose=False):
-        if is_val and not c.run_eval:
-            loader = None
+
+def setup_loader(ap, r, is_val=False, verbose=False):
+    if is_val and not config.run_eval:
+        loader = None
+    else:
+        dataset = MyDataset(
+            r,
+            config.text_cleaner,
+            compute_linear_spec=False,
+            meta_data=meta_data_eval if is_val else meta_data_train,
+            ap=ap,
+            tp=config.characters,
+            add_blank=config["add_blank"],
+            batch_group_size=0 if is_val else config.batch_group_size * config.batch_size,
+            min_seq_len=config.min_seq_len,
+            max_seq_len=config.max_seq_len,
+            phoneme_cache_path=config.phoneme_cache_path,
+            use_phonemes=config.use_phonemes,
+            phoneme_language=config.phoneme_language,
+            enable_eos_bos=config.enable_eos_bos_chars,
+            use_noise_augment=not is_val,
+            verbose=verbose,
+            speaker_mapping=speaker_mapping
+            if config.use_speaker_embedding and config.use_external_speaker_embedding_file
+            else None,
+        )
+
+        if config.use_phonemes and config.compute_input_seq_cache:
+            # precompute phonemes to have a better estimate of sequence lengths.
+            dataset.compute_input_seq(config.num_loader_workers)
+        dataset.sort_items()
+
+        sampler = DistributedSampler(dataset) if num_gpus > 1 else None
+        loader = DataLoader(
+            dataset,
+            batch_size=config.eval_batch_size if is_val else config.batch_size,
+            shuffle=False,
+            collate_fn=dataset.collate_fn,
+            drop_last=False,
+            sampler=sampler,
+            num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers,
+            pin_memory=False,
+        )
+    return loader
+
+
+def format_data(data):
+    # setup input data
+    text_input = data[0]
+    text_lengths = data[1]
+    speaker_names = data[2]
+    mel_input = data[4].permute(0, 2, 1)  # B x D x T
+    mel_lengths = data[5]
+    item_idx = data[7]
+    avg_text_length = torch.mean(text_lengths.float())
+    avg_spec_length = torch.mean(mel_lengths.float())
+
+    if config.use_speaker_embedding:
+        if config.use_external_speaker_embedding_file:
+            # return precomputed embedding vector
+            speaker_c = data[8]
         else:
-            dataset = MyDataset(
-                r,
-                c.text_cleaner,
-                compute_linear_spec=False,
-                meta_data=meta_data_eval if is_val else meta_data_train,
-                ap=ap,
-                tp=c.characters if "characters" in c.keys() else None,
-                add_blank=c["add_blank"] if "add_blank" in c.keys() else False,
-                batch_group_size=0 if is_val else c.batch_group_size * c.batch_size,
-                min_seq_len=c.min_seq_len,
-                max_seq_len=c.max_seq_len,
-                phoneme_cache_path=c.phoneme_cache_path,
-                use_phonemes=c.use_phonemes,
-                phoneme_language=c.phoneme_language,
-                enable_eos_bos=c.enable_eos_bos_chars,
-                use_noise_augment=not is_val,
-                verbose=verbose,
-                speaker_mapping=speaker_mapping
-                if c.use_speaker_embedding and c.use_external_speaker_embedding_file
-                else None,
+            # return speaker_id to be used by an embedding layer
+            speaker_c = [speaker_mapping[speaker_name] for speaker_name in speaker_names]
+            speaker_c = torch.LongTensor(speaker_c)
+    else:
+        speaker_c = None
+    # dispatch data to GPU
+    if use_cuda:
+        text_input = text_input.cuda(non_blocking=True)
+        text_lengths = text_lengths.cuda(non_blocking=True)
+        mel_input = mel_input.cuda(non_blocking=True)
+        mel_lengths = mel_lengths.cuda(non_blocking=True)
+        if speaker_c is not None:
+            speaker_c = speaker_c.cuda(non_blocking=True)
+    return text_input, text_lengths, mel_input, mel_lengths, speaker_c, avg_text_length, avg_spec_length, item_idx
+
+
+def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, training_phase):
+
+    model.train()
+    epoch_time = 0
+    keep_avg = KeepAverage()
+    if use_cuda:
+        batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus))
+    else:
+        batch_n_iter = int(len(data_loader.dataset) / config.batch_size)
+    end_time = time.time()
+    c_logger.print_train_start()
+    scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None
+    for num_iter, data in enumerate(data_loader):
+        start_time = time.time()
+
+        # format data
+        (
+            text_input,
+            text_lengths,
+            mel_targets,
+            mel_lengths,
+            speaker_c,
+            avg_text_length,
+            avg_spec_length,
+            _,
+        ) = format_data(data)
+
+        loader_time = time.time() - end_time
+
+        global_step += 1
+        optimizer.zero_grad()
+
+        # forward pass model
+        with torch.cuda.amp.autocast(enabled=config.mixed_precision):
+            decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward(
+                text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase
             )
 
-            if c.use_phonemes and c.compute_input_seq_cache:
-                # precompute phonemes to have a better estimate of sequence lengths.
-                dataset.compute_input_seq(c.num_loader_workers)
-            dataset.sort_items()
-
-            sampler = DistributedSampler(dataset) if num_gpus > 1 else None
-            loader = DataLoader(
-                dataset,
-                batch_size=c.eval_batch_size if is_val else c.batch_size,
-                shuffle=False,
-                collate_fn=dataset.collate_fn,
-                drop_last=False,
-                sampler=sampler,
-                num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers,
-                pin_memory=False,
+            # compute loss
+            loss_dict = criterion(
+                logp,
+                decoder_output,
+                mel_targets,
+                mel_lengths,
+                dur_output,
+                dur_mas_output,
+                text_lengths,
+                global_step,
+                phase=training_phase,
             )
-        return loader
 
-    def format_data(data):
-        # setup input data
-        text_input = data[0]
-        text_lengths = data[1]
-        speaker_names = data[2]
-        mel_input = data[4].permute(0, 2, 1)  # B x D x T
-        mel_lengths = data[5]
-        item_idx = data[7]
-        avg_text_length = torch.mean(text_lengths.float())
-        avg_spec_length = torch.mean(mel_lengths.float())
+        # backward pass with loss scaling
+        if config.mixed_precision:
+            scaler.scale(loss_dict["loss"]).backward()
+            scaler.unscale_(optimizer)
+            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
+            scaler.step(optimizer)
+            scaler.update()
+        else:
+            loss_dict["loss"].backward()
+            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
+            optimizer.step()
 
-        if c.use_speaker_embedding:
-            if c.use_external_speaker_embedding_file:
-                # return precomputed embedding vector
-                speaker_c = data[8]
+        # setup lr
+        if config.noam_schedule:
+            scheduler.step()
+
+        # current_lr
+        current_lr = optimizer.param_groups[0]["lr"]
+
+        # compute alignment error (the lower the better )
+        align_error = 1 - alignment_diagonal_score(alignments, binary=True)
+        loss_dict["align_error"] = align_error
+
+        step_time = time.time() - start_time
+        epoch_time += step_time
+
+        # aggregate losses from processes
+        if num_gpus > 1:
+            loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus)
+            loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, num_gpus)
+            loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, num_gpus)
+            loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus)
+
+        # detach loss values
+        loss_dict_new = dict()
+        for key, value in loss_dict.items():
+            if isinstance(value, (int, float)):
+                loss_dict_new[key] = value
             else:
-                # return speaker_id to be used by an embedding layer
-                speaker_c = [speaker_mapping[speaker_name] for speaker_name in speaker_names]
-                speaker_c = torch.LongTensor(speaker_c)
-        else:
-            speaker_c = None
-        # dispatch data to GPU
-        if use_cuda:
-            text_input = text_input.cuda(non_blocking=True)
-            text_lengths = text_lengths.cuda(non_blocking=True)
-            mel_input = mel_input.cuda(non_blocking=True)
-            mel_lengths = mel_lengths.cuda(non_blocking=True)
-            if speaker_c is not None:
-                speaker_c = speaker_c.cuda(non_blocking=True)
-        return text_input, text_lengths, mel_input, mel_lengths, speaker_c, avg_text_length, avg_spec_length, item_idx
+                loss_dict_new[key] = value.item()
+        loss_dict = loss_dict_new
 
-    def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, training_phase):
+        # update avg stats
+        update_train_values = dict()
+        for key, value in loss_dict.items():
+            update_train_values["avg_" + key] = value
+        update_train_values["avg_loader_time"] = loader_time
+        update_train_values["avg_step_time"] = step_time
+        keep_avg.update_values(update_train_values)
 
-        model.train()
-        epoch_time = 0
-        keep_avg = KeepAverage()
-        if use_cuda:
-            batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus))
-        else:
-            batch_n_iter = int(len(data_loader.dataset) / c.batch_size)
+        # print training progress
+        if global_step % config.print_step == 0:
+            log_dict = {
+                "avg_spec_length": [avg_spec_length, 1],  # value, precision
+                "avg_text_length": [avg_text_length, 1],
+                "step_time": [step_time, 4],
+                "loader_time": [loader_time, 2],
+                "current_lr": current_lr,
+            }
+            c_logger.print_train_step(batch_n_iter, num_iter, global_step, log_dict, loss_dict, keep_avg.avg_values)
+
+        if args.rank == 0:
+            # Plot Training Iter Stats
+            # reduce TB load
+            if global_step % config.tb_plot_step == 0:
+                iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time}
+                iter_stats.update(loss_dict)
+                tb_logger.tb_train_iter_stats(global_step, iter_stats)
+
+            if global_step % config.save_step == 0:
+                if config.checkpoint:
+                    # save model
+                    save_checkpoint(
+                        model,
+                        optimizer,
+                        global_step,
+                        epoch,
+                        1,
+                        OUT_PATH,
+                        model_characters,
+                        model_loss=loss_dict["loss"],
+                    )
+
+                # wait all kernels to be completed
+                torch.cuda.synchronize()
+
+                # Diagnostic visualizations
+                if decoder_output is not None:
+                    idx = np.random.randint(mel_targets.shape[0])
+                    pred_spec = decoder_output[idx].detach().data.cpu().numpy().T
+                    gt_spec = mel_targets[idx].data.cpu().numpy().T
+                    align_img = alignments[idx].data.cpu()
+
+                    figures = {
+                        "prediction": plot_spectrogram(pred_spec, ap),
+                        "ground_truth": plot_spectrogram(gt_spec, ap),
+                        "alignment": plot_alignment(align_img),
+                    }
+
+                    tb_logger.tb_train_figures(global_step, figures)
+
+                    # Sample audio
+                    train_audio = ap.inv_melspectrogram(pred_spec.T)
+                    tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"])
         end_time = time.time()
-        c_logger.print_train_start()
-        scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None
+
+    # print epoch stats
+    c_logger.print_train_epoch_end(global_step, epoch, epoch_time, keep_avg)
+
+    # Plot Epoch Stats
+    if args.rank == 0:
+        epoch_stats = {"epoch_time": epoch_time}
+        epoch_stats.update(keep_avg.avg_values)
+        tb_logger.tb_train_epoch_stats(global_step, epoch_stats)
+        if config.tb_model_param_stats:
+            tb_logger.tb_model_weights(model, global_step)
+    return keep_avg.avg_values, global_step
+
+
+@torch.no_grad()
+def evaluate(data_loader, model, criterion, ap, global_step, epoch, training_phase):
+    model.eval()
+    epoch_time = 0
+    keep_avg = KeepAverage()
+    c_logger.print_eval_start()
+    if data_loader is not None:
         for num_iter, data in enumerate(data_loader):
             start_time = time.time()
 
             # format data
-            (
-                text_input,
-                text_lengths,
-                mel_targets,
-                mel_lengths,
-                speaker_c,
-                avg_text_length,
-                avg_spec_length,
-                _,
-            ) = format_data(data)
-
-            loader_time = time.time() - end_time
-
-            global_step += 1
-            optimizer.zero_grad()
+            text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _ = format_data(data)
 
             # forward pass model
-            with torch.cuda.amp.autocast(enabled=c.mixed_precision):
+            with torch.cuda.amp.autocast(enabled=config.mixed_precision):
                 decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward(
                     text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase
                 )
 
-                # compute loss
-                loss_dict = criterion(
-                    logp,
-                    decoder_output,
-                    mel_targets,
-                    mel_lengths,
-                    dur_output,
-                    dur_mas_output,
-                    text_lengths,
-                    global_step,
-                    phase=training_phase,
-                )
-
-            # backward pass with loss scaling
-            if c.mixed_precision:
-                scaler.scale(loss_dict["loss"]).backward()
-                scaler.unscale_(optimizer)
-                grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip)
-                scaler.step(optimizer)
-                scaler.update()
-            else:
-                loss_dict["loss"].backward()
-                grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip)
-                optimizer.step()
-
-            # setup lr
-            if c.noam_schedule:
-                scheduler.step()
-
-            # current_lr
-            current_lr = optimizer.param_groups[0]["lr"]
-
-            # compute alignment error (the lower the better )
-            align_error = 1 - alignment_diagonal_score(alignments, binary=True)
-            loss_dict["align_error"] = align_error
+            # compute loss
+            loss_dict = criterion(
+                logp,
+                decoder_output,
+                mel_targets,
+                mel_lengths,
+                dur_output,
+                dur_mas_output,
+                text_lengths,
+                global_step,
+                phase=training_phase,
+            )
 
+            # step time
             step_time = time.time() - start_time
             epoch_time += step_time
 
+            # compute alignment score
+            align_error = 1 - alignment_diagonal_score(alignments, binary=True)
+            loss_dict["align_error"] = align_error
+
             # aggregate losses from processes
             if num_gpus > 1:
                 loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus)
@@ -206,357 +341,222 @@ if __name__ == "__main__":
             update_train_values = dict()
             for key, value in loss_dict.items():
                 update_train_values["avg_" + key] = value
-            update_train_values["avg_loader_time"] = loader_time
-            update_train_values["avg_step_time"] = step_time
             keep_avg.update_values(update_train_values)
 
-            # print training progress
-            if global_step % c.print_step == 0:
-                log_dict = {
-                    "avg_spec_length": [avg_spec_length, 1],  # value, precision
-                    "avg_text_length": [avg_text_length, 1],
-                    "step_time": [step_time, 4],
-                    "loader_time": [loader_time, 2],
-                    "current_lr": current_lr,
-                }
-                c_logger.print_train_step(batch_n_iter, num_iter, global_step, log_dict, loss_dict, keep_avg.avg_values)
+            if config.print_eval:
+                c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values)
 
-            if args.rank == 0:
-                # Plot Training Iter Stats
-                # reduce TB load
-                if global_step % c.tb_plot_step == 0:
-                    iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time}
-                    iter_stats.update(loss_dict)
-                    tb_logger.tb_train_iter_stats(global_step, iter_stats)
-
-                if global_step % c.save_step == 0:
-                    if c.checkpoint:
-                        # save model
-                        save_checkpoint(
-                            model,
-                            optimizer,
-                            global_step,
-                            epoch,
-                            1,
-                            OUT_PATH,
-                            model_characters,
-                            model_loss=loss_dict["loss"],
-                        )
-
-                    # wait all kernels to be completed
-                    torch.cuda.synchronize()
-
-                    # Diagnostic visualizations
-                    if decoder_output is not None:
-                        idx = np.random.randint(mel_targets.shape[0])
-                        pred_spec = decoder_output[idx].detach().data.cpu().numpy().T
-                        gt_spec = mel_targets[idx].data.cpu().numpy().T
-                        align_img = alignments[idx].data.cpu()
-
-                        figures = {
-                            "prediction": plot_spectrogram(pred_spec, ap),
-                            "ground_truth": plot_spectrogram(gt_spec, ap),
-                            "alignment": plot_alignment(align_img),
-                        }
-
-                        tb_logger.tb_train_figures(global_step, figures)
-
-                        # Sample audio
-                        train_audio = ap.inv_melspectrogram(pred_spec.T)
-                        tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"])
-            end_time = time.time()
-
-        # print epoch stats
-        c_logger.print_train_epoch_end(global_step, epoch, epoch_time, keep_avg)
-
-        # Plot Epoch Stats
         if args.rank == 0:
-            epoch_stats = {"epoch_time": epoch_time}
-            epoch_stats.update(keep_avg.avg_values)
-            tb_logger.tb_train_epoch_stats(global_step, epoch_stats)
-            if c.tb_model_param_stats:
-                tb_logger.tb_model_weights(model, global_step)
-        return keep_avg.avg_values, global_step
+            # Diagnostic visualizations
+            idx = np.random.randint(mel_targets.shape[0])
+            pred_spec = decoder_output[idx].detach().data.cpu().numpy().T
+            gt_spec = mel_targets[idx].data.cpu().numpy().T
+            align_img = alignments[idx].data.cpu()
 
-    @torch.no_grad()
-    def evaluate(data_loader, model, criterion, ap, global_step, epoch, training_phase):
-        model.eval()
-        epoch_time = 0
-        keep_avg = KeepAverage()
-        c_logger.print_eval_start()
-        if data_loader is not None:
-            for num_iter, data in enumerate(data_loader):
-                start_time = time.time()
+            eval_figures = {
+                "prediction": plot_spectrogram(pred_spec, ap, output_fig=False),
+                "ground_truth": plot_spectrogram(gt_spec, ap, output_fig=False),
+                "alignment": plot_alignment(align_img, output_fig=False),
+            }
 
-                # format data
-                text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _ = format_data(data)
+            # Sample audio
+            eval_audio = ap.inv_melspectrogram(pred_spec.T)
+            tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"])
 
-                # forward pass model
-                with torch.cuda.amp.autocast(enabled=c.mixed_precision):
-                    decoder_output, dur_output, dur_mas_output, alignments, _, _, logp = model.forward(
-                        text_input, text_lengths, mel_targets, mel_lengths, g=speaker_c, phase=training_phase
-                    )
+            # Plot Validation Stats
+            tb_logger.tb_eval_stats(global_step, keep_avg.avg_values)
+            tb_logger.tb_eval_figures(global_step, eval_figures)
 
-                # compute loss
-                loss_dict = criterion(
-                    logp,
-                    decoder_output,
-                    mel_targets,
-                    mel_lengths,
-                    dur_output,
-                    dur_mas_output,
-                    text_lengths,
-                    global_step,
-                    phase=training_phase,
+    if args.rank == 0 and epoch >= config.test_delay_epochs:
+        if config.test_sentences_file:
+            with open(config.test_sentences_file, "r") as f:
+                test_sentences = [s.strip() for s in f.readlines()]
+        else:
+            test_sentences = [
+                "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
+                "Be a voice, not an echo.",
+                "I'm sorry Dave. I'm afraid I can't do that.",
+                "This cake is great. It's so delicious and moist.",
+                "Prior to November 22, 1963.",
+            ]
+
+        # test sentences
+        test_audios = {}
+        test_figures = {}
+        print(" | > Synthesizing test sentences")
+        if config.use_speaker_embedding:
+            if config.use_external_speaker_embedding_file:
+                speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][
+                    "embedding"
+                ]
+                speaker_id = None
+            else:
+                speaker_id = 0
+                speaker_embedding = None
+        else:
+            speaker_id = None
+            speaker_embedding = None
+
+        for idx, test_sentence in enumerate(test_sentences):
+            try:
+                wav, alignment, _, postnet_output, _, _ = synthesis(
+                    model,
+                    test_sentence,
+                    config,
+                    use_cuda,
+                    ap,
+                    speaker_id=speaker_id,
+                    speaker_embedding=speaker_embedding,
+                    style_wav=None,
+                    truncated=False,
+                    enable_eos_bos_chars=config.enable_eos_bos_chars,  # pylint: disable=unused-argument
+                    use_griffin_lim=True,
+                    do_trim_silence=False,
                 )
 
-                # step time
-                step_time = time.time() - start_time
-                epoch_time += step_time
-
-                # compute alignment score
-                align_error = 1 - alignment_diagonal_score(alignments, binary=True)
-                loss_dict["align_error"] = align_error
-
-                # aggregate losses from processes
-                if num_gpus > 1:
-                    loss_dict["loss_l1"] = reduce_tensor(loss_dict["loss_l1"].data, num_gpus)
-                    loss_dict["loss_ssim"] = reduce_tensor(loss_dict["loss_ssim"].data, num_gpus)
-                    loss_dict["loss_dur"] = reduce_tensor(loss_dict["loss_dur"].data, num_gpus)
-                    loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus)
-
-                # detach loss values
-                loss_dict_new = dict()
-                for key, value in loss_dict.items():
-                    if isinstance(value, (int, float)):
-                        loss_dict_new[key] = value
-                    else:
-                        loss_dict_new[key] = value.item()
-                loss_dict = loss_dict_new
-
-                # update avg stats
-                update_train_values = dict()
-                for key, value in loss_dict.items():
-                    update_train_values["avg_" + key] = value
-                keep_avg.update_values(update_train_values)
-
-                if c.print_eval:
-                    c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values)
-
-            if args.rank == 0:
-                # Diagnostic visualizations
-                idx = np.random.randint(mel_targets.shape[0])
-                pred_spec = decoder_output[idx].detach().data.cpu().numpy().T
-                gt_spec = mel_targets[idx].data.cpu().numpy().T
-                align_img = alignments[idx].data.cpu()
-
-                eval_figures = {
-                    "prediction": plot_spectrogram(pred_spec, ap, output_fig=False),
-                    "ground_truth": plot_spectrogram(gt_spec, ap, output_fig=False),
-                    "alignment": plot_alignment(align_img, output_fig=False),
-                }
-
-                # Sample audio
-                eval_audio = ap.inv_melspectrogram(pred_spec.T)
-                tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"])
-
-                # Plot Validation Stats
-                tb_logger.tb_eval_stats(global_step, keep_avg.avg_values)
-                tb_logger.tb_eval_figures(global_step, eval_figures)
-
-        if args.rank == 0 and epoch >= c.test_delay_epochs:
-            if c.test_sentences_file is None:
-                test_sentences = [
-                    "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
-                    "Be a voice, not an echo.",
-                    "I'm sorry Dave. I'm afraid I can't do that.",
-                    "This cake is great. It's so delicious and moist.",
-                    "Prior to November 22, 1963.",
-                ]
-            else:
-                with open(c.test_sentences_file, "r") as f:
-                    test_sentences = [s.strip() for s in f.readlines()]
-
-            # test sentences
-            test_audios = {}
-            test_figures = {}
-            print(" | > Synthesizing test sentences")
-            if c.use_speaker_embedding:
-                if c.use_external_speaker_embedding_file:
-                    speaker_embedding = speaker_mapping[
-                        list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]
-                    ]["embedding"]
-                    speaker_id = None
-                else:
-                    speaker_id = 0
-                    speaker_embedding = None
-            else:
-                speaker_id = None
-                speaker_embedding = None
-
-            style_wav = c.get("style_wav_for_test")
-            for idx, test_sentence in enumerate(test_sentences):
-                try:
-                    wav, alignment, _, postnet_output, _, _ = synthesis(
-                        model,
-                        test_sentence,
-                        c,
-                        use_cuda,
-                        ap,
-                        speaker_id=speaker_id,
-                        speaker_embedding=speaker_embedding,
-                        style_wav=style_wav,
-                        truncated=False,
-                        enable_eos_bos_chars=c.enable_eos_bos_chars,  # pylint: disable=unused-argument
-                        use_griffin_lim=True,
-                        do_trim_silence=False,
-                    )
-
-                    file_path = os.path.join(AUDIO_PATH, str(global_step))
-                    os.makedirs(file_path, exist_ok=True)
-                    file_path = os.path.join(file_path, "TestSentence_{}.wav".format(idx))
-                    ap.save_wav(wav, file_path)
-                    test_audios["{}-audio".format(idx)] = wav
-                    test_figures["{}-prediction".format(idx)] = plot_spectrogram(postnet_output, ap)
-                    test_figures["{}-alignment".format(idx)] = plot_alignment(alignment)
-                except:  # pylint: disable=bare-except
-                    print(" !! Error creating Test Sentence -", idx)
-                    traceback.print_exc()
-            tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"])
-            tb_logger.tb_test_figures(global_step, test_figures)
-        return keep_avg.avg_values
-
-    def main(args):  # pylint: disable=redefined-outer-name
-        # pylint: disable=global-variable-undefined
-        global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping
-        # Audio processor
-        ap = AudioProcessor(**c.audio)
-        if "characters" in c.keys():
-            symbols, phonemes = make_symbols(**c.characters)
-
-        # DISTRUBUTED
-        if num_gpus > 1:
-            init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"])
-
-        # set model characters
-        model_characters = phonemes if c.use_phonemes else symbols
-        num_chars = len(model_characters)
-
-        # load data instances
-        meta_data_train, meta_data_eval = load_meta_data(c.datasets, eval_split=True)
-
-        # set the portion of the data used for training if set in config.json
-        if "train_portion" in c.keys():
-            meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)]
-        if "eval_portion" in c.keys():
-            meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)]
-
-        # parse speakers
-        num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH)
-
-        # setup model
-        model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim)
-        optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9)
-        criterion = AlignTTSLoss(c)
-
-        if args.restore_path:
-            print(f" > Restoring from {os.path.basename(args.restore_path)} ...")
-            checkpoint = torch.load(args.restore_path, map_location="cpu")
-            try:
-                # TODO: fix optimizer init, model.cuda() needs to be called before
-                # optimizer restore
-                optimizer.load_state_dict(checkpoint["optimizer"])
-                if c.reinit_layers:
-                    raise RuntimeError
-                model.load_state_dict(checkpoint["model"])
+                file_path = os.path.join(AUDIO_PATH, str(global_step))
+                os.makedirs(file_path, exist_ok=True)
+                file_path = os.path.join(file_path, "TestSentence_{}.wav".format(idx))
+                ap.save_wav(wav, file_path)
+                test_audios["{}-audio".format(idx)] = wav
+                test_figures["{}-prediction".format(idx)] = plot_spectrogram(postnet_output, ap)
+                test_figures["{}-alignment".format(idx)] = plot_alignment(alignment)
             except:  # pylint: disable=bare-except
-                print(" > Partial model initialization.")
-                model_dict = model.state_dict()
-                model_dict = set_init_dict(model_dict, checkpoint["model"], c)
-                model.load_state_dict(model_dict)
-                del model_dict
+                print(" !! Error creating Test Sentence -", idx)
+                traceback.print_exc()
+        tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"])
+        tb_logger.tb_test_figures(global_step, test_figures)
+    return keep_avg.avg_values
 
-            for group in optimizer.param_groups:
-                group["initial_lr"] = c.lr
-            print(" > Model restored from step %d" % checkpoint["step"], flush=True)
-            args.restore_step = checkpoint["step"]
-        else:
-            args.restore_step = 0
 
-        if use_cuda:
-            model.cuda()
-            criterion.cuda()
+def main(args):  # pylint: disable=redefined-outer-name
+    # pylint: disable=global-variable-undefined
+    global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping
+    # Audio processor
+    ap = AudioProcessor(**config.audio.to_dict())
+    if config.has("characters") and config.characters:
+        symbols, phonemes = make_symbols(**config.characters.to_dict())
 
-        # DISTRUBUTED
-        if num_gpus > 1:
-            model = DDP_th(model, device_ids=[args.rank])
+    # DISTRUBUTED
+    if num_gpus > 1:
+        init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"])
 
-        if c.noam_schedule:
-            scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1)
-        else:
-            scheduler = None
+    # set model characters
+    model_characters = phonemes if config.use_phonemes else symbols
+    num_chars = len(model_characters)
 
-        num_params = count_parameters(model)
-        print("\n > Model has {} parameters".format(num_params), flush=True)
+    # load data instances
+    meta_data_train, meta_data_eval = load_meta_data(config.datasets, eval_split=True)
 
-        if args.restore_step == 0 or not args.best_path:
-            best_loss = float("inf")
-            print(" > Starting with inf best loss.")
-        else:
-            print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...")
-            best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"]
-            print(f" > Starting with loaded last best loss {best_loss}.")
-        keep_all_best = c.get("keep_all_best", False)
-        keep_after = c.get("keep_after", 10000)  # void if keep_all_best False
+    # parse speakers
+    num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH)
 
-        # define dataloaders
-        train_loader = setup_loader(ap, 1, is_val=False, verbose=True)
-        eval_loader = setup_loader(ap, 1, is_val=True, verbose=True)
+    # setup model
+    model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim)
+    optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9)
+    criterion = AlignTTSLoss(config)
 
-        global_step = args.restore_step
+    if args.restore_path:
+        print(f" > Restoring from {os.path.basename(args.restore_path)} ...")
+        checkpoint = torch.load(args.restore_path, map_location="cpu")
+        try:
+            # TODO: fix optimizer init, model.cuda() needs to be called before
+            # optimizer restore
+            optimizer.load_state_dict(checkpoint["optimizer"])
+            if config.reinit_layers:
+                raise RuntimeError
+            model.load_state_dict(checkpoint["model"])
+        except:  # pylint: disable=bare-except
+            print(" > Partial model initialization.")
+            model_dict = model.state_dict()
+            model_dict = set_init_dict(model_dict, checkpoint["model"], config)
+            model.load_state_dict(model_dict)
+            del model_dict
 
-        def set_phase():
-            """Set AlignTTS training phase"""
-            if isinstance(c.phase_start_steps, list):
-                vals = [i < global_step for i in c.phase_start_steps]
-                if not True in vals:
-                    phase = 0
-                else:
-                    phase = (
-                        len(c.phase_start_steps) - [i < global_step for i in c.phase_start_steps][::-1].index(True) - 1
-                    )
+        for group in optimizer.param_groups:
+            group["initial_lr"] = config.lr
+        print(" > Model restored from step %d" % checkpoint["step"], flush=True)
+        args.restore_step = checkpoint["step"]
+    else:
+        args.restore_step = 0
+
+    if use_cuda:
+        model.cuda()
+        criterion.cuda()
+
+    # DISTRUBUTED
+    if num_gpus > 1:
+        model = DDP_th(model, device_ids=[args.rank])
+
+    if config.noam_schedule:
+        scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1)
+    else:
+        scheduler = None
+
+    num_params = count_parameters(model)
+    print("\n > Model has {} parameters".format(num_params), flush=True)
+
+    if args.restore_step == 0 or not args.best_path:
+        best_loss = float("inf")
+        print(" > Starting with inf best loss.")
+    else:
+        print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...")
+        best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"]
+        print(f" > Starting with loaded last best loss {best_loss}.")
+    keep_all_best = config.keep_all_best
+    keep_after = config.keep_after  # void if keep_all_best False
+
+    # define dataloaders
+    train_loader = setup_loader(ap, 1, is_val=False, verbose=True)
+    eval_loader = setup_loader(ap, 1, is_val=True, verbose=True)
+
+    global_step = args.restore_step
+
+    def set_phase():
+        """Set AlignTTS training phase"""
+        if isinstance(config.phase_start_steps, list):
+            vals = [i < global_step for i in config.phase_start_steps]
+            if not True in vals:
+                phase = 0
             else:
-                phase = None
-            return phase
+                phase = (
+                    len(config.phase_start_steps)
+                    - [i < global_step for i in config.phase_start_steps][::-1].index(True)
+                    - 1
+                )
+        else:
+            phase = None
+        return phase
 
-        for epoch in range(0, c.epochs):
-            cur_phase = set_phase()
-            print(f"\n > Current AlignTTS phase: {cur_phase}")
-            c_logger.print_epoch_start(epoch, c.epochs)
-            train_avg_loss_dict, global_step = train(
-                train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, cur_phase
-            )
-            eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch, cur_phase)
-            c_logger.print_epoch_end(epoch, eval_avg_loss_dict)
-            target_loss = train_avg_loss_dict["avg_loss"]
-            if c.run_eval:
-                target_loss = eval_avg_loss_dict["avg_loss"]
-            best_loss = save_best_model(
-                target_loss,
-                best_loss,
-                model,
-                optimizer,
-                global_step,
-                epoch,
-                1,
-                OUT_PATH,
-                model_characters,
-                keep_all_best=keep_all_best,
-                keep_after=keep_after,
-            )
+    for epoch in range(0, config.epochs):
+        cur_phase = set_phase()
+        print(f"\n > Current AlignTTS phase: {cur_phase}")
+        c_logger.print_epoch_start(epoch, config.epochs)
+        train_avg_loss_dict, global_step = train(
+            train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch, cur_phase
+        )
+        eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch, cur_phase)
+        c_logger.print_epoch_end(epoch, eval_avg_loss_dict)
+        target_loss = train_avg_loss_dict["avg_loss"]
+        if config.run_eval:
+            target_loss = eval_avg_loss_dict["avg_loss"]
+        best_loss = save_best_model(
+            target_loss,
+            best_loss,
+            model,
+            optimizer,
+            global_step,
+            epoch,
+            1,
+            OUT_PATH,
+            model_characters,
+            keep_all_best=keep_all_best,
+            keep_after=keep_after,
+        )
 
-    args = parse_arguments(sys.argv)
-    c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts")
+
+if __name__ == "__main__":
+    args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv)
 
     try:
         main(args)

TTS/bin/train_encoder.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import argparse
 import os
 import sys
 import time
@@ -13,20 +12,13 @@ from torch.utils.data import DataLoader
 from TTS.speaker_encoder.dataset import MyDataset
 from TTS.speaker_encoder.losses import AngleProtoLoss, GE2ELoss
 from TTS.speaker_encoder.model import SpeakerEncoder
-from TTS.speaker_encoder.utils.generic_utils import check_config_speaker_encoder, save_best_model
+from TTS.speaker_encoder.utils.io import save_best_model, save_checkpoint
 from TTS.speaker_encoder.utils.visual import plot_embeddings
 from TTS.tts.datasets.preprocess import load_meta_data
+from TTS.utils.arguments import init_training
 from TTS.utils.audio import AudioProcessor
-from TTS.utils.generic_utils import (
-    count_parameters,
-    create_experiment_folder,
-    get_git_branch,
-    remove_experiment_folder,
-    set_init_dict,
-)
-from TTS.utils.io import copy_model_files, load_config
+from TTS.utils.generic_utils import count_parameters, remove_experiment_folder, set_init_dict
 from TTS.utils.radam import RAdam
-from TTS.utils.tensorboard_logger import TensorboardLogger
 from TTS.utils.training import NoamLR, check_update
 
 torch.backends.cudnn.enabled = True
@@ -105,8 +97,9 @@ def train(model, criterion, optimizer, scheduler, ap, global_step):
 
         # Averaged Loss and Averaged Loader Time
         avg_loss = 0.01 * loss.item() + 0.99 * avg_loss if avg_loss != 0 else loss.item()
+        num_loader_workers = c.num_loader_workers if c.num_loader_workers > 0 else 1
         avg_loader_time = (
-            1 / c.num_loader_workers * loader_time + (c.num_loader_workers - 1) / c.num_loader_workers * avg_loader_time
+            1 / num_loader_workers * loader_time + (num_loader_workers - 1) / num_loader_workers * avg_loader_time
             if avg_loader_time != 0
             else loader_time
         )
@@ -139,8 +132,13 @@ def train(model, criterion, optimizer, scheduler, ap, global_step):
 
         # save best model
         best_loss = save_best_model(model, optimizer, avg_loss, best_loss, OUT_PATH, global_step)
-
         end_time = time.time()
+
+        # checkpoint and check stop train cond.
+        if global_step >= c.max_train_step or global_step % c.save_step == 0:
+            save_checkpoint(model, optimizer, avg_loss, OUT_PATH, global_step)
+            if global_step >= c.max_train_step:
+                break
     return avg_loss, global_step
 
 
@@ -149,12 +147,12 @@ def main(args):  # pylint: disable=redefined-outer-name
     global meta_data_train
     global meta_data_eval
 
-    ap = AudioProcessor(**c.audio)
+    ap = AudioProcessor(**c.audio.to_dict())
     model = SpeakerEncoder(
-        input_dim=c.model["input_dim"],
-        proj_dim=c.model["proj_dim"],
-        lstm_dim=c.model["lstm_dim"],
-        num_lstm_layers=c.model["num_lstm_layers"],
+        input_dim=c.model_params["input_dim"],
+        proj_dim=c.model_params["proj_dim"],
+        lstm_dim=c.model_params["lstm_dim"],
+        num_lstm_layers=c.model_params["num_lstm_layers"],
     )
     optimizer = RAdam(model.parameters(), lr=c.lr)
 
@@ -168,11 +166,6 @@ def main(args):  # pylint: disable=redefined-outer-name
     if args.restore_path:
         checkpoint = torch.load(args.restore_path)
         try:
-            # TODO: fix optimizer init, model.cuda() needs to be called before
-            # optimizer restore
-            # optimizer.load_state_dict(checkpoint['optimizer'])
-            if c.reinit_layers:
-                raise RuntimeError
             model.load_state_dict(checkpoint["model"])
         except KeyError:
             print(" > Partial model initialization.")
@@ -207,47 +200,7 @@ def main(args):  # pylint: disable=redefined-outer-name
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--restore_path", type=str, help="Path to model outputs (checkpoint, tensorboard etc.).", default=0
-    )
-    parser.add_argument(
-        "--config_path",
-        type=str,
-        required=True,
-        help="Path to config file for training.",
-    )
-    parser.add_argument("--debug", type=bool, default=True, help="Do not verify commit integrity to run training.")
-    parser.add_argument("--data_path", type=str, default="", help="Defines the data path. It overwrites config.json.")
-    parser.add_argument("--output_path", type=str, help="path for training outputs.", default="")
-    parser.add_argument("--output_folder", type=str, default="", help="folder name for training outputs.")
-    args = parser.parse_args()
-
-    # setup output paths and read configs
-    c = load_config(args.config_path)
-    check_config_speaker_encoder(c)
-    _ = os.path.dirname(os.path.realpath(__file__))
-    if args.data_path != "":
-        c.data_path = args.data_path
-
-    if args.output_path == "":
-        OUT_PATH = os.path.join(_, c.output_path)
-    else:
-        OUT_PATH = args.output_path
-
-    if args.output_folder == "":
-        OUT_PATH = create_experiment_folder(OUT_PATH, c.run_name, args.debug)
-    else:
-        OUT_PATH = os.path.join(OUT_PATH, args.output_folder)
-
-    new_fields = {}
-    if args.restore_path:
-        new_fields["restore_path"] = args.restore_path
-    new_fields["github_branch"] = get_git_branch()
-    copy_model_files(c, args.config_path, OUT_PATH, new_fields)
-
-    LOG_DIR = OUT_PATH
-    tb_logger = TensorboardLogger(LOG_DIR, model_name="Speaker_Encoder")
+    args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv)
 
     try:
         main(args)

TTS/bin/train_glow_tts.py

@@ -24,7 +24,7 @@ from TTS.tts.utils.speakers import parse_speakers
 from TTS.tts.utils.synthesis import synthesis
 from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols
 from TTS.tts.utils.visual import plot_alignment, plot_spectrogram
-from TTS.utils.arguments import parse_arguments, process_args
+from TTS.utils.arguments import init_training
 from TTS.utils.audio import AudioProcessor
 from TTS.utils.distribute import init_distributed, reduce_tensor
 from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict
@@ -35,45 +35,45 @@ use_cuda, num_gpus = setup_torch_training_env(True, False)
 
 
 def setup_loader(ap, r, is_val=False, verbose=False):
-    if is_val and not c.run_eval:
+    if is_val and not config.run_eval:
         loader = None
     else:
         dataset = MyDataset(
             r,
-            c.text_cleaner,
+            config.text_cleaner,
             compute_linear_spec=False,
             meta_data=meta_data_eval if is_val else meta_data_train,
             ap=ap,
-            tp=c.characters if "characters" in c.keys() else None,
-            add_blank=c["add_blank"] if "add_blank" in c.keys() else False,
-            batch_group_size=0 if is_val else c.batch_group_size * c.batch_size,
-            min_seq_len=c.min_seq_len,
-            max_seq_len=c.max_seq_len,
-            phoneme_cache_path=c.phoneme_cache_path,
-            use_phonemes=c.use_phonemes,
-            phoneme_language=c.phoneme_language,
-            enable_eos_bos=c.enable_eos_bos_chars,
-            use_noise_augment=c["use_noise_augment"] and not is_val,
+            tp=config.characters,
+            add_blank=config["add_blank"],
+            batch_group_size=0 if is_val else config.batch_group_size * config.batch_size,
+            min_seq_len=config.min_seq_len,
+            max_seq_len=config.max_seq_len,
+            phoneme_cache_path=config.phoneme_cache_path,
+            use_phonemes=config.use_phonemes,
+            phoneme_language=config.phoneme_language,
+            enable_eos_bos=config.enable_eos_bos_chars,
+            use_noise_augment=not is_val,
             verbose=verbose,
             speaker_mapping=speaker_mapping
-            if c.use_speaker_embedding and c.use_external_speaker_embedding_file
+            if config.use_speaker_embedding and config.use_external_speaker_embedding_file
             else None,
         )
 
-        if c.use_phonemes and c.compute_input_seq_cache:
+        if config.use_phonemes and config.compute_input_seq_cache:
             # precompute phonemes to have a better estimate of sequence lengths.
-            dataset.compute_input_seq(c.num_loader_workers)
+            dataset.compute_input_seq(config.num_loader_workers)
         dataset.sort_items()
 
         sampler = DistributedSampler(dataset) if num_gpus > 1 else None
         loader = DataLoader(
             dataset,
-            batch_size=c.eval_batch_size if is_val else c.batch_size,
+            batch_size=config.eval_batch_size if is_val else config.batch_size,
             shuffle=False,
             collate_fn=dataset.collate_fn,
             drop_last=False,
             sampler=sampler,
-            num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers,
+            num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers,
             pin_memory=False,
         )
     return loader
@@ -91,8 +91,8 @@ def format_data(data):
     avg_text_length = torch.mean(text_lengths.float())
     avg_spec_length = torch.mean(mel_lengths.float())
 
-    if c.use_speaker_embedding:
-        if c.use_external_speaker_embedding_file:
+    if config.use_speaker_embedding:
+        if config.use_external_speaker_embedding_file:
             # return precomputed embedding vector
             speaker_c = data[8]
         else:
@@ -147,7 +147,7 @@ def data_depended_init(data_loader, model):
 
             # forward pass model
             _ = model.forward(text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=spekaer_embed)
-            if num_iter == c.data_dep_init_iter:
+            if num_iter == config.data_dep_init_steps:
                 break
             num_iter += 1
 
@@ -168,12 +168,12 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
     epoch_time = 0
     keep_avg = KeepAverage()
     if use_cuda:
-        batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus))
+        batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus))
     else:
-        batch_n_iter = int(len(data_loader.dataset) / c.batch_size)
+        batch_n_iter = int(len(data_loader.dataset) / config.batch_size)
     end_time = time.time()
     c_logger.print_train_start()
-    scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None
+    scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None
     for num_iter, data in enumerate(data_loader):
         start_time = time.time()
 
@@ -196,7 +196,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
         optimizer.zero_grad()
 
         # forward pass model
-        with torch.cuda.amp.autocast(enabled=c.mixed_precision):
+        with torch.cuda.amp.autocast(enabled=config.mixed_precision):
             z, logdet, y_mean, y_log_scale, alignments, o_dur_log, o_total_dur = model.forward(
                 text_input, text_lengths, mel_input, mel_lengths, attn_mask, g=speaker_c
             )
@@ -205,19 +205,19 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
             loss_dict = criterion(z, y_mean, y_log_scale, logdet, mel_lengths, o_dur_log, o_total_dur, text_lengths)
 
         # backward pass with loss scaling
-        if c.mixed_precision:
+        if config.mixed_precision:
             scaler.scale(loss_dict["loss"]).backward()
             scaler.unscale_(optimizer)
-            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip)
+            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
             scaler.step(optimizer)
             scaler.update()
         else:
             loss_dict["loss"].backward()
-            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip)
+            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
             optimizer.step()
 
         # setup lr
-        if c.noam_schedule:
+        if config.noam_schedule:
             scheduler.step()
 
         # current_lr
@@ -254,7 +254,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
         keep_avg.update_values(update_train_values)
 
         # print training progress
-        if global_step % c.print_step == 0:
+        if global_step % config.print_step == 0:
             log_dict = {
                 "avg_spec_length": [avg_spec_length, 1],  # value, precision
                 "avg_text_length": [avg_text_length, 1],
@@ -267,13 +267,13 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
         if args.rank == 0:
             # Plot Training Iter Stats
             # reduce TB load
-            if global_step % c.tb_plot_step == 0:
+            if global_step % config.tb_plot_step == 0:
                 iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time}
                 iter_stats.update(loss_dict)
                 tb_logger.tb_train_iter_stats(global_step, iter_stats)
 
-            if global_step % c.save_step == 0:
-                if c.checkpoint:
+            if global_step % config.save_step == 0:
+                if config.checkpoint:
                     # save model
                     save_checkpoint(
                         model,
@@ -314,7 +314,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
 
                 # Sample audio
                 train_audio = ap.inv_melspectrogram(const_spec.T)
-                tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"])
+                tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"])
         end_time = time.time()
 
     # print epoch stats
@@ -325,7 +325,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
         epoch_stats = {"epoch_time": epoch_time}
         epoch_stats.update(keep_avg.avg_values)
         tb_logger.tb_train_epoch_stats(global_step, epoch_stats)
-        if c.tb_model_param_stats:
+        if config.tb_model_param_stats:
             tb_logger.tb_model_weights(model, global_step)
     return keep_avg.avg_values, global_step
 
@@ -380,7 +380,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
                 update_train_values["avg_" + key] = value
             keep_avg.update_values(update_train_values)
 
-            if c.print_eval:
+            if config.print_eval:
                 c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values)
 
         if args.rank == 0:
@@ -406,14 +406,17 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
 
             # Sample audio
             eval_audio = ap.inv_melspectrogram(const_spec.T)
-            tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"])
+            tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"])
 
             # Plot Validation Stats
             tb_logger.tb_eval_stats(global_step, keep_avg.avg_values)
             tb_logger.tb_eval_figures(global_step, eval_figures)
 
-    if args.rank == 0 and epoch >= c.test_delay_epochs:
-        if c.test_sentences_file is None:
+    if args.rank == 0 and epoch >= config.test_delay_epochs:
+        if config.test_sentences_file:
+            with open(config.test_sentences_file, "r") as f:
+                test_sentences = [s.strip() for s in f.readlines()]
+        else:
             test_sentences = [
                 "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
                 "Be a voice, not an echo.",
@@ -421,16 +424,13 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
                 "This cake is great. It's so delicious and moist.",
                 "Prior to November 22, 1963.",
             ]
-        else:
-            with open(c.test_sentences_file, "r") as f:
-                test_sentences = [s.strip() for s in f.readlines()]
 
         # test sentences
         test_audios = {}
         test_figures = {}
         print(" | > Synthesizing test sentences")
-        if c.use_speaker_embedding:
-            if c.use_external_speaker_embedding_file:
+        if config.use_speaker_embedding:
+            if config.use_external_speaker_embedding_file:
                 speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][
                     "embedding"
                 ]
@@ -442,20 +442,20 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             speaker_id = None
             speaker_embedding = None
 
-        style_wav = c.get("style_wav_for_test")
+        style_wav = config.style_wav_for_test
         for idx, test_sentence in enumerate(test_sentences):
             try:
                 wav, alignment, _, postnet_output, _, _ = synthesis(
                     model,
                     test_sentence,
-                    c,
+                    config,
                     use_cuda,
                     ap,
                     speaker_id=speaker_id,
                     speaker_embedding=speaker_embedding,
                     style_wav=style_wav,
                     truncated=False,
-                    enable_eos_bos_chars=c.enable_eos_bos_chars,  # pylint: disable=unused-argument
+                    enable_eos_bos_chars=config.enable_eos_bos_chars,  # pylint: disable=unused-argument
                     use_griffin_lim=True,
                     do_trim_silence=False,
                 )
@@ -470,7 +470,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             except:  # pylint: disable=bare-except
                 print(" !! Error creating Test Sentence -", idx)
                 traceback.print_exc()
-        tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"])
+        tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"])
         tb_logger.tb_test_figures(global_step, test_figures)
     return keep_avg.avg_values
 
@@ -479,33 +479,27 @@ def main(args):  # pylint: disable=redefined-outer-name
     # pylint: disable=global-variable-undefined
     global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping
     # Audio processor
-    ap = AudioProcessor(**c.audio)
-    if "characters" in c.keys():
-        symbols, phonemes = make_symbols(**c.characters)
+    ap = AudioProcessor(**config.audio.to_dict())
+    if config.has("characters") and config.characters:
+        symbols, phonemes = make_symbols(**config.characters.to_dict())
 
     # DISTRUBUTED
     if num_gpus > 1:
-        init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"])
+        init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"])
 
     # set model characters
-    model_characters = phonemes if c.use_phonemes else symbols
+    model_characters = phonemes if config.use_phonemes else symbols
     num_chars = len(model_characters)
 
     # load data instances
-    meta_data_train, meta_data_eval = load_meta_data(c.datasets)
-
-    # set the portion of the data used for training
-    if "train_portion" in c.keys():
-        meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)]
-    if "eval_portion" in c.keys():
-        meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)]
+    meta_data_train, meta_data_eval = load_meta_data(config.datasets)
 
     # parse speakers
-    num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH)
+    num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH)
 
     # setup model
-    model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim)
-    optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9)
+    model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim)
+    optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9)
     criterion = GlowTTSLoss()
 
     if args.restore_path:
@@ -515,18 +509,16 @@ def main(args):  # pylint: disable=redefined-outer-name
             # TODO: fix optimizer init, model.cuda() needs to be called before
             # optimizer restore
             optimizer.load_state_dict(checkpoint["optimizer"])
-            if c.reinit_layers:
-                raise RuntimeError
             model.load_state_dict(checkpoint["model"])
         except:  # pylint: disable=bare-except
             print(" > Partial model initialization.")
             model_dict = model.state_dict()
-            model_dict = set_init_dict(model_dict, checkpoint["model"], c)
+            model_dict = set_init_dict(model_dict, checkpoint["model"], config)
             model.load_state_dict(model_dict)
             del model_dict
 
         for group in optimizer.param_groups:
-            group["initial_lr"] = c.lr
+            group["initial_lr"] = config.lr
         print(f" > Model restored from step {checkpoint['step']:d}", flush=True)
         args.restore_step = checkpoint["step"]
     else:
@@ -540,8 +532,8 @@ def main(args):  # pylint: disable=redefined-outer-name
     if num_gpus > 1:
         model = DDP_th(model, device_ids=[args.rank])
 
-    if c.noam_schedule:
-        scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1)
+    if config.noam_schedule:
+        scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1)
     else:
         scheduler = None
 
@@ -555,8 +547,8 @@ def main(args):  # pylint: disable=redefined-outer-name
         print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...")
         best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"]
         print(f" > Starting with loaded last best loss {best_loss}.")
-    keep_all_best = c.get("keep_all_best", False)
-    keep_after = c.get("keep_after", 10000)  # void if keep_all_best False
+    keep_all_best = config.keep_all_best
+    keep_after = config.keep_after  # void if keep_all_best False
 
     # define dataloaders
     train_loader = setup_loader(ap, 1, is_val=False, verbose=True)
@@ -564,15 +556,15 @@ def main(args):  # pylint: disable=redefined-outer-name
 
     global_step = args.restore_step
     model = data_depended_init(train_loader, model)
-    for epoch in range(0, c.epochs):
-        c_logger.print_epoch_start(epoch, c.epochs)
+    for epoch in range(0, config.epochs):
+        c_logger.print_epoch_start(epoch, config.epochs)
         train_avg_loss_dict, global_step = train(
             train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch
         )
         eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch)
         c_logger.print_epoch_end(epoch, eval_avg_loss_dict)
         target_loss = train_avg_loss_dict["avg_loss"]
-        if c.run_eval:
+        if config.run_eval:
             target_loss = eval_avg_loss_dict["avg_loss"]
         best_loss = save_best_model(
             target_loss,
@@ -581,7 +573,7 @@ def main(args):  # pylint: disable=redefined-outer-name
             optimizer,
             global_step,
             epoch,
-            c.r,
+            config.r,
             OUT_PATH,
             model_characters,
             keep_all_best=keep_all_best,
@@ -590,8 +582,7 @@ def main(args):  # pylint: disable=redefined-outer-name
 
 
 if __name__ == "__main__":
-    args = parse_arguments(sys.argv)
-    c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts")
+    args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv)
 
     try:
         main(args)

TTS/bin/train_speedy_speech.py

@@ -25,7 +25,7 @@ from TTS.tts.utils.speakers import parse_speakers
 from TTS.tts.utils.synthesis import synthesis
 from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols
 from TTS.tts.utils.visual import plot_alignment, plot_spectrogram
-from TTS.utils.arguments import parse_arguments, process_args
+from TTS.utils.arguments import init_training
 from TTS.utils.audio import AudioProcessor
 from TTS.utils.distribute import init_distributed, reduce_tensor
 from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict
@@ -36,45 +36,45 @@ use_cuda, num_gpus = setup_torch_training_env(True, False)
 
 
 def setup_loader(ap, r, is_val=False, verbose=False):
-    if is_val and not c.run_eval:
+    if is_val and not config.run_eval:
         loader = None
     else:
         dataset = MyDataset(
             r,
-            c.text_cleaner,
+            config.text_cleaner,
             compute_linear_spec=False,
             meta_data=meta_data_eval if is_val else meta_data_train,
             ap=ap,
-            tp=c.characters if "characters" in c.keys() else None,
-            add_blank=c["add_blank"] if "add_blank" in c.keys() else False,
-            batch_group_size=0 if is_val else c.batch_group_size * c.batch_size,
-            min_seq_len=c.min_seq_len,
-            max_seq_len=c.max_seq_len,
-            phoneme_cache_path=c.phoneme_cache_path,
-            use_phonemes=c.use_phonemes,
-            phoneme_language=c.phoneme_language,
-            enable_eos_bos=c.enable_eos_bos_chars,
+            tp=config.characters,
+            add_blank=config["add_blank"],
+            batch_group_size=0 if is_val else config.batch_group_size * config.batch_size,
+            min_seq_len=config.min_seq_len,
+            max_seq_len=config.max_seq_len,
+            phoneme_cache_path=config.phoneme_cache_path,
+            use_phonemes=config.use_phonemes,
+            phoneme_language=config.phoneme_language,
+            enable_eos_bos=config.enable_eos_bos_chars,
             use_noise_augment=not is_val,
             verbose=verbose,
             speaker_mapping=speaker_mapping
-            if c.use_speaker_embedding and c.use_external_speaker_embedding_file
+            if config.use_speaker_embedding and config.use_external_speaker_embedding_file
             else None,
         )
 
-        if c.use_phonemes and c.compute_input_seq_cache:
+        if config.use_phonemes and config.compute_input_seq_cache:
             # precompute phonemes to have a better estimate of sequence lengths.
-            dataset.compute_input_seq(c.num_loader_workers)
+            dataset.compute_input_seq(config.num_loader_workers)
         dataset.sort_items()
 
         sampler = DistributedSampler(dataset) if num_gpus > 1 else None
         loader = DataLoader(
             dataset,
-            batch_size=c.eval_batch_size if is_val else c.batch_size,
+            batch_size=config.eval_batch_size if is_val else config.batch_size,
             shuffle=False,
             collate_fn=dataset.collate_fn,
             drop_last=False,
             sampler=sampler,
-            num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers,
+            num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers,
             pin_memory=False,
         )
     return loader
@@ -92,8 +92,8 @@ def format_data(data):
     avg_text_length = torch.mean(text_lengths.float())
     avg_spec_length = torch.mean(mel_lengths.float())
 
-    if c.use_speaker_embedding:
-        if c.use_external_speaker_embedding_file:
+    if config.use_speaker_embedding:
+        if config.use_external_speaker_embedding_file:
             # return precomputed embedding vector
             speaker_c = data[8]
         else:
@@ -150,12 +150,12 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
     epoch_time = 0
     keep_avg = KeepAverage()
     if use_cuda:
-        batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus))
+        batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus))
     else:
-        batch_n_iter = int(len(data_loader.dataset) / c.batch_size)
+        batch_n_iter = int(len(data_loader.dataset) / config.batch_size)
     end_time = time.time()
     c_logger.print_train_start()
-    scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None
+    scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None
     for num_iter, data in enumerate(data_loader):
         start_time = time.time()
 
@@ -179,7 +179,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
         optimizer.zero_grad()
 
         # forward pass model
-        with torch.cuda.amp.autocast(enabled=c.mixed_precision):
+        with torch.cuda.amp.autocast(enabled=config.mixed_precision):
             decoder_output, dur_output, alignments = model.forward(
                 text_input, text_lengths, mel_lengths, dur_target, g=speaker_c
             )
@@ -190,19 +190,19 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
             )
 
         # backward pass with loss scaling
-        if c.mixed_precision:
+        if config.mixed_precision:
             scaler.scale(loss_dict["loss"]).backward()
             scaler.unscale_(optimizer)
-            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip)
+            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
             scaler.step(optimizer)
             scaler.update()
         else:
             loss_dict["loss"].backward()
-            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip)
+            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
             optimizer.step()
 
         # setup lr
-        if c.noam_schedule:
+        if config.noam_schedule:
             scheduler.step()
 
         # current_lr
@@ -240,7 +240,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
         keep_avg.update_values(update_train_values)
 
         # print training progress
-        if global_step % c.print_step == 0:
+        if global_step % config.print_step == 0:
             log_dict = {
                 "avg_spec_length": [avg_spec_length, 1],  # value, precision
                 "avg_text_length": [avg_text_length, 1],
@@ -253,13 +253,13 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
         if args.rank == 0:
             # Plot Training Iter Stats
             # reduce TB load
-            if global_step % c.tb_plot_step == 0:
+            if global_step % config.tb_plot_step == 0:
                 iter_stats = {"lr": current_lr, "grad_norm": grad_norm, "step_time": step_time}
                 iter_stats.update(loss_dict)
                 tb_logger.tb_train_iter_stats(global_step, iter_stats)
 
-            if global_step % c.save_step == 0:
-                if c.checkpoint:
+            if global_step % config.save_step == 0:
+                if config.checkpoint:
                     # save model
                     save_checkpoint(
                         model,
@@ -291,7 +291,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
 
                 # Sample audio
                 train_audio = ap.inv_melspectrogram(pred_spec.T)
-                tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"])
+                tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"])
         end_time = time.time()
 
     # print epoch stats
@@ -302,7 +302,7 @@ def train(data_loader, model, criterion, optimizer, scheduler, ap, global_step,
         epoch_stats = {"epoch_time": epoch_time}
         epoch_stats.update(keep_avg.avg_values)
         tb_logger.tb_train_epoch_stats(global_step, epoch_stats)
-        if c.tb_model_param_stats:
+        if config.tb_model_param_stats:
             tb_logger.tb_model_weights(model, global_step)
     return keep_avg.avg_values, global_step
 
@@ -321,7 +321,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             text_input, text_lengths, mel_targets, mel_lengths, speaker_c, _, _, _, dur_target, _ = format_data(data)
 
             # forward pass model
-            with torch.cuda.amp.autocast(enabled=c.mixed_precision):
+            with torch.cuda.amp.autocast(enabled=config.mixed_precision):
                 decoder_output, dur_output, alignments = model.forward(
                     text_input, text_lengths, mel_lengths, dur_target, g=speaker_c
                 )
@@ -361,7 +361,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
                 update_train_values["avg_" + key] = value
             keep_avg.update_values(update_train_values)
 
-            if c.print_eval:
+            if config.print_eval:
                 c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values)
 
         if args.rank == 0:
@@ -379,14 +379,17 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
 
             # Sample audio
             eval_audio = ap.inv_melspectrogram(pred_spec.T)
-            tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"])
+            tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"])
 
             # Plot Validation Stats
             tb_logger.tb_eval_stats(global_step, keep_avg.avg_values)
             tb_logger.tb_eval_figures(global_step, eval_figures)
 
-    if args.rank == 0 and epoch >= c.test_delay_epochs:
-        if c.test_sentences_file is None:
+    if args.rank == 0 and epoch >= config.test_delay_epochs:
+        if config.test_sentences_file:
+            with open(config.test_sentences_file, "r") as f:
+                test_sentences = [s.strip() for s in f.readlines()]
+        else:
             test_sentences = [
                 "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
                 "Be a voice, not an echo.",
@@ -394,16 +397,13 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
                 "This cake is great. It's so delicious and moist.",
                 "Prior to November 22, 1963.",
             ]
-        else:
-            with open(c.test_sentences_file, "r") as f:
-                test_sentences = [s.strip() for s in f.readlines()]
 
         # test sentences
         test_audios = {}
         test_figures = {}
         print(" | > Synthesizing test sentences")
-        if c.use_speaker_embedding:
-            if c.use_external_speaker_embedding_file:
+        if config.use_speaker_embedding:
+            if config.use_external_speaker_embedding_file:
                 speaker_embedding = speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]][
                     "embedding"
                 ]
@@ -415,20 +415,19 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             speaker_id = None
             speaker_embedding = None
 
-        style_wav = c.get("style_wav_for_test")
         for idx, test_sentence in enumerate(test_sentences):
             try:
                 wav, alignment, _, postnet_output, _, _ = synthesis(
                     model,
                     test_sentence,
-                    c,
+                    config,
                     use_cuda,
                     ap,
                     speaker_id=speaker_id,
                     speaker_embedding=speaker_embedding,
-                    style_wav=style_wav,
+                    style_wav=None,
                     truncated=False,
-                    enable_eos_bos_chars=c.enable_eos_bos_chars,  # pylint: disable=unused-argument
+                    enable_eos_bos_chars=config.enable_eos_bos_chars,  # pylint: disable=unused-argument
                     use_griffin_lim=True,
                     do_trim_silence=False,
                 )
@@ -443,7 +442,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             except:  # pylint: disable=bare-except
                 print(" !! Error creating Test Sentence -", idx)
                 traceback.print_exc()
-        tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"])
+        tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"])
         tb_logger.tb_test_figures(global_step, test_figures)
     return keep_avg.avg_values
 
@@ -453,34 +452,34 @@ def main(args):  # pylint: disable=redefined-outer-name
     # pylint: disable=global-variable-undefined
     global meta_data_train, meta_data_eval, symbols, phonemes, model_characters, speaker_mapping
     # Audio processor
-    ap = AudioProcessor(**c.audio)
-    if "characters" in c.keys():
-        symbols, phonemes = make_symbols(**c.characters)
+    ap = AudioProcessor(**config.audio.to_dict())
+    if config.characters is not None:
+        symbols, phonemes = make_symbols(**config.characters.to_dict())
 
     # DISTRUBUTED
     if num_gpus > 1:
-        init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"])
+        init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"])
 
     # set model characters
-    model_characters = phonemes if c.use_phonemes else symbols
+    model_characters = phonemes if config.use_phonemes else symbols
     num_chars = len(model_characters)
 
     # load data instances
-    meta_data_train, meta_data_eval = load_meta_data(c.datasets, eval_split=True)
+    meta_data_train, meta_data_eval = load_meta_data(config.datasets, eval_split=True)
 
     # set the portion of the data used for training if set in config.json
-    if "train_portion" in c.keys():
-        meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)]
-    if "eval_portion" in c.keys():
-        meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)]
+    if config.has("train_portion"):
+        meta_data_train = meta_data_train[: int(len(meta_data_train) * config.train_portion)]
+    if config.has("eval_portion"):
+        meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * config.eval_portion)]
 
     # parse speakers
-    num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH)
+    num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH)
 
     # setup model
-    model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim=speaker_embedding_dim)
-    optimizer = RAdam(model.parameters(), lr=c.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9)
-    criterion = SpeedySpeechLoss(c)
+    model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim=speaker_embedding_dim)
+    optimizer = RAdam(model.parameters(), lr=config.lr, weight_decay=0, betas=(0.9, 0.98), eps=1e-9)
+    criterion = SpeedySpeechLoss(config)
 
     if args.restore_path:
         print(f" > Restoring from {os.path.basename(args.restore_path)} ...")
@@ -489,18 +488,18 @@ def main(args):  # pylint: disable=redefined-outer-name
             # TODO: fix optimizer init, model.cuda() needs to be called before
             # optimizer restore
             optimizer.load_state_dict(checkpoint["optimizer"])
-            if c.reinit_layers:
+            if config.reinit_layers:
                 raise RuntimeError
             model.load_state_dict(checkpoint["model"])
         except:  # pylint: disable=bare-except
             print(" > Partial model initialization.")
             model_dict = model.state_dict()
-            model_dict = set_init_dict(model_dict, checkpoint["model"], c)
+            model_dict = set_init_dict(model_dict, checkpoint["model"], config)
             model.load_state_dict(model_dict)
             del model_dict
 
         for group in optimizer.param_groups:
-            group["initial_lr"] = c.lr
+            group["initial_lr"] = config.lr
         print(" > Model restored from step %d" % checkpoint["step"], flush=True)
         args.restore_step = checkpoint["step"]
     else:
@@ -514,8 +513,8 @@ def main(args):  # pylint: disable=redefined-outer-name
     if num_gpus > 1:
         model = DDP_th(model, device_ids=[args.rank])
 
-    if c.noam_schedule:
-        scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1)
+    if config.noam_schedule:
+        scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1)
     else:
         scheduler = None
 
@@ -529,23 +528,23 @@ def main(args):  # pylint: disable=redefined-outer-name
         print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...")
         best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"]
         print(f" > Starting with loaded last best loss {best_loss}.")
-    keep_all_best = c.get("keep_all_best", False)
-    keep_after = c.get("keep_after", 10000)  # void if keep_all_best False
+    keep_all_best = config.keep_all_best
+    keep_after = config.keep_after  # void if keep_all_best False
 
     # define dataloaders
     train_loader = setup_loader(ap, 1, is_val=False, verbose=True)
     eval_loader = setup_loader(ap, 1, is_val=True, verbose=True)
 
     global_step = args.restore_step
-    for epoch in range(0, c.epochs):
-        c_logger.print_epoch_start(epoch, c.epochs)
+    for epoch in range(0, config.epochs):
+        c_logger.print_epoch_start(epoch, config.epochs)
         train_avg_loss_dict, global_step = train(
             train_loader, model, criterion, optimizer, scheduler, ap, global_step, epoch
         )
         eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch)
         c_logger.print_epoch_end(epoch, eval_avg_loss_dict)
         target_loss = train_avg_loss_dict["avg_loss"]
-        if c.run_eval:
+        if config.run_eval:
             target_loss = eval_avg_loss_dict["avg_loss"]
         best_loss = save_best_model(
             target_loss,
@@ -554,7 +553,7 @@ def main(args):  # pylint: disable=redefined-outer-name
             optimizer,
             global_step,
             epoch,
-            c.r,
+            config.r,
             OUT_PATH,
             model_characters,
             keep_all_best=keep_all_best,
@@ -563,8 +562,7 @@ def main(args):  # pylint: disable=redefined-outer-name
 
 
 if __name__ == "__main__":
-    args = parse_arguments(sys.argv)
-    c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts")
+    args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv)
 
     try:
         main(args)

TTS/bin/train_tacotron.py

@@ -21,7 +21,7 @@ from TTS.tts.utils.speakers import parse_speakers
 from TTS.tts.utils.synthesis import synthesis
 from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols
 from TTS.tts.utils.visual import plot_alignment, plot_spectrogram
-from TTS.utils.arguments import parse_arguments, process_args
+from TTS.utils.arguments import init_training
 from TTS.utils.audio import AudioProcessor
 from TTS.utils.distribute import DistributedSampler, apply_gradient_allreduce, init_distributed, reduce_tensor
 from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict
@@ -39,45 +39,47 @@ use_cuda, num_gpus = setup_torch_training_env(True, False)
 
 
 def setup_loader(ap, r, is_val=False, verbose=False, dataset=None):
-    if is_val and not c.run_eval:
+    if is_val and not config.run_eval:
         loader = None
     else:
         if dataset is None:
             dataset = MyDataset(
                 r,
-                c.text_cleaner,
-                compute_linear_spec=c.model.lower() == "tacotron",
+                config.text_cleaner,
+                compute_linear_spec=config.model.lower() == "tacotron",
                 meta_data=meta_data_eval if is_val else meta_data_train,
                 ap=ap,
-                tp=c.characters if "characters" in c.keys() else None,
-                add_blank=c["add_blank"] if "add_blank" in c.keys() else False,
-                batch_group_size=0 if is_val else c.batch_group_size * c.batch_size,
-                min_seq_len=c.min_seq_len,
-                max_seq_len=c.max_seq_len,
-                phoneme_cache_path=c.phoneme_cache_path,
-                use_phonemes=c.use_phonemes,
-                phoneme_language=c.phoneme_language,
-                enable_eos_bos=c.enable_eos_bos_chars,
+                tp=config.characters,
+                add_blank=config["add_blank"],
+                batch_group_size=0 if is_val else config.batch_group_size * config.batch_size,
+                min_seq_len=config.min_seq_len,
+                max_seq_len=config.max_seq_len,
+                phoneme_cache_path=config.phoneme_cache_path,
+                use_phonemes=config.use_phonemes,
+                phoneme_language=config.phoneme_language,
+                enable_eos_bos=config.enable_eos_bos_chars,
                 verbose=verbose,
                 speaker_mapping=(
-                    speaker_mapping if (c.use_speaker_embedding and c.use_external_speaker_embedding_file) else None
+                    speaker_mapping
+                    if (config.use_speaker_embedding and config.use_external_speaker_embedding_file)
+                    else None
                 ),
             )
 
-            if c.use_phonemes and c.compute_input_seq_cache:
+            if config.use_phonemes and config.compute_input_seq_cache:
                 # precompute phonemes to have a better estimate of sequence lengths.
-                dataset.compute_input_seq(c.num_loader_workers)
+                dataset.compute_input_seq(config.num_loader_workers)
             dataset.sort_items()
 
         sampler = DistributedSampler(dataset) if num_gpus > 1 else None
         loader = DataLoader(
             dataset,
-            batch_size=c.eval_batch_size if is_val else c.batch_size,
+            batch_size=config.eval_batch_size if is_val else config.batch_size,
             shuffle=False,
             collate_fn=dataset.collate_fn,
             drop_last=False,
             sampler=sampler,
-            num_workers=c.num_val_loader_workers if is_val else c.num_loader_workers,
+            num_workers=config.num_val_loader_workers if is_val else config.num_loader_workers,
             pin_memory=False,
         )
     return loader
@@ -88,15 +90,15 @@ def format_data(data):
     text_input = data[0]
     text_lengths = data[1]
     speaker_names = data[2]
-    linear_input = data[3] if c.model.lower() in ["tacotron"] else None
+    linear_input = data[3] if config.model.lower() in ["tacotron"] else None
     mel_input = data[4]
     mel_lengths = data[5]
     stop_targets = data[6]
     max_text_length = torch.max(text_lengths.float())
     max_spec_length = torch.max(mel_lengths.float())
 
-    if c.use_speaker_embedding:
-        if c.use_external_speaker_embedding_file:
+    if config.use_speaker_embedding:
+        if config.use_external_speaker_embedding_file:
             speaker_embeddings = data[8]
             speaker_ids = None
         else:
@@ -108,7 +110,7 @@ def format_data(data):
         speaker_ids = None
 
     # set stop targets view, we predict a single stop token per iteration.
-    stop_targets = stop_targets.view(text_input.shape[0], stop_targets.size(1) // c.r, -1)
+    stop_targets = stop_targets.view(text_input.shape[0], stop_targets.size(1) // config.r, -1)
     stop_targets = (stop_targets.sum(2) > 0.0).unsqueeze(2).float().squeeze(2)
 
     # dispatch data to GPU
@@ -117,7 +119,7 @@ def format_data(data):
         text_lengths = text_lengths.cuda(non_blocking=True)
         mel_input = mel_input.cuda(non_blocking=True)
         mel_lengths = mel_lengths.cuda(non_blocking=True)
-        linear_input = linear_input.cuda(non_blocking=True) if c.model in ["Tacotron"] else None
+        linear_input = linear_input.cuda(non_blocking=True) if config.model.lower() in ["tacotron"] else None
         stop_targets = stop_targets.cuda(non_blocking=True)
         if speaker_ids is not None:
             speaker_ids = speaker_ids.cuda(non_blocking=True)
@@ -143,9 +145,9 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
     epoch_time = 0
     keep_avg = KeepAverage()
     if use_cuda:
-        batch_n_iter = int(len(data_loader.dataset) / (c.batch_size * num_gpus))
+        batch_n_iter = int(len(data_loader.dataset) / (config.batch_size * num_gpus))
     else:
-        batch_n_iter = int(len(data_loader.dataset) / c.batch_size)
+        batch_n_iter = int(len(data_loader.dataset) / config.batch_size)
     end_time = time.time()
     c_logger.print_train_start()
     for num_iter, data in enumerate(data_loader):
@@ -169,16 +171,16 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
         global_step += 1
 
         # setup lr
-        if c.noam_schedule:
+        if config.noam_schedule:
             scheduler.step()
 
         optimizer.zero_grad()
         if optimizer_st:
             optimizer_st.zero_grad()
 
-        with torch.cuda.amp.autocast(enabled=c.mixed_precision):
+        with torch.cuda.amp.autocast(enabled=config.mixed_precision):
             # forward pass model
-            if c.bidirectional_decoder or c.double_decoder_consistency:
+            if config.bidirectional_decoder or config.double_decoder_consistency:
                 (
                     decoder_output,
                     postnet_output,
@@ -235,17 +237,17 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
             raise RuntimeError(f"Detected NaN loss at step {global_step}.")
 
         # optimizer step
-        if c.mixed_precision:
+        if config.mixed_precision:
             # model optimizer step in mixed precision mode
             scaler.scale(loss_dict["loss"]).backward()
             scaler.unscale_(optimizer)
             optimizer, current_lr = adam_weight_decay(optimizer)
-            grad_norm, _ = check_update(model, c.grad_clip, ignore_stopnet=True)
+            grad_norm, _ = check_update(model, config.grad_clip, ignore_stopnet=True)
             scaler.step(optimizer)
             scaler.update()
 
             # stopnet optimizer step
-            if c.separate_stopnet:
+            if config.separate_stopnet:
                 scaler_st.scale(loss_dict["stopnet_loss"]).backward()
                 scaler.unscale_(optimizer_st)
                 optimizer_st, _ = adam_weight_decay(optimizer_st)
@@ -258,11 +260,11 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
             # main model optimizer step
             loss_dict["loss"].backward()
             optimizer, current_lr = adam_weight_decay(optimizer)
-            grad_norm, _ = check_update(model, c.grad_clip, ignore_stopnet=True)
+            grad_norm, _ = check_update(model, config.grad_clip, ignore_stopnet=True)
             optimizer.step()
 
             # stopnet optimizer step
-            if c.separate_stopnet:
+            if config.separate_stopnet:
                 loss_dict["stopnet_loss"].backward()
                 optimizer_st, _ = adam_weight_decay(optimizer_st)
                 grad_norm_st, _ = check_update(model.decoder.stopnet, 1.0)
@@ -283,7 +285,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
             loss_dict["decoder_loss"] = reduce_tensor(loss_dict["decoder_loss"].data, num_gpus)
             loss_dict["loss"] = reduce_tensor(loss_dict["loss"].data, num_gpus)
             loss_dict["stopnet_loss"] = (
-                reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) if c.stopnet else loss_dict["stopnet_loss"]
+                reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus) if config.stopnet else loss_dict["stopnet_loss"]
             )
 
         # detach loss values
@@ -304,7 +306,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
         keep_avg.update_values(update_train_values)
 
         # print training progress
-        if global_step % c.print_step == 0:
+        if global_step % config.print_step == 0:
             log_dict = {
                 "max_spec_length": [max_spec_length, 1],  # value, precision
                 "max_text_length": [max_text_length, 1],
@@ -317,7 +319,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
         if args.rank == 0:
             # Plot Training Iter Stats
             # reduce TB load
-            if global_step % c.tb_plot_step == 0:
+            if global_step % config.tb_plot_step == 0:
                 iter_stats = {
                     "lr": current_lr,
                     "grad_norm": grad_norm,
@@ -327,8 +329,8 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
                 iter_stats.update(loss_dict)
                 tb_logger.tb_train_iter_stats(global_step, iter_stats)
 
-            if global_step % c.save_step == 0:
-                if c.checkpoint:
+            if global_step % config.save_step == 0:
+                if config.checkpoint:
                     # save model
                     save_checkpoint(
                         model,
@@ -340,14 +342,14 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
                         optimizer_st=optimizer_st,
                         model_loss=loss_dict["postnet_loss"],
                         characters=model_characters,
-                        scaler=scaler.state_dict() if c.mixed_precision else None,
+                        scaler=scaler.state_dict() if config.mixed_precision else None,
                     )
 
                 # Diagnostic visualizations
                 const_spec = postnet_output[0].data.cpu().numpy()
                 gt_spec = (
                     linear_input[0].data.cpu().numpy()
-                    if c.model in ["Tacotron", "TacotronGST"]
+                    if config.model in ["Tacotron", "TacotronGST"]
                     else mel_input[0].data.cpu().numpy()
                 )
                 align_img = alignments[0].data.cpu().numpy()
@@ -358,7 +360,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
                     "alignment": plot_alignment(align_img, output_fig=False),
                 }
 
-                if c.bidirectional_decoder or c.double_decoder_consistency:
+                if config.bidirectional_decoder or config.double_decoder_consistency:
                     figures["alignment_backward"] = plot_alignment(
                         alignments_backward[0].data.cpu().numpy(), output_fig=False
                     )
@@ -366,11 +368,11 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
                 tb_logger.tb_train_figures(global_step, figures)
 
                 # Sample audio
-                if c.model in ["Tacotron", "TacotronGST"]:
+                if config.model in ["Tacotron", "TacotronGST"]:
                     train_audio = ap.inv_spectrogram(const_spec.T)
                 else:
                     train_audio = ap.inv_melspectrogram(const_spec.T)
-                tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, c.audio["sample_rate"])
+                tb_logger.tb_train_audios(global_step, {"TrainAudio": train_audio}, config.audio["sample_rate"])
         end_time = time.time()
 
     # print epoch stats
@@ -381,7 +383,7 @@ def train(data_loader, model, criterion, optimizer, optimizer_st, scheduler, ap,
         epoch_stats = {"epoch_time": epoch_time}
         epoch_stats.update(keep_avg.avg_values)
         tb_logger.tb_train_epoch_stats(global_step, epoch_stats)
-        if c.tb_model_param_stats:
+        if config.tb_model_param_stats:
             tb_logger.tb_model_weights(model, global_step)
     return keep_avg.avg_values, global_step
 
@@ -412,7 +414,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             assert mel_input.shape[1] % model.decoder.r == 0
 
             # forward pass model
-            if c.bidirectional_decoder or c.double_decoder_consistency:
+            if config.bidirectional_decoder or config.double_decoder_consistency:
                 (
                     decoder_output,
                     postnet_output,
@@ -466,7 +468,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             if num_gpus > 1:
                 loss_dict["postnet_loss"] = reduce_tensor(loss_dict["postnet_loss"].data, num_gpus)
                 loss_dict["decoder_loss"] = reduce_tensor(loss_dict["decoder_loss"].data, num_gpus)
-                if c.stopnet:
+                if config.stopnet:
                     loss_dict["stopnet_loss"] = reduce_tensor(loss_dict["stopnet_loss"].data, num_gpus)
 
             # detach loss values
@@ -484,7 +486,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
                 update_train_values["avg_" + key] = value
             keep_avg.update_values(update_train_values)
 
-            if c.print_eval:
+            if config.print_eval:
                 c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values)
 
         if args.rank == 0:
@@ -493,7 +495,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             const_spec = postnet_output[idx].data.cpu().numpy()
             gt_spec = (
                 linear_input[idx].data.cpu().numpy()
-                if c.model in ["Tacotron", "TacotronGST"]
+                if config.model in ["Tacotron", "TacotronGST"]
                 else mel_input[idx].data.cpu().numpy()
             )
             align_img = alignments[idx].data.cpu().numpy()
@@ -505,22 +507,25 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             }
 
             # Sample audio
-            if c.model in ["Tacotron", "TacotronGST"]:
+            if config.model.lower() in ["tacotron"]:
                 eval_audio = ap.inv_spectrogram(const_spec.T)
             else:
                 eval_audio = ap.inv_melspectrogram(const_spec.T)
-            tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, c.audio["sample_rate"])
+            tb_logger.tb_eval_audios(global_step, {"ValAudio": eval_audio}, config.audio["sample_rate"])
 
             # Plot Validation Stats
 
-            if c.bidirectional_decoder or c.double_decoder_consistency:
+            if config.bidirectional_decoder or config.double_decoder_consistency:
                 align_b_img = alignments_backward[idx].data.cpu().numpy()
                 eval_figures["alignment2"] = plot_alignment(align_b_img, output_fig=False)
             tb_logger.tb_eval_stats(global_step, keep_avg.avg_values)
             tb_logger.tb_eval_figures(global_step, eval_figures)
 
-    if args.rank == 0 and epoch > c.test_delay_epochs:
-        if c.test_sentences_file is None:
+    if args.rank == 0 and epoch > config.test_delay_epochs:
+        if config.test_sentences_file:
+            with open(config.test_sentences_file, "r") as f:
+                test_sentences = [s.strip() for s in f.readlines()]
+        else:
             test_sentences = [
                 "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
                 "Be a voice, not an echo.",
@@ -528,41 +533,37 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
                 "This cake is great. It's so delicious and moist.",
                 "Prior to November 22, 1963.",
             ]
-        else:
-            with open(c.test_sentences_file, "r") as f:
-                test_sentences = [s.strip() for s in f.readlines()]
 
         # test sentences
         test_audios = {}
         test_figures = {}
         print(" | > Synthesizing test sentences")
-        speaker_id = 0 if c.use_speaker_embedding else None
+        speaker_id = 0 if config.use_speaker_embedding else None
         speaker_embedding = (
             speaker_mapping[list(speaker_mapping.keys())[randrange(len(speaker_mapping) - 1)]]["embedding"]
-            if c.use_external_speaker_embedding_file and c.use_speaker_embedding
+            if config.use_external_speaker_embedding_file and config.use_speaker_embedding
             else None
         )
-        style_wav = c.get("gst_style_input")
-        if style_wav is None and c.use_gst:
+        style_wav = config.gst_style_input
+        if style_wav is None and config.gst is not None:
             # inicialize GST with zero dict.
             style_wav = {}
             print("WARNING: You don't provided a gst style wav, for this reason we use a zero tensor!")
-            for i in range(c.gst["gst_style_tokens"]):
+            for i in range(config.gst["gst_num_style_tokens"]):
                 style_wav[str(i)] = 0
-        style_wav = c.get("gst_style_input")
         for idx, test_sentence in enumerate(test_sentences):
             try:
                 wav, alignment, decoder_output, postnet_output, stop_tokens, _ = synthesis(
                     model,
                     test_sentence,
-                    c,
+                    config,
                     use_cuda,
                     ap,
                     speaker_id=speaker_id,
                     speaker_embedding=speaker_embedding,
                     style_wav=style_wav,
                     truncated=False,
-                    enable_eos_bos_chars=c.enable_eos_bos_chars,  # pylint: disable=unused-argument
+                    enable_eos_bos_chars=config.enable_eos_bos_chars,  # pylint: disable=unused-argument
                     use_griffin_lim=True,
                     do_trim_silence=False,
                 )
@@ -577,7 +578,7 @@ def evaluate(data_loader, model, criterion, ap, global_step, epoch):
             except:  # pylint: disable=bare-except
                 print(" !! Error creating Test Sentence -", idx)
                 traceback.print_exc()
-        tb_logger.tb_test_audios(global_step, test_audios, c.audio["sample_rate"])
+        tb_logger.tb_test_audios(global_step, test_audios, config.audio["sample_rate"])
         tb_logger.tb_test_figures(global_step, test_figures)
     return keep_avg.avg_values
 
@@ -586,45 +587,45 @@ def main(args):  # pylint: disable=redefined-outer-name
     # pylint: disable=global-variable-undefined
     global meta_data_train, meta_data_eval, speaker_mapping, symbols, phonemes, model_characters
     # Audio processor
-    ap = AudioProcessor(**c.audio)
+    ap = AudioProcessor(**config.audio.to_dict())
 
     # setup custom characters if set in config file.
-    if "characters" in c.keys():
-        symbols, phonemes = make_symbols(**c.characters)
+    if config.characters is not None:
+        symbols, phonemes = make_symbols(**config.characters.to_dict())
 
     # DISTRUBUTED
     if num_gpus > 1:
-        init_distributed(args.rank, num_gpus, args.group_id, c.distributed["backend"], c.distributed["url"])
-    num_chars = len(phonemes) if c.use_phonemes else len(symbols)
-    model_characters = phonemes if c.use_phonemes else symbols
+        init_distributed(args.rank, num_gpus, args.group_id, config.distributed["backend"], config.distributed["url"])
+    num_chars = len(phonemes) if config.use_phonemes else len(symbols)
+    model_characters = phonemes if config.use_phonemes else symbols
 
     # load data instances
-    meta_data_train, meta_data_eval = load_meta_data(c.datasets)
+    meta_data_train, meta_data_eval = load_meta_data(config.datasets)
 
     # set the portion of the data used for training
-    if "train_portion" in c.keys():
-        meta_data_train = meta_data_train[: int(len(meta_data_train) * c.train_portion)]
-    if "eval_portion" in c.keys():
-        meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * c.eval_portion)]
+    if config.has("train_portion"):
+        meta_data_train = meta_data_train[: int(len(meta_data_train) * config.train_portion)]
+    if config.has("eval_portion"):
+        meta_data_eval = meta_data_eval[: int(len(meta_data_eval) * config.eval_portion)]
 
     # parse speakers
-    num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(c, args, meta_data_train, OUT_PATH)
+    num_speakers, speaker_embedding_dim, speaker_mapping = parse_speakers(config, args, meta_data_train, OUT_PATH)
 
-    model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim)
+    model = setup_model(num_chars, num_speakers, config, speaker_embedding_dim)
 
     # scalers for mixed precision training
-    scaler = torch.cuda.amp.GradScaler() if c.mixed_precision else None
-    scaler_st = torch.cuda.amp.GradScaler() if c.mixed_precision and c.separate_stopnet else None
+    scaler = torch.cuda.amp.GradScaler() if config.mixed_precision else None
+    scaler_st = torch.cuda.amp.GradScaler() if config.mixed_precision and config.separate_stopnet else None
 
-    params = set_weight_decay(model, c.wd)
-    optimizer = RAdam(params, lr=c.lr, weight_decay=0)
-    if c.stopnet and c.separate_stopnet:
-        optimizer_st = RAdam(model.decoder.stopnet.parameters(), lr=c.lr, weight_decay=0)
+    params = set_weight_decay(model, config.wd)
+    optimizer = RAdam(params, lr=config.lr, weight_decay=0)
+    if config.stopnet and config.separate_stopnet:
+        optimizer_st = RAdam(model.decoder.stopnet.parameters(), lr=config.lr, weight_decay=0)
     else:
         optimizer_st = None
 
     # setup criterion
-    criterion = TacotronLoss(c, stopnet_pos_weight=c.stopnet_pos_weight, ga_sigma=0.4)
+    criterion = TacotronLoss(config, stopnet_pos_weight=config.stopnet_pos_weight, ga_sigma=0.4)
     if args.restore_path:
         print(f" > Restoring from {os.path.basename(args.restore_path)}...")
         checkpoint = torch.load(args.restore_path, map_location="cpu")
@@ -634,22 +635,18 @@ def main(args):  # pylint: disable=redefined-outer-name
             # optimizer restore
             print(" > Restoring Optimizer...")
             optimizer.load_state_dict(checkpoint["optimizer"])
-            if "scaler" in checkpoint and c.mixed_precision:
+            if "scaler" in checkpoint and config.mixed_precision:
                 print(" > Restoring AMP Scaler...")
                 scaler.load_state_dict(checkpoint["scaler"])
-            if c.reinit_layers:
-                raise RuntimeError
         except (KeyError, RuntimeError):
             print(" > Partial model initialization...")
             model_dict = model.state_dict()
-            model_dict = set_init_dict(model_dict, checkpoint["model"], c)
-            # torch.save(model_dict, os.path.join(OUT_PATH, 'state_dict.pt'))
-            # print("State Dict saved for debug in: ", os.path.join(OUT_PATH, 'state_dict.pt'))
+            model_dict = set_init_dict(model_dict, checkpoint["model"], config)
             model.load_state_dict(model_dict)
             del model_dict
 
         for group in optimizer.param_groups:
-            group["lr"] = c.lr
+            group["lr"] = config.lr
         print(" > Model restored from step %d" % checkpoint["step"], flush=True)
         args.restore_step = checkpoint["step"]
     else:
@@ -663,8 +660,8 @@ def main(args):  # pylint: disable=redefined-outer-name
     if num_gpus > 1:
         model = apply_gradient_allreduce(model)
 
-    if c.noam_schedule:
-        scheduler = NoamLR(optimizer, warmup_steps=c.warmup_steps, last_epoch=args.restore_step - 1)
+    if config.noam_schedule:
+        scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps, last_epoch=args.restore_step - 1)
     else:
         scheduler = None
 
@@ -678,22 +675,22 @@ def main(args):  # pylint: disable=redefined-outer-name
         print(" > Restoring best loss from " f"{os.path.basename(args.best_path)} ...")
         best_loss = torch.load(args.best_path, map_location="cpu")["model_loss"]
         print(f" > Starting with loaded last best loss {best_loss}.")
-    keep_all_best = c.get("keep_all_best", False)
-    keep_after = c.get("keep_after", 10000)  # void if keep_all_best False
+    keep_all_best = config.keep_all_best
+    keep_after = config.keep_after  # void if keep_all_best False
 
     # define data loaders
     train_loader = setup_loader(ap, model.decoder.r, is_val=False, verbose=True)
     eval_loader = setup_loader(ap, model.decoder.r, is_val=True)
 
     global_step = args.restore_step
-    for epoch in range(0, c.epochs):
-        c_logger.print_epoch_start(epoch, c.epochs)
+    for epoch in range(0, config.epochs):
+        c_logger.print_epoch_start(epoch, config.epochs)
         # set gradual training
-        if c.gradual_training is not None:
-            r, c.batch_size = gradual_training_scheduler(global_step, c)
-            c.r = r
+        if config.gradual_training is not None:
+            r, config.batch_size = gradual_training_scheduler(global_step, config)
+            config.r = r
             model.decoder.set_r(r)
-            if c.bidirectional_decoder:
+            if config.bidirectional_decoder:
                 model.decoder_backward.set_r(r)
             train_loader.dataset.outputs_per_step = r
             eval_loader.dataset.outputs_per_step = r
@@ -718,7 +715,7 @@ def main(args):  # pylint: disable=redefined-outer-name
         eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, global_step, epoch)
         c_logger.print_epoch_end(epoch, eval_avg_loss_dict)
         target_loss = train_avg_loss_dict["avg_postnet_loss"]
-        if c.run_eval:
+        if config.run_eval:
             target_loss = eval_avg_loss_dict["avg_postnet_loss"]
         best_loss = save_best_model(
             target_loss,
@@ -727,19 +724,17 @@ def main(args):  # pylint: disable=redefined-outer-name
             optimizer,
             global_step,
             epoch,
-            c.r,
+            config.r,
             OUT_PATH,
             model_characters,
             keep_all_best=keep_all_best,
             keep_after=keep_after,
-            scaler=scaler.state_dict() if c.mixed_precision else None,
+            scaler=scaler.state_dict() if config.mixed_precision else None,
         )
 
 
 if __name__ == "__main__":
-    args = parse_arguments(sys.argv)
-    c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="tts")
-
+    args, config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv)
     try:
         main(args)
     except KeyboardInterrupt:

TTS/bin/train_vocoder_gan.py

@@ -2,6 +2,7 @@
 # TODO: mixed precision training
 """Trains GAN based vocoder model."""
 
+import itertools
 import os
 import sys
 import time
@@ -15,7 +16,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP_th
 from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
 
-from TTS.utils.arguments import parse_arguments, process_args
+from TTS.utils.arguments import init_training
 from TTS.utils.audio import AudioProcessor
 from TTS.utils.distribute import init_distributed
 from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict
@@ -162,7 +163,6 @@ def train(
             y_hat_sub=y_hat_sub,
             y_sub=y_G_sub,
         )
-
         loss_G = loss_G_dict["G_loss"]
 
         # optimizer generator
@@ -468,7 +468,7 @@ def main(args):  # pylint: disable=redefined-outer-name
         eval_data, train_data = load_wav_data(c.data_path, c.eval_split_size)
 
     # setup audio processor
-    ap = AudioProcessor(**c.audio)
+    ap = AudioProcessor(**c.audio.to_dict())
 
     # DISTRUBUTED
     if num_gpus > 1:
@@ -495,7 +495,15 @@ def main(args):  # pylint: disable=redefined-outer-name
     optimizer_gen = getattr(torch.optim, c.optimizer)
     optimizer_gen = optimizer_gen(model_gen.parameters(), lr=c.lr_gen, **c.optimizer_params)
     optimizer_disc = getattr(torch.optim, c.optimizer)
-    optimizer_disc = optimizer_disc(model_disc.parameters(), lr=c.lr_disc, **c.optimizer_params)
+
+    if c.discriminator_model == "hifigan_discriminator":
+        optimizer_disc = optimizer_disc(
+            itertools.chain(model_disc.msd.parameters(), model_disc.mpd.parameters()),
+            lr=c.lr_disc,
+            **c.optimizer_params,
+        )
+    else:
+        optimizer_disc = optimizer_disc(model_disc.parameters(), lr=c.lr_disc, **c.optimizer_params)
 
     # schedulers
     scheduler_gen = None
@@ -615,13 +623,15 @@ def main(args):  # pylint: disable=redefined-outer-name
 
 
 if __name__ == "__main__":
-    args = parse_arguments(sys.argv)
-    c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder")
-
+    args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv)
     try:
         main(args)
     except KeyboardInterrupt:
         remove_experiment_folder(OUT_PATH)
+        try:
+            sys.exit(0)
+        except SystemExit:
+            os._exit(0)  # pylint: disable=protected-access
     except Exception:  # pylint: disable=broad-except
         remove_experiment_folder(OUT_PATH)
         traceback.print_exc()

TTS/bin/train_vocoder_wavegrad.py

@@ -15,7 +15,7 @@ from torch.optim import Adam
 from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
 
-from TTS.utils.arguments import parse_arguments, process_args
+from TTS.utils.arguments import init_training
 from TTS.utils.audio import AudioProcessor
 from TTS.utils.distribute import init_distributed
 from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict
@@ -131,12 +131,12 @@ def train(model, criterion, optimizer, scheduler, scaler, ap, global_step, epoch
         if c.mixed_precision:
             scaler.scale(loss).backward()
             scaler.unscale_(optimizer)
-            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.clip_grad)
+            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.grad_clip)
             scaler.step(optimizer)
             scaler.update()
         else:
             loss.backward()
-            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), c.clip_grad)
+            grad_norm = torch.nn.utils.grad_clip_norm_(model.parameters(), c.clip_grad)
             optimizer.step()
 
         # schedule update
@@ -311,7 +311,7 @@ def main(args):  # pylint: disable=redefined-outer-name
         eval_data, train_data = load_wav_data(c.data_path, c.eval_split_size)
 
     # setup audio processor
-    ap = AudioProcessor(**c.audio)
+    ap = AudioProcessor(**c.audio.to_dict())
 
     # DISTRUBUTED
     if num_gpus > 1:
@@ -416,9 +416,7 @@ def main(args):  # pylint: disable=redefined-outer-name
 
 
 if __name__ == "__main__":
-    args = parse_arguments(sys.argv)
-    c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder")
-
+    args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv)
     try:
         main(args)
     except KeyboardInterrupt:

TTS/bin/train_vocoder_wavernn.py

@@ -11,7 +11,7 @@ import torch
 from torch.utils.data import DataLoader
 
 from TTS.tts.utils.visual import plot_spectrogram
-from TTS.utils.arguments import parse_arguments, process_args
+from TTS.utils.arguments import init_training
 from TTS.utils.audio import AudioProcessor
 from TTS.utils.generic_utils import KeepAverage, count_parameters, remove_experiment_folder, set_init_dict
 from TTS.utils.radam import RAdam
@@ -307,29 +307,7 @@ def main(args):  # pylint: disable=redefined-outer-name
     global train_data, eval_data
 
     # setup audio processor
-    ap = AudioProcessor(**c.audio)
-
-    # print(f" > Loading wavs from: {c.data_path}")
-    # if c.feature_path is not None:
-    #     print(f" > Loading features from: {c.feature_path}")
-    #     eval_data, train_data = load_wav_feat_data(
-    #         c.data_path, c.feature_path, c.eval_split_size
-    #     )
-    # else:
-    #     mel_feat_path = os.path.join(OUT_PATH, "mel")
-    #     feat_data = find_feat_files(mel_feat_path)
-    #     if feat_data:
-    #         print(f" > Loading features from: {mel_feat_path}")
-    #         eval_data, train_data = load_wav_feat_data(
-    #             c.data_path, mel_feat_path, c.eval_split_size
-    #         )
-    #     else:
-    #         print(" > No feature data found. Preprocessing...")
-    #         # preprocessing feature data from given wav files
-    #         preprocess_wav_files(OUT_PATH, CONFIG, ap)
-    #         eval_data, train_data = load_wav_feat_data(
-    #             c.data_path, mel_feat_path, c.eval_split_size
-    #         )
+    ap = AudioProcessor(**c.audio.to_dict())
 
     print(f" > Loading wavs from: {c.data_path}")
     if c.feature_path is not None:
@@ -438,9 +416,7 @@ def main(args):  # pylint: disable=redefined-outer-name
 
 
 if __name__ == "__main__":
-    args = parse_arguments(sys.argv)
-    c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args, model_class="vocoder")
-
+    args, c, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = init_training(sys.argv)
     try:
         main(args)
     except KeyboardInterrupt:

TTS/config/__init__.py

@@ -0,0 +1,75 @@
+import json
+import os
+import re
+
+import yaml
+
+from TTS.config.shared_configs import *
+from TTS.utils.generic_utils import find_module
+
+
+def read_json_with_comments(json_path):
+    """for backward compat."""
+    # fallback to json
+    with open(json_path, "r", encoding="utf-8") as f:
+        input_str = f.read()
+    # handle comments
+    input_str = re.sub(r"\\\n", "", input_str)
+    input_str = re.sub(r"//.*\n", "\n", input_str)
+    data = json.loads(input_str)
+    return data
+
+
+def _search_configs(model_name):
+    config_class = None
+    paths = ["TTS.tts.configs", "TTS.vocoder.configs", "TTS.speaker_encoder"]
+    for path in paths:
+        try:
+            config_class = find_module(path, model_name + "_config")
+        except ModuleNotFoundError:
+            pass
+    if config_class is None:
+        raise ModuleNotFoundError(f" [!] Config for {model_name} cannot be found.")
+    return config_class
+
+
+def _process_model_name(config_dict):
+    model_name = config_dict["model"] if "model" in config_dict else config_dict["generator_model"]
+    model_name = model_name.replace("_generator", "").replace("_discriminator", "")
+    return model_name
+
+
+def load_config(config_path: str) -> None:
+    """Import `json` or `yaml` files as TTS configs. First, load the input file as a `dict` and check the model name
+    to find the corresponding Config class. Then initialize the Config.
+
+    Args:
+        config_path (str): path to the config file.
+
+    Raises:
+        TypeError: given config file has an unknown type.
+
+    Returns:
+        Coqpit: TTS config object.
+    """
+    config_dict = {}
+    ext = os.path.splitext(config_path)[1]
+    if ext in (".yml", ".yaml"):
+        with open(config_path, "r", encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+    elif ext == ".json":
+        try:
+            with open(config_path, "r", encoding="utf-8") as f:
+                input_str = f.read()
+                data = json.loads(input_str)
+        except json.decoder.JSONDecodeError:
+            # backwards compat.
+            data = read_json_with_comments(config_path)
+    else:
+        raise TypeError(f" [!] Unknown config file type {ext}")
+    config_dict.update(data)
+    model_name = _process_model_name(config_dict)
+    config_class = _search_configs(model_name.lower())
+    config = config_class()
+    config.from_dict(config_dict)
+    return config

TTS/config/shared_configs.py

@@ -0,0 +1,253 @@
+from dataclasses import asdict, dataclass
+from typing import List
+
+from coqpit import MISSING, Coqpit, check_argument
+
+
+@dataclass
+class BaseAudioConfig(Coqpit):
+    """Base config to definge audio processing parameters. It is used to initialize
+    ```TTS.utils.audio.AudioProcessor.```
+
+    Args:
+        fft_size (int):
+            Number of STFT frequency levels aka.size of the linear spectogram frame. Defaults to 1024.
+        win_length (int):
+            Each frame of audio is windowed by window of length ```win_length``` and then padded with zeros to match
+            ```fft_size```. Defaults to 1024.
+        hop_length (int):
+            Number of audio samples between adjacent STFT columns. Defaults to 1024.
+        frame_shift_ms (int):
+            Set ```hop_length``` based on milliseconds and sampling rate.
+        frame_length_ms (int):
+            Set ```win_length``` based on milliseconds and sampling rate.
+        stft_pad_mode (str):
+            Padding method used in STFT. 'reflect' or 'center'. Defaults to 'reflect'.
+        sample_rate (int):
+            Audio sampling rate. Defaults to 22050.
+        resample (bool):
+            Enable / Disable resampling audio to ```sample_rate```. Defaults to ```False```.
+        preemphasis (float):
+            Preemphasis coefficient. Defaults to 0.0.
+        ref_level_db (int): 20
+            Reference Db level to rebase the audio signal and ignore the level below. 20Db is assumed the sound of air.
+            Defaults to 20.
+        do_sound_norm (bool):
+            Enable / Disable sound normalization to reconcile the volume differences among samples. Defaults to False.
+        do_trim_silence (bool):
+            Enable / Disable trimming silences at the beginning and the end of the audio clip. Defaults to ```True```.
+        trim_db (int):
+            Silence threshold used for silence trimming. Defaults to 45.
+        power (float):
+            Exponent used for expanding spectrogra levels before running Griffin Lim. It helps to reduce the
+            artifacts in the synthesized voice. Defaults to 1.5.
+        griffin_lim_iters (int):
+            Number of Griffing Lim iterations. Defaults to 60.
+        num_mels (int):
+            Number of mel-basis frames that defines the frame lengths of each mel-spectrogram frame. Defaults to 80.
+        mel_fmin (float): Min frequency level used for the mel-basis filters. ~50 for male and ~95 for female voices.
+            It needs to be adjusted for a dataset. Defaults to 0.
+        mel_fmax (float):
+            Max frequency level used for the mel-basis filters. It needs to be adjusted for a dataset.
+        spec_gain (int):
+            Gain applied when converting amplitude to DB. Defaults to 20.
+        signal_norm (bool):
+            enable/disable signal normalization. Defaults to True.
+        min_level_db (int):
+            minimum db threshold for the computed melspectrograms. Defaults to -100.
+        symmetric_norm (bool):
+            enable/disable symmetric normalization. If set True normalization is performed in the range [-k, k] else
+            [0, k], Defaults to True.
+        max_norm (float):
+            ```k``` defining the normalization range. Defaults to 4.0.
+        clip_norm (bool):
+            enable/disable clipping the our of range values in the normalized audio signal. Defaults to True.
+        stats_path (str):
+            Path to the computed stats file. Defaults to None.
+    """
+
+    # stft parameters
+    fft_size: int = 1024
+    win_length: int = 1024
+    hop_length: int = 256
+    frame_shift_ms: int = None
+    frame_length_ms: int = None
+    stft_pad_mode: str = "reflect"
+    # audio processing parameters
+    sample_rate: int = 22050
+    resample: bool = False
+    preemphasis: float = 0.0
+    ref_level_db: int = 20
+    do_sound_norm: bool = False
+    log_func = "np.log10"
+    # silence trimming
+    do_trim_silence: bool = True
+    trim_db: int = 45
+    # griffin-lim params
+    power: float = 1.5
+    griffin_lim_iters: int = 60
+    # mel-spec params
+    num_mels: int = 80
+    mel_fmin: float = 0.0
+    mel_fmax: float = None
+    spec_gain: int = 20
+    # normalization params
+    signal_norm: bool = True
+    min_level_db: int = -100
+    symmetric_norm: bool = True
+    max_norm: float = 4.0
+    clip_norm: bool = True
+    stats_path: str = None
+
+    def check_values(
+        self,
+    ):
+        """Check config fields"""
+        c = asdict(self)
+        check_argument("num_mels", c, restricted=True, min_val=10, max_val=2056)
+        check_argument("fft_size", c, restricted=True, min_val=128, max_val=4058)
+        check_argument("sample_rate", c, restricted=True, min_val=512, max_val=100000)
+        check_argument(
+            "frame_length_ms",
+            c,
+            restricted=True,
+            min_val=10,
+            max_val=1000,
+            alternative="win_length",
+        )
+        check_argument("frame_shift_ms", c, restricted=True, min_val=1, max_val=1000, alternative="hop_length")
+        check_argument("preemphasis", c, restricted=True, min_val=0, max_val=1)
+        check_argument("min_level_db", c, restricted=True, min_val=-1000, max_val=10)
+        check_argument("ref_level_db", c, restricted=True, min_val=0, max_val=1000)
+        check_argument("power", c, restricted=True, min_val=1, max_val=5)
+        check_argument("griffin_lim_iters", c, restricted=True, min_val=10, max_val=1000)
+
+        # normalization parameters
+        check_argument("signal_norm", c, restricted=True)
+        check_argument("symmetric_norm", c, restricted=True)
+        check_argument("max_norm", c, restricted=True, min_val=0.1, max_val=1000)
+        check_argument("clip_norm", c, restricted=True)
+        check_argument("mel_fmin", c, restricted=True, min_val=0.0, max_val=1000)
+        check_argument("mel_fmax", c, restricted=True, min_val=500.0, allow_none=True)
+        check_argument("spec_gain", c, restricted=True, min_val=1, max_val=100)
+        check_argument("do_trim_silence", c, restricted=True)
+        check_argument("trim_db", c, restricted=True)
+
+
+@dataclass
+class BaseDatasetConfig(Coqpit):
+    """Base config for TTS datasets.
+
+    Args:
+        name (str):
+            Dataset name that defines the preprocessor in use. Defaults to None.
+        path (str):
+            Root path to the dataset files. Defaults to None.
+        meta_file_train (str):
+            Name of the dataset meta file. Or a list of speakers to be ignored at training for multi-speaker datasets.
+            Defaults to None.
+        unused_speakers (List):
+            List of speakers IDs that are not used at the training. Default None.
+        meta_file_val (str):
+            Name of the dataset meta file that defines the instances used at validation.
+        meta_file_attn_mask (str):
+            Path to the file that lists the attention mask files used with models that require attention masks to
+            train the duration predictor.
+    """
+
+    name: str = ""
+    path: str = ""
+    meta_file_train: str = ""
+    ununsed_speakers: List[str] = None
+    meta_file_val: str = ""
+    meta_file_attn_mask: str = ""
+
+    def check_values(
+        self,
+    ):
+        """Check config fields"""
+        c = asdict(self)
+        check_argument("name", c, restricted=True)
+        check_argument("path", c, restricted=True)
+        check_argument("meta_file_train", c, restricted=True)
+        check_argument("meta_file_val", c, restricted=False)
+        check_argument("meta_file_attn_mask", c, restricted=False)
+
+
+@dataclass
+class BaseTrainingConfig(Coqpit):
+    """Base config to define the basic training parameters that are shared
+    among all the models.
+
+    Args:
+        batch_size (int):
+            Training batch size.
+        eval_batch_size (int):
+            Validation batch size.
+        mixed_precision (bool):
+            Enable / Disable mixed precision training. It reduces the VRAM use and allows larger batch sizes, however
+            it may also cause numerical unstability in some cases.
+        run_eval (bool):
+            Enable / Disable evaluation (validation) run. Defaults to True.
+        test_delay_epochs (int):
+            Number of epochs before starting to use evaluation runs. Initially, models do not generate meaningful
+            results, hence waiting for a couple of epochs might save some time.
+        print_eval (bool):
+            Enable / Disable console logging for evalutaion steps. If disabled then it only shows the final values at
+            the end of the evaluation. Default to ```False```.
+        print_step (int):
+            Number of steps required to print the next training log.
+        tb_plot_step (int):
+            Number of steps required to log training on Tensorboard.
+        tb_model_param_stats (bool):
+            Enable / Disable logging internal model stats for model diagnostic. It might be useful for model debugging.
+            Defaults to ```False```.
+        save_step (int):ipt
+            Number of steps required to save the next checkpoint.
+        checkpoint (bool):
+            Enable / Disable checkpointing.
+        keep_all_best (bool):
+            Enable / Disable keeping all the saved best models instead of overwriting the previous one. Defaults
+            to ```False```.
+        keep_after (int):
+            Number of steps to wait before saving all the best models. In use if ```keep_all_best == True```. Defaults
+            to 10000.
+        num_loader_workers (int):
+            Number of workers for training time dataloader.
+        num_val_loader_workers (int):
+            Number of workers for evaluation time dataloader.
+        output_path (str):
+            Path for training output folder. The nonexist part of the given path is created automatically.
+            All training outputs are saved there.
+    """
+
+    model: str = None
+    run_name: str = ""
+    run_description: str = ""
+    # training params
+    epochs: int = 10000
+    batch_size: int = MISSING
+    eval_batch_size: int = None
+    mixed_precision: bool = False
+    # eval params
+    run_eval: bool = True
+    test_delay_epochs: int = 0
+    print_eval: bool = False
+    # logging
+    print_step: int = 25
+    tb_plot_step: int = 100
+    tb_model_param_stats: bool = False
+    # checkpointing
+    save_step: int = 10000
+    checkpoint: bool = True
+    keep_all_best: bool = False
+    keep_after: int = 10000
+    # dataloading
+    num_loader_workers: int = MISSING
+    num_val_loader_workers: int = 0
+    use_noise_augment: bool = False
+    # paths
+    output_path: str = None
+    # distributed
+    distributed_backend: str = "nccl"
+    distributed_url: str = "tcp://localhost:54321"

TTS/server/README.md

@@ -22,42 +22,3 @@ Run the server with the official models on a GPU.
 
 Run the server with a custom models.
 ```python TTS/server/server.py  --tts_checkpoint /path/to/tts/model.pth.tar --tts_config /path/to/tts/config.json --vocoder_checkpoint /path/to/vocoder/model.pth.tar --vocoder_config /path/to/vocoder/config.json```
-
-<!-- ##### Using .whl
-1. apt-get install -y espeak libsndfile1 python3-venv
-2. python3 -m venv /tmp/venv
-3. source /tmp/venv/bin/activate
-4. pip install -U pip setuptools wheel
-5. pip install -U https//example.com/url/to/python/package.whl
-6. python -m TTS.server.server
-
-You can now open http://localhost:5002 in a browser -->
-
-<!-- #### Running with nginx/uwsgi:
-
-**Note:** This method uses an old TTS model, so quality might be low.
-
-1. apt-get install -y uwsgi uwsgi-plugin-python3 nginx espeak libsndfile1 python3-venv
-2. python3 -m venv /tmp/venv
-3. source /tmp/venv/bin/activate
-4. pip install -U pip setuptools wheel
-5. pip install -U https//example.com/url/to/python/package.whl
-6. curl -LO https://github.com/reuben/TTS/releases/download/t2-ljspeech-mold/t2-ljspeech-mold-nginx-uwsgi.zip
-7. unzip *-nginx-uwsgi.zip
-8. cp tts_site_nginx /etc/nginx/sites-enabled/default
-9. service nginx restart
-10. uwsgi --ini uwsgi.ini
-
-You can now open http://localhost:80 in a browser (edit the port in /etc/nginx/sites-enabled/tts_site_nginx).
-Configure number of workers (number of requests that will be processed in parallel) by editing the `uwsgi.ini` file, specifically the `processes` setting. -->
-
-<!-- #### Creating a server package with an embedded model
-
-[setup.py](../setup.py) was extended with two new parameters when running the `bdist_wheel` command:
-
-- `--checkpoint <path to checkpoint file>` - path to model checkpoint file you want to embed in the package
-- `--model_config <path to config.json file>` - path to corresponding config.json file for the checkpoint
-
-To create a package, run `python setup.py bdist_wheel --checkpoint /path/to/checkpoint --model_config /path/to/config.json`.
-
-A Python `.whl` file will be created in the `dist/` folder with the checkpoint and config embedded in it. -->

TTS/server/server.py

@@ -9,7 +9,7 @@ from typing import Union
 
 from flask import Flask, render_template, request, send_file
 
-from TTS.utils.io import load_config
+from TTS.config import load_config
 from TTS.utils.manage import ModelManager
 from TTS.utils.synthesizer import Synthesizer
 

TTS/speaker_encoder/config.json

@@ -1,103 +0,0 @@
-
-{
-    "run_name": "mueller91",
-    "run_description": "train speaker encoder with voxceleb1, voxceleb2 and libriSpeech ",
-    "audio":{
-        // Audio processing parameters
-        "num_mels": 40,         // size of the mel spec frame. 
-        "fft_size": 400,       // number of stft frequency levels. Size of the linear spectogram frame.
-        "sample_rate": 16000,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "win_length": 400,     // stft window length in ms.
-        "hop_length": 160,      // stft window hop-lengh in ms.
-        "frame_length_ms": null,  // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,   // stft window hop-lengh in ms. If null, 'hop_length' is used.
-        "preemphasis": 0.98,    // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "min_level_db": -100,   // normalization range
-        "ref_level_db": 20,     // reference level db, theoretically 20db is the sound of air.
-        "power": 1.5,           // value to sharpen wav signals after GL algorithm.
-        "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
-        // Normalization parameters
-        "signal_norm": true,    // normalize the spec values in range [0, 1]
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,          // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "mel_fmin": 0.0,         // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 8000.0,        // maximum freq level for mel-spec. Tune for dataset!!
-        "do_trim_silence": true,  // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60          // threshold for timming silence. Set this according to your dataset.
-    },
-    "reinit_layers": [],
-    "loss": "angleproto", // "ge2e" to use Generalized End-to-End loss and "angleproto" to use Angular Prototypical loss (new SOTA)
-    "grad_clip": 3.0, // upper limit for gradients for clipping.
-    "epochs": 1000, // total number of epochs to train.
-    "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "lr_decay": false, // if true, Noam learning rate decaying is applied through training.
-    "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr"
-    "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. 
-    "steps_plot_stats": 10, // number of steps to plot embeddings.
-    "num_speakers_in_batch": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-    "num_utters_per_speaker": 10,  //
-    "num_loader_workers": 8,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "wd": 0.000001, // Weight decay weight.
-    "checkpoint": true, // If true, it saves checkpoints per "save_step"
-    "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints.
-    "print_step": 20, // Number of steps to log traning on console.
-    "output_path": "../../MozillaTTSOutput/checkpoints/voxceleb_librispeech/speaker_encoder/", // DATASET-RELATED: output path for all training outputs.
-    "model": {
-        "input_dim": 40,
-        "proj_dim": 256,
-        "lstm_dim": 768,
-        "num_lstm_layers": 3,
-        "use_lstm_with_projection": true
-    },
-    "storage": {
-        "sample_from_storage_p": 0.66,  // the probability with which we'll sample from the DataSet in-memory storage
-        "storage_size": 15,   // the size of the in-memory storage with respect to a single batch
-        "additive_noise": 1e-5   // add very small gaussian noise to the data in order to increase robustness
-    },
-    "datasets": 
-        [
-            {
-                "name": "vctk_slim",
-                "path": "../../../audio-datasets/en/VCTK-Corpus/",
-                "meta_file_train": null,
-                "meta_file_val": null
-            },
-            {
-                "name": "libri_tts",
-                "path": "../../../audio-datasets/en/LibriTTS/train-clean-100",
-                "meta_file_train": null,
-                "meta_file_val": null
-            },
-            {
-                "name": "libri_tts",
-                "path": "../../../audio-datasets/en/LibriTTS/train-clean-360",
-                "meta_file_train": null,
-                "meta_file_val": null
-            },
-            {
-                "name": "libri_tts",
-                "path": "../../../audio-datasets/en/LibriTTS/train-other-500",
-                "meta_file_train": null,
-                "meta_file_val": null
-            },
-            {
-                "name": "voxceleb1",
-                "path": "../../../audio-datasets/en/voxceleb1/",
-                "meta_file_train": null,
-                "meta_file_val": null
-            },
-            {
-                "name": "voxceleb2",
-                "path": "../../../audio-datasets/en/voxceleb2/",
-                "meta_file_train": null,
-                "meta_file_val": null
-            },
-            {
-                "name": "common_voice",
-                "path": "../../../audio-datasets/en/MozillaCommonVoice",
-                "meta_file_train": "train.tsv",
-                "meta_file_val": "test.tsv"
-            }
-        ]
-}

TTS/speaker_encoder/losses.py

@@ -23,7 +23,7 @@ class GE2ELoss(nn.Module):
         self.b = nn.Parameter(torch.tensor(init_b))
         self.loss_method = loss_method
 
-        print(" > Initialised Generalized End-to-End loss")
+        print(" > Initialized Generalized End-to-End loss")
 
         assert self.loss_method in ["softmax", "contrast"]
 
@@ -136,7 +136,7 @@ class AngleProtoLoss(nn.Module):
         self.b = nn.Parameter(torch.tensor(init_b))
         self.criterion = torch.nn.CrossEntropyLoss()
 
-        print(" > Initialised Angular Prototypical loss")
+        print(" > Initialized Angular Prototypical loss")
 
     def forward(self, x):
         """

TTS/speaker_encoder/speaker_encoder_config.py

@@ -0,0 +1,62 @@
+from dataclasses import asdict, dataclass, field
+from typing import List
+
+from coqpit import MISSING
+
+from TTS.config.shared_configs import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig
+
+
+@dataclass
+class SpeakerEncoderConfig(BaseTrainingConfig):
+    """Defines parameters for Speaker Encoder model."""
+
+    model: str = "speaker_encoder"
+    audio: BaseAudioConfig = field(default_factory=BaseAudioConfig)
+    datasets: List[BaseDatasetConfig] = field(default_factory=lambda: [BaseDatasetConfig()])
+
+    # model params
+    model_params: dict = field(
+        default_factory=lambda: {
+            "input_dim": 40,
+            "proj_dim": 256,
+            "lstm_dim": 768,
+            "num_lstm_layers": 3,
+            "use_lstm_with_projection": True,
+        }
+    )
+
+    storage: dict = field(
+        default_factory=lambda: {
+            "sample_from_storage_p": 0.66,  # the probability with which we'll sample from the DataSet in-memory storage
+            "storage_size": 15,  # the size of the in-memory storage with respect to a single batch
+            "additive_noise": 1e-5,  # add very small gaussian noise to the data in order to increase robustness
+        }
+    )
+
+    # training params
+    max_train_step: int = 1000  # end training when number of training steps reaches this value.
+    loss: str = "angleproto"
+    grad_clip: float = 3.0
+    lr: float = 0.0001
+    lr_decay: bool = False
+    warmup_steps: int = 4000
+    wd: float = 1e-6
+
+    # logging params
+    tb_model_param_stats: bool = False
+    steps_plot_stats: int = 10
+    checkpoint: bool = True
+    save_step: int = 1000
+    print_step: int = 20
+
+    # data loader
+    num_speakers_in_batch: int = MISSING
+    num_utters_per_speaker: int = MISSING
+    num_loader_workers: int = MISSING
+
+    def check_values(self):
+        super().check_values()
+        c = asdict(self)
+        assert (
+            c["model_params"]["input_dim"] == self.audio.num_mels
+        ), " [!] model input dimendion must be equal to melspectrogram dimension."

TTS/speaker_encoder/utils/generic_utils.py

@@ -1,11 +1,6 @@
-import datetime
-import os
 import re
 
-import torch
-
 from TTS.speaker_encoder.model import SpeakerEncoder
-from TTS.utils.generic_utils import check_argument
 
 
 def to_camel(text):
@@ -14,110 +9,10 @@ def to_camel(text):
 
 
 def setup_model(c):
-    model = SpeakerEncoder(c.model["input_dim"], c.model["proj_dim"], c.model["lstm_dim"], c.model["num_lstm_layers"])
+    model = SpeakerEncoder(
+        c.model_params["input_dim"],
+        c.model_params["proj_dim"],
+        c.model_params["lstm_dim"],
+        c.model_params["num_lstm_layers"],
+    )
     return model
-
-
-def save_checkpoint(model, optimizer, model_loss, out_path, current_step, epoch):
-    checkpoint_path = "checkpoint_{}.pth.tar".format(current_step)
-    checkpoint_path = os.path.join(out_path, checkpoint_path)
-    print(" | | > Checkpoint saving : {}".format(checkpoint_path))
-
-    new_state_dict = model.state_dict()
-    state = {
-        "model": new_state_dict,
-        "optimizer": optimizer.state_dict() if optimizer is not None else None,
-        "step": current_step,
-        "epoch": epoch,
-        "loss": model_loss,
-        "date": datetime.date.today().strftime("%B %d, %Y"),
-    }
-    torch.save(state, checkpoint_path)
-
-
-def save_best_model(model, optimizer, model_loss, best_loss, out_path, current_step):
-    if model_loss < best_loss:
-        new_state_dict = model.state_dict()
-        state = {
-            "model": new_state_dict,
-            "optimizer": optimizer.state_dict(),
-            "step": current_step,
-            "loss": model_loss,
-            "date": datetime.date.today().strftime("%B %d, %Y"),
-        }
-        best_loss = model_loss
-        bestmodel_path = "best_model.pth.tar"
-        bestmodel_path = os.path.join(out_path, bestmodel_path)
-        print("\n > BEST MODEL ({0:.5f}) : {1:}".format(model_loss, bestmodel_path))
-        torch.save(state, bestmodel_path)
-    return best_loss
-
-
-def check_config_speaker_encoder(c):
-    """Check the config.json file of the speaker encoder"""
-    check_argument("run_name", c, restricted=True, val_type=str)
-    check_argument("run_description", c, val_type=str)
-
-    # audio processing parameters
-    check_argument("audio", c, restricted=True, val_type=dict)
-    check_argument("num_mels", c["audio"], restricted=True, val_type=int, min_val=10, max_val=2056)
-    check_argument("fft_size", c["audio"], restricted=True, val_type=int, min_val=128, max_val=4058)
-    check_argument("sample_rate", c["audio"], restricted=True, val_type=int, min_val=512, max_val=100000)
-    check_argument(
-        "frame_length_ms",
-        c["audio"],
-        restricted=True,
-        val_type=float,
-        min_val=10,
-        max_val=1000,
-        alternative="win_length",
-    )
-    check_argument(
-        "frame_shift_ms", c["audio"], restricted=True, val_type=float, min_val=1, max_val=1000, alternative="hop_length"
-    )
-    check_argument("preemphasis", c["audio"], restricted=True, val_type=float, min_val=0, max_val=1)
-    check_argument("min_level_db", c["audio"], restricted=True, val_type=int, min_val=-1000, max_val=10)
-    check_argument("ref_level_db", c["audio"], restricted=True, val_type=int, min_val=0, max_val=1000)
-    check_argument("power", c["audio"], restricted=True, val_type=float, min_val=1, max_val=5)
-    check_argument("griffin_lim_iters", c["audio"], restricted=True, val_type=int, min_val=10, max_val=1000)
-
-    # training parameters
-    check_argument("loss", c, enum_list=["ge2e", "angleproto"], restricted=True, val_type=str)
-    check_argument("grad_clip", c, restricted=True, val_type=float)
-    check_argument("epochs", c, restricted=True, val_type=int, min_val=1)
-    check_argument("lr", c, restricted=True, val_type=float, min_val=0)
-    check_argument("lr_decay", c, restricted=True, val_type=bool)
-    check_argument("warmup_steps", c, restricted=True, val_type=int, min_val=0)
-    check_argument("tb_model_param_stats", c, restricted=True, val_type=bool)
-    check_argument("num_speakers_in_batch", c, restricted=True, val_type=int)
-    check_argument("num_loader_workers", c, restricted=True, val_type=int)
-    check_argument("wd", c, restricted=True, val_type=float, min_val=0.0, max_val=1.0)
-
-    # checkpoint and output parameters
-    check_argument("steps_plot_stats", c, restricted=True, val_type=int)
-    check_argument("checkpoint", c, restricted=True, val_type=bool)
-    check_argument("save_step", c, restricted=True, val_type=int)
-    check_argument("print_step", c, restricted=True, val_type=int)
-    check_argument("output_path", c, restricted=True, val_type=str)
-
-    # model parameters
-    check_argument("model", c, restricted=True, val_type=dict)
-    check_argument("input_dim", c["model"], restricted=True, val_type=int)
-    check_argument("proj_dim", c["model"], restricted=True, val_type=int)
-    check_argument("lstm_dim", c["model"], restricted=True, val_type=int)
-    check_argument("num_lstm_layers", c["model"], restricted=True, val_type=int)
-    check_argument("use_lstm_with_projection", c["model"], restricted=True, val_type=bool)
-
-    # in-memory storage parameters
-    check_argument("storage", c, restricted=True, val_type=dict)
-    check_argument("sample_from_storage_p", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0)
-    check_argument("storage_size", c["storage"], restricted=True, val_type=int, min_val=1, max_val=100)
-    check_argument("additive_noise", c["storage"], restricted=True, val_type=float, min_val=0.0, max_val=1.0)
-
-    # datasets - checking only the first entry
-    check_argument("datasets", c, restricted=True, val_type=list)
-    for dataset_entry in c["datasets"]:
-        check_argument("name", dataset_entry, restricted=True, val_type=str)
-        check_argument("path", dataset_entry, restricted=True, val_type=str)
-        check_argument("meta_file_train", dataset_entry, restricted=True, val_type=[str, list])
-        check_argument("meta_file_val", dataset_entry, restricted=True, val_type=str)

TTS/speaker_encoder/utils/io.py

@@ -0,0 +1,38 @@
+import datetime
+import os
+
+import torch
+
+
+def save_checkpoint(model, optimizer, model_loss, out_path, current_step):
+    checkpoint_path = "checkpoint_{}.pth.tar".format(current_step)
+    checkpoint_path = os.path.join(out_path, checkpoint_path)
+    print(" | | > Checkpoint saving : {}".format(checkpoint_path))
+
+    new_state_dict = model.state_dict()
+    state = {
+        "model": new_state_dict,
+        "optimizer": optimizer.state_dict() if optimizer is not None else None,
+        "step": current_step,
+        "loss": model_loss,
+        "date": datetime.date.today().strftime("%B %d, %Y"),
+    }
+    torch.save(state, checkpoint_path)
+
+
+def save_best_model(model, optimizer, model_loss, best_loss, out_path, current_step):
+    if model_loss < best_loss:
+        new_state_dict = model.state_dict()
+        state = {
+            "model": new_state_dict,
+            "optimizer": optimizer.state_dict(),
+            "step": current_step,
+            "loss": model_loss,
+            "date": datetime.date.today().strftime("%B %d, %Y"),
+        }
+        best_loss = model_loss
+        bestmodel_path = "best_model.pth.tar"
+        bestmodel_path = os.path.join(out_path, bestmodel_path)
+        print("\n > BEST MODEL ({0:.5f}) : {1:}".format(model_loss, bestmodel_path))
+        torch.save(state, bestmodel_path)
+    return best_loss

TTS/tts/configs/__init__.py

@@ -0,0 +1,17 @@
+import importlib
+import os
+from inspect import isclass
+
+# import all files under configs/
+configs_dir = os.path.dirname(__file__)
+for file in os.listdir(configs_dir):
+    path = os.path.join(configs_dir, file)
+    if not file.startswith("_") and not file.startswith(".") and (file.endswith(".py") or os.path.isdir(path)):
+        config_name = file[: file.find(".py")] if file.endswith(".py") else file
+        module = importlib.import_module("TTS.tts.configs." + config_name)
+        for attribute_name in dir(module):
+            attribute = getattr(module, attribute_name)
+
+            if isclass(attribute):
+                # Add the class to this package's variables
+                globals()[attribute_name] = attribute

TTS/tts/configs/align_tts_config.py

@@ -0,0 +1,104 @@
+from dataclasses import dataclass, field
+from typing import List
+
+from TTS.tts.configs.shared_configs import BaseTTSConfig
+
+
+@dataclass
+class AlignTTSConfig(BaseTTSConfig):
+    """Defines parameters for AlignTTS model.
+    Example:
+
+        >>> from TTS.tts.configs import AlignTTSConfig
+        >>> config = AlignTTSConfig()
+
+    Args:
+        model(str):
+            Model name used for selecting the right model at initialization. Defaults to `align_tts`.
+        positional_encoding (bool):
+            enable / disable positional encoding applied to the encoder output. Defaults to True.
+        hidden_channels (int):
+            Base number of hidden channels. Defines all the layers expect ones defined by the specific encoder or decoder
+            parameters. Defaults to 256.
+        hidden_channels_dp (int):
+            Number of hidden channels of the duration predictor's layers. Defaults to 256.
+        encoder_type (str):
+            Type of the encoder used by the model. Look at `TTS.tts.layers.feed_forward.encoder` for more details.
+            Defaults to `fftransformer`.
+        encoder_params (dict):
+            Parameters used to define the encoder network. Look at `TTS.tts.layers.feed_forward.encoder` for more details.
+            Defaults to `{"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}`.
+        decoder_type (str):
+            Type of the decoder used by the model. Look at `TTS.tts.layers.feed_forward.decoder` for more details.
+            Defaults to `fftransformer`.
+        decoder_params (dict):
+            Parameters used to define the decoder network. Look at `TTS.tts.layers.feed_forward.decoder` for more details.
+            Defaults to `{"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}`.
+        phase_start_steps (List[int]):
+            A list of number of steps required to start the next training phase. AlignTTS has 4 different training
+            phases. Thus you need to define 4 different values to enable phase based training. If None, it
+            trains the whole model together. Defaults to None.
+        ssim_alpha (float):
+            Weight for the SSIM loss. If set <= 0, disables the SSIM loss. Defaults to 1.0.
+        duration_loss_alpha (float):
+            Weight for the duration predictor's loss. Defaults to 1.0.
+        mdn_alpha (float):
+            Weight for the MDN loss. Defaults to 1.0.
+        spec_loss_alpha (float):
+            Weight for the MSE spectrogram loss. If set <= 0, disables the L1 loss. Defaults to 1.0.
+        use_speaker_embedding (bool):
+            enable / disable using speaker embeddings for multi-speaker models. If set True, the model is
+            in the multi-speaker mode. Defaults to False.
+        use_external_speaker_embedding_file (bool):
+            enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False.
+        external_speaker_embedding_file (str):
+            Path to the file including pre-computed speaker embeddings. Defaults to None.
+        noam_schedule (bool):
+            enable / disable the use of Noam LR scheduler. Defaults to False.
+        warmup_steps (int):
+            Number of warm-up steps for the Noam scheduler. Defaults 4000.
+        lr (float):
+            Initial learning rate. Defaults to `1e-3`.
+        wd (float):
+            Weight decay coefficient. Defaults to `1e-7`.
+        min_seq_len (int):
+            Minimum input sequence length to be used at training.
+        max_seq_len (int):
+            Maximum input sequence length to be used at training. Larger values result in more VRAM usage."""
+
+    model: str = "align_tts"
+    # model specific params
+    positional_encoding: bool = True
+    hidden_channels_dp: int = 256
+    hidden_channels: int = 256
+    encoder_type: str = "fftransformer"
+    encoder_params: dict = field(
+        default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}
+    )
+    decoder_type: str = "fftransformer"
+    decoder_params: dict = field(
+        default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}
+    )
+    phase_start_steps: List[int] = None
+
+    ssim_alpha: float = 1.0
+    spec_loss_alpha: float = 1.0
+    dur_loss_alpha: float = 1.0
+    mdn_alpha: float = 1.0
+
+    # multi-speaker settings
+    use_speaker_embedding: bool = False
+    use_external_speaker_embedding_file: bool = False
+    external_speaker_embedding_file: str = False
+
+    # optimizer parameters
+    noam_schedule: bool = False
+    warmup_steps: int = 4000
+    lr: float = 1e-4
+    wd: float = 1e-6
+    grad_clip: float = 5.0
+
+    # overrides
+    min_seq_len: int = 13
+    max_seq_len: int = 200
+    r: int = 1

TTS/tts/configs/config.json

@@ -1,173 +0,0 @@
-{
-    "model": "Tacotron2",
-    "run_name": "ljspeech-ddc",
-    "run_description": "tacotron2 with DDC and differential spectral loss.",
-
-    // AUDIO PARAMETERS
-    "audio":{
-        // stft parameters
-        "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 20,     // reference level db, theoretically 20db is the sound of air.
-
-        // Silence trimming
-        "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // Griffin-Lim
-        "power": 1.5,           // value to sharpen wav signals after GL algorithm.
-        "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 1,
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy"    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // VOCABULARY PARAMETERS
-    // if custom character set is not defined,
-    // default set in symbols.py is used
-    // "characters":{
-    //     "pad": "_",
-    //     "eos": "~",
-    //     "bos": "^",
-    //     "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ",
-    //     "punctuations":"!'(),-.:;? ",
-    //     "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ"
-    // },
-
-    // DISTRIBUTED TRAINING
-    "distributed":{
-        "backend": "nccl",
-        "url": "tcp:\/\/localhost:54321"
-    },
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // TRAINING
-    "batch_size": 32,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-    "eval_batch_size":16,
-    "r": 7,                 // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled.
-    "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed.
-    "mixed_precision": true,     // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate.
-
-    // LOSS SETTINGS
-    "loss_masking": true,       // enable / disable loss masking against the sequence padding.
-    "decoder_loss_alpha": 0.5,  // original decoder loss weight. If > 0, it is enabled
-    "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled
-    "postnet_diff_spec_alpha": 0.25,     // differential spectral loss weight. If > 0, it is enabled
-    "decoder_diff_spec_alpha": 0.25,     // differential spectral loss weight. If > 0, it is enabled
-    "decoder_ssim_alpha": 0.5,     // decoder ssim loss weight. If > 0, it is enabled
-    "postnet_ssim_alpha": 0.25,     // postnet ssim loss weight. If > 0, it is enabled
-    "ga_alpha": 5.0,           // weight for guided attention loss. If > 0, guided attention is enabled.
-    "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples.
-
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": 10,  //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "noam_schedule": false,        // use noam warmup and lr schedule.
-    "grad_clip": 1.0,              // upper limit for gradients for clipping.
-    "epochs": 1000,                // total number of epochs to train.
-    "lr": 0.0001,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "wd": 0.000001,                // Weight decay weight.
-    "warmup_steps": 4000,          // Noam decay steps to increase the learning rate from 0 to "lr"
-    "seq_len_norm": false,         // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths.
-
-    // TACOTRON PRENET
-    "memory_size": -1,             // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame.
-    "prenet_type": "original",     // "original" or "bn".
-    "prenet_dropout": true,       // enable/disable dropout at prenet.
-
-    // TACOTRON ATTENTION
-    "attention_type": "original",  // 'original' , 'graves', 'dynamic_convolution'
-    "attention_heads": 4,          // number of attention heads (only for 'graves')
-    "attention_norm": "sigmoid",   // softmax or sigmoid.
-    "windowing": false,            // Enables attention windowing. Used only in eval mode.
-    "use_forward_attn": false,     // if it uses forward attention. In general, it aligns faster.
-    "forward_attn_mask": false,    // Additional masking forcing monotonicity only in eval mode.
-    "transition_agent": false,     // enable/disable transition agent of forward attention.
-    "location_attn": true,         // enable_disable location sensitive attention. It is enabled for TACOTRON by default.
-    "bidirectional_decoder": false,  // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset.
-    "double_decoder_consistency": true,  // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/
-    "ddc_r": 7,                           // reduction rate for coarse decoder.
-
-    // STOPNET
-    "stopnet": true,               // Train stopnet predicting the end of synthesis.
-    "separate_stopnet": true,      // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER.
-
-    // TENSORBOARD and LOGGING
-    "print_step": 25,       // Number of steps to log training on console.
-    "tb_plot_step": 100,    // Number of steps to plot TB training figures.
-    "print_eval": false,     // If True, it prints intermediate loss values in evalulation.
-    "save_step": 10000,      // Number of training steps expected to save traninpg stats and checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-    // DATA LOADING
-    "text_cleaner": "phoneme_cleaners",
-    "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars.
-    "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "batch_group_size": 4,  //Number of batches to shuffle after bucketing.
-    "min_seq_len": 6,       // DATASET-RELATED: minimum text length to use in training
-    "max_seq_len": 153,     // DATASET-RELATED: maximum text length
-    "compute_input_seq_cache": false,  // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage.
-    "use_noise_augment": true,
-
-    // PATHS
-    "output_path": "/home/erogol/Models/LJSpeech/",
-
-    // PHONEMES
-    "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/",  // phoneme computation is slow, therefore, it caches results in the given folder.
-    "use_phonemes": true,           // use phonemes instead of raw characters. It is suggested for better pronounciation.
-    "phoneme_language": "en-us",     // depending on your target language, pick one from  https://github.com/bootphon/phonemizer#languages
-
-    // MULTI-SPEAKER and GST
-    "use_speaker_embedding": false,      // use speaker embedding to enable multi-speaker learning.
-    "use_gst": false,       			    // use global style tokens
-    "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558
-    "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558
-    "gst":	{			                // gst parameter if gst is enabled
-        "gst_style_input": null,        // Condition the style input either on a
-                                        // -> wave file [path to wave] or
-                                        // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15}
-                                        // with the dictionary being len(dict) <= len(gst_style_tokens).
-        "gst_embedding_dim": 512,
-        "gst_num_heads": 4,
-        "gst_style_tokens": 10,
-        "gst_use_speaker_embedding": false
-	},
-
-    // DATASETS
-    "datasets":   // List of datasets. They all merged and they get different speaker_ids.
-        [
-            {
-                "name": "ljspeech",
-                "path": "/home/erogol/Data/LJSpeech-1.1/",
-                "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers
-                "meta_file_val": null
-            }
-        ]
-}

TTS/tts/configs/glow_tts_config.py

@@ -0,0 +1,102 @@
+from dataclasses import dataclass, field
+
+from TTS.tts.configs.shared_configs import BaseTTSConfig
+
+
+@dataclass
+class GlowTTSConfig(BaseTTSConfig):
+    """Defines parameters for GlowTTS model.
+
+     Example:
+
+        >>> from TTS.tts.configs import GlowTTSConfig
+        >>> config = GlowTTSConfig()
+
+    Args:
+        model(str):
+            Model name used for selecting the right model at initialization. Defaults to `glow_tts`.
+        encoder_type (str):
+            Type of the encoder used by the model. Look at `TTS.tts.layers.glow_tts.encoder` for more details.
+            Defaults to `rel_pos_transformers`.
+        encoder_params (dict):
+            Parameters used to define the encoder network. Look at `TTS.tts.layers.glow_tts.encoder` for more details.
+            Defaults to `{"kernel_size": 3, "dropout_p": 0.1, "num_layers": 6, "num_heads": 2, "hidden_channels_ffn": 768}`
+        use_encoder_prenet (bool):
+            enable / disable the use of a prenet for the encoder. Defaults to True.
+        hidden_channels_encoder (int):
+            Number of base hidden channels used by the encoder network. It defines the input and the output channel sizes,
+            and for some encoder types internal hidden channels sizes too. Defaults to 192.
+        hidden_channels_decoder (int):
+            Number of base hidden channels used by the decoder WaveNet network. Defaults to 192 as in the original work.
+        hidden_channels_duration_predictor (int):
+            Number of layer channels of the duration predictor network. Defaults to 256 as in the original work.
+        data_dep_init_steps (int):
+            Number of steps used for computing normalization parameters at the beginning of the training. GlowTTS uses
+            Activation Normalization that pre-computes normalization stats at the beginning and use the same values
+            for the rest. Defaults to 10.
+        style_wav_for_test (str):
+            Path to the wav file used for changing the style of the speech. Defaults to None.
+        inference_noise_scale (float):
+            Variance used for sampling the random noise added to the decoder's input at inference. Defaults to 0.0.
+        use_speaker_embedding (bool):
+            enable / disable using speaker embeddings for multi-speaker models. If set True, the model is
+            in the multi-speaker mode. Defaults to False.
+        use_external_speaker_embedding_file (bool):
+            enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False.
+        external_speaker_embedding_file (str):
+            Path to the file including pre-computed speaker embeddings. Defaults to None.
+        noam_schedule (bool):
+            enable / disable the use of Noam LR scheduler. Defaults to False.
+        warmup_steps (int):
+            Number of warm-up steps for the Noam scheduler. Defaults 4000.
+        lr (float):
+            Initial learning rate. Defaults to `1e-3`.
+        wd (float):
+            Weight decay coefficient. Defaults to `1e-7`.
+        min_seq_len (int):
+            Minimum input sequence length to be used at training.
+        max_seq_len (int):
+            Maximum input sequence length to be used at training. Larger values result in more VRAM usage.
+    """
+
+    model: str = "glow_tts"
+
+    # model params
+    encoder_type: str = "rel_pos_transformer"
+    encoder_params: dict = field(
+        default_factory=lambda: {
+            "kernel_size": 3,
+            "dropout_p": 0.1,
+            "num_layers": 6,
+            "num_heads": 2,
+            "hidden_channels_ffn": 768,
+        }
+    )
+    use_encoder_prenet: bool = True
+    hidden_channels_encoder: int = 192
+    hidden_channels_decoder: int = 192
+    hidden_channels_duration_predictor: int = 256
+
+    # training params
+    data_dep_init_steps: int = 10
+
+    # inference params
+    style_wav_for_test: str = None
+    inference_noise_scale: float = 0.0
+
+    # multi-speaker settings
+    use_speaker_embedding: bool = False
+    use_external_speaker_embedding_file: bool = False
+    external_speaker_embedding_file: str = False
+
+    # optimizer params
+    noam_schedule: bool = True
+    warmup_steps: int = 4000
+    grad_clip: float = 5.0
+    lr: float = 1e-3
+    wd: float = 0.000001
+
+    # overrides
+    min_seq_len: int = 3
+    max_seq_len: int = 500
+    r: int = 1  # DO NOT CHANGE - TODO: make this immutable once coqpit implements it.

TTS/tts/configs/glow_tts_gated_conv.json

@@ -1,138 +0,0 @@
-{
-    "model": "glow_tts",
-    "run_name": "glow-tts-gatedconv",
-    "run_description": "glow-tts model training with gated conv.",
-
-    // AUDIO PARAMETERS
-    "audio":{
-        "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 0,     // reference level db, theoretically 20db is the sound of air.
-
-        // Griffin-Lim
-        "power": 1.1,           // value to sharpen wav signals after GL algorithm.
-        "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
-
-        // Silence trimming
-        "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 1.0,         // scaler value appplied after log transform of spectrogram.
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 1.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy"    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // VOCABULARY PARAMETERS
-    // if custom character set is not defined,
-    // default set in symbols.py is used
-    // "characters":{
-    //     "pad": "_",
-    //     "eos": "~",
-    //     "bos": "^",
-    //     "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ",
-    //     "punctuations":"!'(),-.:;? ",
-    //     "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ"
-    // },
-
-    "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model.
-
-    // DISTRIBUTED TRAINING
-    "apex_amp_level": null,     // APEX amp optimization level. "O1" is currently supported.
-    "distributed":{
-        "backend": "nccl",
-        "url": "tcp:\/\/localhost:54323"
-    },
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // MODEL PARAMETERS
-    "use_mas": false,       // use Monotonic Alignment Search if true. Otherwise use pre-computed attention alignments.
-
-    // TRAINING
-    "batch_size": 32,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-    "eval_batch_size":16,
-    "r": 1,                 // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled.
-    "loss_masking": true,   // enable / disable loss masking against the sequence padding.
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": 0,       //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "noam_schedule": true,         // use noam warmup and lr schedule.
-    "grad_clip": 5.0,              // upper limit for gradients for clipping.
-    "epochs": 10000,               // total number of epochs to train.
-    "lr": 1e-3,                    // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "wd": 0.000001,                // Weight decay weight.
-    "warmup_steps": 4000,          // Noam decay steps to increase the learning rate from 0 to "lr"
-    "seq_len_norm": false,         // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths.
-
-    "encoder_type": "gatedconv",
-
-    // TENSORBOARD and LOGGING
-    "print_step": 25,       // Number of steps to log training on console.
-    "tb_plot_step": 100,    // Number of steps to plot TB training figures.
-    "print_eval": false,     // If True, it prints intermediate loss values in evalulation.
-    "save_step": 5000,      // Number of training steps expected to save traninpg stats and checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-    "apex_amp_level": null,
-
-    // DATA LOADING
-    "text_cleaner": "phoneme_cleaners",
-    "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars.
-    "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "batch_group_size": 0,  //Number of batches to shuffle after bucketing.
-    "min_seq_len": 3,       // DATASET-RELATED: minimum text length to use in training
-    "max_seq_len": 500,     // DATASET-RELATED: maximum text length
-    "compute_f0": false,     // compute f0 values in data-loader
-    "compute_input_seq_cache": false,  // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage.
-
-    // PATHS
-    "output_path": "/home/erogol/Models/LJSpeech/",
-
-    // PHONEMES
-    "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/",  // phoneme computation is slow, therefore, it caches results in the given folder.
-    "use_phonemes": true,           // use phonemes instead of raw characters. It is suggested for better pronounciation.
-    "phoneme_language": "en-us",     // depending on your target language, pick one from  https://github.com/bootphon/phonemizer#languages
-
-    // MULTI-SPEAKER and GST
-    "use_speaker_embedding": false,     // use speaker embedding to enable multi-speaker learning.
-    "style_wav_for_test": null,          // path to style wav file to be used in TacotronGST inference.
-    "use_gst": false,       // TACOTRON ONLY: use global style tokens
-
-    // DATASETS
-    "datasets":   // List of datasets. They all merged and they get different speaker_ids.
-        [
-            {
-                "name": "ljspeech",
-                "path": "/home/erogol/Data/LJSpeech-1.1/",
-                "meta_file_train": "metadata.csv",
-                "meta_file_val": null
-                // "path_for_attn": "/home/erogol/Data/LJSpeech-1.1/alignments/"
-            }
-        ]
-}
-
-

TTS/tts/configs/glow_tts_ljspeech.json

@@ -1,151 +0,0 @@
-{
-        "model": "glow_tts",
-        "run_name": "glow-tts-residual_bn_conv",
-        "run_description": "glow-tts model training with residual BN conv.",
-
-        // AUDIO PARAMETERS
-        "audio":{
-            "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-            "win_length": 1024,      // stft window length in ms.
-            "hop_length": 256,       // stft window hop-lengh in ms.
-            "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-            "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-            // Audio processing parameters
-            "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-            "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-            "ref_level_db": 0,     // reference level db, theoretically 20db is the sound of air.
-
-            // Griffin-Lim
-            "power": 1.1,           // value to sharpen wav signals after GL algorithm.
-            "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
-
-            // Silence trimming
-            "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-            "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-            // MelSpectrogram parameters
-            "num_mels": 80,         // size of the mel spec frame.
-            "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-            "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-            "spec_gain": 1.0,         // scaler value appplied after log transform of spectrogram.00
-
-            // Normalization parameters
-            "signal_norm": false,   // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-            "min_level_db": -100,   // lower bound for normalization
-            "symmetric_norm": true, // move normalization to range [-1, 1]
-            "max_norm": 1.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-            "clip_norm": true,      // clip normalized values into the range.
-            "stats_path": null    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-        },
-
-        // VOCABULARY PARAMETERS
-        // if custom character set is not defined,
-        // default set in symbols.py is used
-        // "characters":{
-        //     "pad": "_",
-        //     "eos": "~",
-        //     "bos": "^",
-        //     "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ",
-        //     "punctuations":"!'(),-.:;? ",
-        //     "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ"
-        // },
-
-        "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model.
-
-        // DISTRIBUTED TRAINING
-        "distributed":{
-            "backend": "nccl",
-            "url": "tcp:\/\/localhost:54321"
-        },
-
-        "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-        // MODEL PARAMETERS
-        // "use_mas": false,       // use Monotonic Alignment Search if true. Otherwise use pre-computed attention alignments.
-        "hidden_channels_encoder": 192,
-        "hidden_channels_decoder": 192,
-        "hidden_channels_duration_predictor": 256,
-        "use_encoder_prenet": true,
-        "encoder_type": "rel_pos_transformer",
-        "encoder_params": {
-            "kernel_size":3,
-            "dropout_p": 0.1,
-            "num_layers": 6,
-            "num_heads": 2,
-            "hidden_channels_ffn": 768,
-            "input_length": null
-        },
-
-        // TRAINING
-        "batch_size": 32,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-        "eval_batch_size":16,
-        "r": 1,                 // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled.
-        "loss_masking": true,   // enable / disable loss masking against the sequence padding.
-        "mixed_precision": true,
-        "data_dep_init_iter": 10,
-
-        // VALIDATION
-        "run_eval": true,
-        "test_delay_epochs": 0,       //Until attention is aligned, testing only wastes computation time.
-        "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-        // OPTIMIZER
-        "noam_schedule": true,         // use noam warmup and lr schedule.
-        "grad_clip": 5.0,              // upper limit for gradients for clipping.
-        "epochs": 10000,               // total number of epochs to train.
-        "lr": 1e-3,                    // Initial learning rate. If Noam decay is active, maximum learning rate.
-        "wd": 0.000001,                // Weight decay weight.
-        "warmup_steps": 4000,          // Noam decay steps to increase the learning rate from 0 to "lr"
-        "seq_len_norm": false,         // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths.
-
-        // TENSORBOARD and LOGGING
-        "print_step": 25,       // Number of steps to log training on console.
-        "tb_plot_step": 100,    // Number of steps to plot TB training figures.
-        "print_eval": false,     // If True, it prints intermediate loss values in evalulation.
-        "save_step": 5000,      // Number of training steps expected to save traninpg stats and checkpoints.
-        "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-        "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-        "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-        "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-        // DATA LOADING
-        "text_cleaner": "phoneme_cleaners",
-        "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars.
-        "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-        "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-        "batch_group_size": 0,  //Number of batches to shuffle after bucketing.
-        "min_seq_len": 3,       // DATASET-RELATED: minimum text length to use in training
-        "max_seq_len": 500,     // DATASET-RELATED: maximum text length
-        "compute_f0": false,     // compute f0 values in data-loader
-        "use_noise_augment": true,  //add a random noise to audio signal for augmentation at training .
-        "compute_input_seq_cache": true,
-
-        // PATHS
-        "output_path": "/home/erogol/Models/LJSpeech/",
-
-        // PHONEMES
-        "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/",  // phoneme computation is slow, therefore, it caches results in the given folder.
-        "use_phonemes": true,           // use phonemes instead of raw characters. It is suggested for better pronounciation.
-        "phoneme_language": "en-us",     // depending on your target language, pick one from  https://github.com/bootphon/phonemizer#languages
-
-        // MULTI-SPEAKER and GST
-        "use_speaker_embedding": false,     // use speaker embedding to enable multi-speaker learning.
-        "use_external_speaker_embedding_file": false,
-        "style_wav_for_test": null,          // path to style wav file to be used in TacotronGST inference.
-        "use_gst": false,       // TACOTRON ONLY: use global style tokens
-
-        // DATASETS
-        "datasets":   // List of datasets. They all merged and they get different speaker_ids.
-            [
-                {
-                    "name": "ljspeech",
-                    "path": "/home/erogol/Data/LJSpeech-1.1/",
-                    "meta_file_train": "metadata.csv",
-                    "meta_file_val": null
-                    // "path_for_attn": "/home/erogol/Data/LJSpeech-1.1/alignments/"
-                }
-            ]
-    }
-
-

TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json

@@ -1,173 +0,0 @@
-{
-    "model": "Tacotron2",
-    "run_name": "ljspeech-dcattn",
-    "run_description": "tacotron2 with dynamic convolution attention.",
-
-    // AUDIO PARAMETERS
-    "audio":{
-        // stft parameters
-        "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 20,     // reference level db, theoretically 20db is the sound of air.
-
-        // Silence trimming
-        "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // Griffin-Lim
-        "power": 1.5,           // value to sharpen wav signals after GL algorithm.
-        "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 1,
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy"    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // VOCABULARY PARAMETERS
-    // if custom character set is not defined,
-    // default set in symbols.py is used
-    // "characters":{
-    //     "pad": "_",
-    //     "eos": "~",
-    //     "bos": "^",
-    //     "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ",
-    //     "punctuations":"!'(),-.:;? ",
-    //     "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ"
-    // },
-
-    // DISTRIBUTED TRAINING
-    "distributed":{
-        "backend": "nccl",
-        "url": "tcp:\/\/localhost:54321"
-    },
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // TRAINING
-    "batch_size": 32,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-    "eval_batch_size":16,
-    "r": 7,                 // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled.
-    "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed.
-    "mixed_precision": true,     // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate.
-
-    // LOSS SETTINGS
-    "loss_masking": true,       // enable / disable loss masking against the sequence padding.
-    "decoder_loss_alpha": 0.5,  // original decoder loss weight. If > 0, it is enabled
-    "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled
-    "postnet_diff_spec_alpha": 0.25,     // differential spectral loss weight. If > 0, it is enabled
-    "decoder_diff_spec_alpha": 0.25,     // differential spectral loss weight. If > 0, it is enabled
-    "decoder_ssim_alpha": 0.5,     // decoder ssim loss weight. If > 0, it is enabled
-    "postnet_ssim_alpha": 0.25,     // postnet ssim loss weight. If > 0, it is enabled
-    "ga_alpha": 0.0,           // weight for guided attention loss. If > 0, guided attention is enabled.
-    "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples.
-
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": 10,  //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "noam_schedule": false,        // use noam warmup and lr schedule.
-    "grad_clip": 1.0,              // upper limit for gradients for clipping.
-    "epochs": 1000,                // total number of epochs to train.
-    "lr": 0.0001,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "wd": 0.000001,                // Weight decay weight.
-    "warmup_steps": 4000,          // Noam decay steps to increase the learning rate from 0 to "lr"
-    "seq_len_norm": false,         // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths.
-
-    // TACOTRON PRENET
-    "memory_size": -1,             // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame.
-    "prenet_type": "original",     // "original" or "bn".
-    "prenet_dropout": false,       // enable/disable dropout at prenet.
-
-    // TACOTRON ATTENTION
-    "attention_type": "dynamic_convolution",  // 'original' , 'graves', 'dynamic_convolution'
-    "attention_heads": 4,          // number of attention heads (only for 'graves')
-    "attention_norm": "softmax",   // softmax or sigmoid.
-    "windowing": false,            // Enables attention windowing. Used only in eval mode.
-    "use_forward_attn": false,     // if it uses forward attention. In general, it aligns faster.
-    "forward_attn_mask": false,    // Additional masking forcing monotonicity only in eval mode.
-    "transition_agent": false,     // enable/disable transition agent of forward attention.
-    "location_attn": true,         // enable_disable location sensitive attention. It is enabled for TACOTRON by default.
-    "bidirectional_decoder": false,  // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset.
-    "double_decoder_consistency": false,  // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/
-    "ddc_r": 7,                           // reduction rate for coarse decoder.
-
-    // STOPNET
-    "stopnet": true,               // Train stopnet predicting the end of synthesis.
-    "separate_stopnet": true,      // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER.
-
-    // TENSORBOARD and LOGGING
-    "print_step": 25,       // Number of steps to log training on console.
-    "tb_plot_step": 100,    // Number of steps to plot TB training figures.
-    "print_eval": false,     // If True, it prints intermediate loss values in evalulation.
-    "save_step": 10000,      // Number of training steps expected to save traninpg stats and checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-    // DATA LOADING
-    "text_cleaner": "phoneme_cleaners",
-    "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars.
-    "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "batch_group_size": 4,  //Number of batches to shuffle after bucketing.
-    "min_seq_len": 6,       // DATASET-RELATED: minimum text length to use in training
-    "max_seq_len": 153,     // DATASET-RELATED: maximum text length
-    "compute_input_seq_cache": false,  // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage.
-
-    // PATHS
-    "output_path": "/home/erogol/Models/LJSpeech/",
-
-    // PHONEMES
-    "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/",  // phoneme computation is slow, therefore, it caches results in the given folder.
-    "use_phonemes": true,           // use phonemes instead of raw characters. It is suggested for better pronounciation.
-    "phoneme_language": "en-us",     // depending on your target language, pick one from  https://github.com/bootphon/phonemizer#languages
-
-    // MULTI-SPEAKER and GST
-    "use_speaker_embedding": false,      // use speaker embedding to enable multi-speaker learning.
-    "use_gst": false,       			    // use global style tokens
-    "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558
-    "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558
-    "gst":	{			                // gst parameter if gst is enabled
-        "gst_style_input": null,        // Condition the style input either on a
-                                        // -> wave file [path to wave] or
-                                        // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15}
-                                        // with the dictionary being len(dict) <= len(gst_style_tokens).
-        "gst_embedding_dim": 512,
-        "gst_num_heads": 4,
-        "gst_style_tokens": 10,
-        "gst_use_speaker_embedding": false
-	},
-
-    // DATASETS
-    "datasets":   // List of datasets. They all merged and they get different speaker_ids.
-        [
-            {
-                "name": "ljspeech",
-                "path": "/home/erogol/Data/LJSpeech-1.1/",
-                "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers
-                "meta_file_val": null
-            }
-        ]
-}
-

TTS/tts/configs/shared_configs.py

@@ -0,0 +1,157 @@
+from dataclasses import asdict, dataclass, field
+from typing import List
+
+from coqpit import MISSING, Coqpit, check_argument
+
+from TTS.config import BaseAudioConfig, BaseDatasetConfig, BaseTrainingConfig
+
+
+@dataclass
+class GSTConfig(Coqpit):
+    """Defines the Global Style Token Module
+
+    Args:
+        gst_style_input_wav (str):
+            Path to the wav file used to define the style of the output speech at inference. Defaults to None.
+        gst_style_input_weights (dict):
+            Defines the weights for each style token used at inference. Defaults to None.
+        gst_embedding_dim (int):
+            Defines the size of the GST embedding vector dimensions. Defaults to 256.
+        gst_num_heads (int):
+            Number of attention heads used by the multi-head attention. Defaults to 4.
+        gst_num_style_tokens (int):
+            Number of style token vectors. Defaults to 10.
+    """
+
+    gst_style_input_wav: str = None
+    gst_style_input_weights: dict = None
+    gst_embedding_dim: int = 256
+    gst_use_speaker_embedding: bool = False
+    gst_num_heads: int = 4
+    gst_num_style_tokens: int = 10
+
+    def check_values(
+        self,
+    ):
+        """Check config fields"""
+        c = asdict(self)
+        super().check_values()
+        check_argument("gst_style_input_weights", c, restricted=False)
+        check_argument("gst_style_input_wav", c, restricted=False)
+        check_argument("gst_embedding_dim", c, restricted=True, min_val=0, max_val=1000)
+        check_argument("gst_use_speaker_embedding", c, restricted=False)
+        check_argument("gst_num_heads", c, restricted=True, min_val=2, max_val=10)
+        check_argument("gst_num_style_tokens", c, restricted=True, min_val=1, max_val=1000)
+
+
+@dataclass
+class CharactersConfig(Coqpit):
+    """Defines character or phoneme set used by the model
+
+    Args:
+        pad (str):
+            characters in place of empty padding. Defaults to None.
+        eos (str):
+            characters showing the end of a sentence. Defaults to None.
+        bos (str):
+            characters showing the beginning of a sentence. Defaults to None.
+        characters (str):
+            character set used by the model. Characters not in this list are ignored when converting input text to
+            a list of sequence IDs. Defaults to None.
+        punctuations (str):
+            characters considered as punctuation as parsing the input sentence. Defaults to None.
+        phonemes (str):
+            characters considered as parsing phonemes. Defaults to None.
+        unique (bool):
+            remove any duplicate characters in the character lists. It is a bandaid for compatibility with the old
+            models trained with character lists with duplicates.
+    """
+
+    pad: str = None
+    eos: str = None
+    bos: str = None
+    characters: str = None
+    punctuations: str = None
+    phonemes: str = None
+    unique: bool = True  # for backwards compatibility of models trained with char sets with duplicates
+
+    def check_values(
+        self,
+    ):
+        """Check config fields"""
+        c = asdict(self)
+        check_argument("pad", c, "characters", restricted=True)
+        check_argument("eos", c, "characters", restricted=True)
+        check_argument("bos", c, "characters", restricted=True)
+        check_argument("characters", c, "characters", restricted=True)
+        check_argument("phonemes", c, restricted=True)
+        check_argument("punctuations", c, "characters", restricted=True)
+
+
+@dataclass
+class BaseTTSConfig(BaseTrainingConfig):
+    """Shared parameters among all the tts models.
+
+    Args:
+        audio (BaseAudioConfig):
+            Audio processor config object instance.
+        use_phonemes (bool):
+            enable / disable phoneme use.
+        compute_input_seq_cache (bool):
+            enable / disable precomputation of the phoneme sequences. At the expense of some delay at the beginning of
+            the training, It allows faster data loader time and precise limitation with `max_seq_len` and
+            `min_seq_len`.
+        text_cleaner (str):
+            Name of the text cleaner used for cleaning and formatting transcripts.
+        enable_eos_bos_chars (bool):
+            enable / disable the use of eos and bos characters.
+        test_senteces_file (str):
+            Path to a txt file that has sentences used at test time. The file must have a sentence per line.
+        phoneme_cache_path (str):
+            Path to the output folder caching the computed phonemes for each sample.
+        characters (CharactersConfig):
+            Instance of a CharactersConfig class.
+        batch_group_size (int):
+            Size of the batch groups used for bucketing. By default, the dataloader orders samples by the sequence
+            length for a more efficient and stable training. If `batch_group_size > 1` then it performs bucketing to
+            prevent using the same batches for each epoch.
+        loss_masking (bool):
+            enable / disable masking loss values against padded segments of samples in a batch.
+        min_seq_len (int):
+            Minimum input sequence length to be used at training.
+        max_seq_len (int):
+            Maximum input sequence length to be used at training. Larger values result in more VRAM usage.
+        compute_f0 (int):
+            (Not in use yet).
+        use_noise_augment (bool):
+            Augment the input audio with random noise.
+        add_blank (bool):
+            Add blank characters between each other two characters. It improves performance for some models at expense
+            of slower run-time due to the longer input sequence.
+        datasets (List[BaseDatasetConfig]):
+            List of datasets used for training. If multiple datasets are provided, they are merged and used together
+            for training.
+    """
+
+    audio: BaseAudioConfig = field(default_factory=BaseAudioConfig)
+    # phoneme settings
+    use_phonemes: bool = False
+    phoneme_language: str = None
+    compute_input_seq_cache: bool = False
+    text_cleaner: str = MISSING
+    enable_eos_bos_chars: bool = False
+    test_sentences_file: str = ""
+    phoneme_cache_path: str = None
+    # vocabulary parameters
+    characters: CharactersConfig = None
+    # training params
+    batch_group_size: int = 0
+    loss_masking: bool = None
+    # dataloading
+    min_seq_len: int = 1
+    max_seq_len: int = float("inf")
+    compute_f0: bool = False
+    use_noise_augment: bool = False
+    add_blank: bool = False
+    # dataset
+    datasets: List[BaseDatasetConfig] = field(default_factory=lambda: [BaseDatasetConfig()])

TTS/tts/configs/speedy_speech_config.py

@@ -0,0 +1,116 @@
+from dataclasses import dataclass, field
+
+from TTS.tts.configs.shared_configs import BaseTTSConfig
+
+
+@dataclass
+class SpeedySpeechConfig(BaseTTSConfig):
+    """Defines parameters for Speedy Speech (feed-forward encoder-decoder) based models.
+
+    Example:
+
+        >>> from TTS.tts.configs import SpeedySpeechConfig
+        >>> config = SpeedySpeechConfig()
+
+    Args:
+        model (str):
+            Model name used for selecting the right model at initialization. Defaults to `speedy_speech`.
+        positional_encoding (bool):
+            enable / disable positional encoding applied to the encoder output. Defaults to True.
+        hidden_channels (int):
+            Base number of hidden channels. Defines all the layers expect ones defined by the specific encoder or decoder
+            parameters. Defaults to 128.
+        encoder_type (str):
+            Type of the encoder used by the model. Look at `TTS.tts.layers.feed_forward.encoder` for more details.
+            Defaults to `residual_conv_bn`.
+        encoder_params (dict):
+            Parameters used to define the encoder network. Look at `TTS.tts.layers.feed_forward.encoder` for more details.
+            Defaults to `{"kernel_size": 4, "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1], "num_conv_blocks": 2, "num_res_blocks": 13}`
+        decoder_type (str):
+            Type of the decoder used by the model. Look at `TTS.tts.layers.feed_forward.decoder` for more details.
+            Defaults to `residual_conv_bn`.
+        decoder_params (dict):
+            Parameters used to define the decoder network. Look at `TTS.tts.layers.feed_forward.decoder` for more details.
+            Defaults to `{"kernel_size": 4, "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1], "num_conv_blocks": 2, "num_res_blocks": 17}`
+        hidden_channels_encoder (int):
+            Number of base hidden channels used by the encoder network. It defines the input and the output channel sizes,
+            and for some encoder types internal hidden channels sizes too. Defaults to 192.
+        hidden_channels_decoder (int):
+            Number of base hidden channels used by the decoder WaveNet network. Defaults to 192 as in the original work.
+        hidden_channels_duration_predictor (int):
+            Number of layer channels of the duration predictor network. Defaults to 256 as in the original work.
+        data_dep_init_steps (int):
+            Number of steps used for computing normalization parameters at the beginning of the training. GlowTTS uses
+            Activation Normalization that pre-computes normalization stats at the beginning and use the same values
+            for the rest. Defaults to 10.
+        use_speaker_embedding (bool):
+            enable / disable using speaker embeddings for multi-speaker models. If set True, the model is
+            in the multi-speaker mode. Defaults to False.
+        use_external_speaker_embedding_file (bool):
+            enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False.
+        external_speaker_embedding_file (str):
+            Path to the file including pre-computed speaker embeddings. Defaults to None.
+        noam_schedule (bool):
+            enable / disable the use of Noam LR scheduler. Defaults to False.
+        warmup_steps (int):
+            Number of warm-up steps for the Noam scheduler. Defaults 4000.
+        lr (float):
+            Initial learning rate. Defaults to `1e-3`.
+        wd (float):
+            Weight decay coefficient. Defaults to `1e-7`.
+        ssim_alpha (float):
+            Weight for the SSIM loss. If set <= 0, disables the SSIM loss. Defaults to 1.0.
+        huber_alpha (float):
+            Weight for the duration predictor's loss. Defaults to 1.0.
+        l1_alpha (float):
+            Weight for the L1 spectrogram loss. If set <= 0, disables the L1 loss. Defaults to 1.0.
+        min_seq_len (int):
+            Minimum input sequence length to be used at training.
+        max_seq_len (int):
+            Maximum input sequence length to be used at training. Larger values result in more VRAM usage.
+    """
+
+    model: str = "speedy_speech"
+    # model specific params
+    positional_encoding: bool = True
+    hidden_channels: int = 128
+    encoder_type: str = "residual_conv_bn"
+    encoder_params: dict = field(
+        default_factory=lambda: {
+            "kernel_size": 4,
+            "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1],
+            "num_conv_blocks": 2,
+            "num_res_blocks": 13,
+        }
+    )
+    decoder_type: str = "residual_conv_bn"
+    decoder_params: dict = field(
+        default_factory=lambda: {
+            "kernel_size": 4,
+            "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1],
+            "num_conv_blocks": 2,
+            "num_res_blocks": 17,
+        }
+    )
+
+    # multi-speaker settings
+    use_speaker_embedding: bool = False
+    use_external_speaker_embedding_file: bool = False
+    external_speaker_embedding_file: str = False
+
+    # optimizer parameters
+    noam_schedule: bool = False
+    warmup_steps: int = 4000
+    lr: float = 1e-4
+    wd: float = 1e-6
+    grad_clip: float = 5.0
+
+    # loss params
+    ssim_alpha: float = 1.0
+    huber_alpha: float = 1.0
+    l1_alpha: float = 1.0
+
+    # overrides
+    min_seq_len: int = 13
+    max_seq_len: int = 200
+    r: int = 1  # DO NOT CHANGE

TTS/tts/configs/speedy_speech_ljspeech.json

@@ -1,153 +0,0 @@
-{
-    "model": "speedy_speech",
-    "run_name": "speedy-speech-ljspeech",
-    "run_description": "speedy-speech model for LJSpeech dataset.",
-
-    // AUDIO PARAMETERS
-    "audio":{
-         // stft parameters
-         "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-         "win_length": 1024,      // stft window length in ms.
-         "hop_length": 256,       // stft window hop-lengh in ms.
-         "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-         "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-         // Audio processing parameters
-         "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate.
-         "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-         "ref_level_db": 20,     // reference level db, theoretically 20db is the sound of air.
-
-         // Silence trimming
-         "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true)
-         "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-         // Griffin-Lim
-         "power": 1.5,           // value to sharpen wav signals after GL algorithm.
-         "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
-
-         // MelSpectrogram parameters
-         "num_mels": 80,         // size of the mel spec frame.
-         "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-         "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-         "spec_gain": 1,
-
-         // Normalization parameters
-         "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-         "min_level_db": -100,   // lower bound for normalization
-         "symmetric_norm": true, // move normalization to range [-1, 1]
-         "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-         "clip_norm": true,      // clip normalized values into the range.
-         "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy"    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // VOCABULARY PARAMETERS
-    // if custom character set is not defined,
-    // default set in symbols.py is used
-    // "characters":{
-    //     "pad": "_",
-    //     "eos": "&",
-    //     "bos": "*",
-    //     "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZÇÃÀÁÂÊÉÍÓÔÕÚÛabcdefghijklmnopqrstuvwxyzçãàáâêéíóôõúû!(),-.:;? ",
-    //     "punctuations":"!'(),-.:;? ",
-    //     "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ'̃' "
-    // },
-
-    "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model.
-
-    // DISTRIBUTED TRAINING
-    "distributed":{
-        "backend": "nccl",
-        "url": "tcp:\/\/localhost:54321"
-    },
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // MODEL PARAMETERS
-    "positional_encoding": true,
-    "hidden_channels": 128,  // defined globally all the hidden channels of the model - 128 default
-    "encoder_type": "residual_conv_bn",
-    "encoder_params":{
-        "kernel_size": 4,
-        "dilations": [1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1],
-        "num_conv_blocks": 2,
-        "num_res_blocks": 13
-    },
-    "decoder_type": "residual_conv_bn",
-    "decoder_params":{
-        "kernel_size": 4,
-        "dilations": [1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1],
-        "num_conv_blocks": 2,
-        "num_res_blocks": 17
-    },
-
-    // TRAINING
-    "batch_size":64,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-    "eval_batch_size":32,
-    "r": 1,                 // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled.
-    "loss_masking": true,   // enable / disable loss masking against the sequence padding.
-
-    // LOSS PARAMETERS
-    "ssim_alpha": 1,
-    "l1_alpha": 1,
-    "huber_alpha": 1,
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": -1,       //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "noam_schedule": true,         // use noam warmup and lr schedule.
-    "grad_clip": 1.0,              // upper limit for gradients for clipping.
-    "epochs": 10000,               // total number of epochs to train.
-    "lr": 0.002,                    // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "warmup_steps": 4000,          // Noam decay steps to increase the learning rate from 0 to "lr"
-
-   // TENSORBOARD and LOGGING
-   "print_step": 25,       // Number of steps to log training on console.
-   "tb_plot_step": 100,    // Number of steps to plot TB training figures.
-   "print_eval": false,     // If True, it prints intermediate loss values in evalulation.
-   "save_step": 5000,      // Number of training steps expected to save traninpg stats and checkpoints.
-   "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-   "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-   "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-   "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.:set n
-   "mixed_precision": false,
-
-   // DATA LOADING
-   "text_cleaner": "english_cleaners",
-   "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars.
-   "num_loader_workers": 8,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-   "num_val_loader_workers": 8,    // number of evaluation data loader processes.
-   "batch_group_size": 4,  //Number of batches to shuffle after bucketing.
-   "min_seq_len": 2,       // DATASET-RELATED: minimum text length to use in training
-   "max_seq_len": 300,     // DATASET-RELATED: maximum text length
-   "compute_f0": false,     // compute f0 values in data-loader
-   "compute_input_seq_cache": false,  // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage.
-
-   // PATHS
-   "output_path": "/home/erogol/Models/ljspeech/",
-
-   // PHONEMES
-   "phoneme_cache_path": "/home/erogol/Models/ljspeech_phonemes/",  // phoneme computation is slow, therefore, it caches results in the given folder.
-   "use_phonemes": true,           // use phonemes instead of raw characters. It is suggested for better pronoun[ciation.
-   "phoneme_language": "en-us",     // depending on your target language, pick one from  https://github.com/bootphon/phonemizer#languages
-
-   // MULTI-SPEAKER and GST
-   "use_speaker_embedding": false,     // use speaker embedding to enable multi-speaker learning.
-   "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558
-   "external_speaker_embedding_file": "/home/erogol/Data/libritts/speakers.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558
-
-
-    // DATASETS
-    "datasets":   // List of datasets. They all merged and they get different s$
-    [
-        {
-            "name": "ljspeech",
-            "path": "/home/erogol/Data/LJSpeech-1.1/",
-            "meta_file_train": "metadata.csv",
-            "meta_file_val": null,
-            "meta_file_attn_mask": "/home/erogol/Data/LJSpeech-1.1/metadata_attn_mask.txt"  // created by bin/compute_attention_masks.py
-        }
-    ]
-}

TTS/tts/configs/tacotron2_config.py

@@ -0,0 +1,118 @@
+from dataclasses import dataclass
+
+from TTS.tts.configs.tacotron_config import TacotronConfig
+
+
+@dataclass
+class Tacotron2Config(TacotronConfig):
+    """Defines parameters for Tacotron2 based models.
+
+    Example:
+
+        >>> from TTS.tts.configs import Tacotron2Config
+        >>> config = Tacotron2Config()
+
+    Args:
+        model (str):
+            Model name used to select the right model class to initilize. Defaults to `Tacotron2`.
+        use_gst (bool):
+            enable / disable the use of Global Style Token modules. Defaults to False.
+        gst (GSTConfig):
+            Instance of `GSTConfig` class.
+        gst_style_input (str):
+            Path to the wav file used at inference to set the speech style through GST. If `GST` is enabled and
+            this is not defined, the model uses a zero vector as an input. Defaults to None.
+        r (int):
+            Number of output frames that the decoder computed per iteration. Larger values makes training and inference
+            faster but reduces the quality of the output frames. This needs to be tuned considering your own needs.
+            Defaults to 1.
+        gradual_trainin (List[List]):
+            Parameters for the gradual training schedule. It is in the form `[[a, b, c], [d ,e ,f] ..]` where `a` is
+            the step number to start using the rest of the values, `b` is the `r` value and `c` is the batch size.
+            If sets None, no gradual training is used. Defaults to None.
+        memory_size (int):
+            Defines the number of previous frames used by the Prenet. If set to < 0, then it uses only the last frame.
+            Defaults to -1.
+        prenet_type (str):
+            `original` or `bn`. `original` sets the default Prenet and `bn` uses Batch Normalization version of the
+            Prenet. Defaults to `original`.
+        prenet_dropout (bool):
+            enables / disables the use of dropout in the Prenet. Defaults to True.
+        prenet_dropout_at_inference (bool):
+            enable / disable the use of dropout in the Prenet at the inference time. Defaults to False.
+        stopnet (bool):
+            enable /disable the Stopnet that predicts the end of the decoder sequence. Defaults to True.
+        stopnet_pos_weight (float):
+            Weight that is applied to over-weight positive instances in the Stopnet loss. Use larger values with
+            datasets with longer sentences. Defaults to 10.
+        separate_stopnet (bool):
+            Use a distinct Stopnet which is trained separately from the rest of the model. Defaults to True.
+        attention_type (str):
+            attention type. Check ```TTS.tts.layers.attentions.init_attn```. Defaults to 'original'.
+        attention_heads (int):
+            Number of attention heads for GMM attention. Defaults to 5.
+        windowing (bool):
+            It especially useful at inference to keep attention alignment diagonal. Defaults to False.
+        use_forward_attn (bool):
+            It is only valid if ```attn_type``` is ```original```.  Defaults to False.
+        forward_attn_mask (bool):
+            enable/disable extra masking over forward attention. It is useful at inference to prevent
+            possible attention failures. Defaults to False.
+        transition_agent (bool):
+            enable/disable transition agent in forward attention. Defaults to False.
+        location_attn (bool):
+            enable/disable location sensitive attention as in the original Tacotron2 paper.
+            It is only valid if ```attn_type``` is ```original```. Defaults to True.
+        bidirectional_decoder (bool):
+            enable/disable bidirectional decoding. Defaults to False.
+        double_decoder_consistency (bool):
+            enable/disable double decoder consistency. Defaults to False.
+        ddc_r (int):
+            reduction rate used by the coarse decoder when `double_decoder_consistency` is in use. Set this
+            as a multiple of the `r` value. Defaults to 6.
+        use_speaker_embedding (bool):
+            enable / disable using speaker embeddings for multi-speaker models. If set True, the model is
+            in the multi-speaker mode. Defaults to False.
+        use_external_speaker_embedding_file (bool):
+            enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False.
+        external_speaker_embedding_file (str):
+            Path to the file including pre-computed speaker embeddings. Defaults to None.
+        noam_schedule (bool):
+            enable / disable the use of Noam LR scheduler. Defaults to False.
+        warmup_steps (int):
+            Number of warm-up steps for the Noam scheduler. Defaults 4000.
+        lr (float):
+            Initial learning rate. Defaults to `1e-4`.
+        wd (float):
+            Weight decay coefficient. Defaults to `1e-6`.
+        grad_clip (float):
+            Gradient clipping threshold. Defaults to `5`.
+        seq_len_notm (bool):
+            enable / disable the sequnce length normalization in the loss functions. If set True, loss of a sample
+            is divided by the sequence length. Defaults to False.
+        loss_masking (bool):
+            enable / disable masking the paddings of the samples in loss computation. Defaults to True.
+        decoder_loss_alpha (float):
+            Weight for the decoder loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        postnet_loss_alpha (float):
+            Weight for the postnet loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        postnet_diff_spec_alpha (float):
+            Weight for the postnet differential loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        decoder_diff_spec_alpha (float):
+            Weight for the decoder differential loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        decoder_ssim_alpha (float):
+            Weight for the decoder SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        postnet_ssim_alpha (float):
+            Weight for the postnet SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        ga_alpha (float):
+            Weight for the guided attention loss. If set less than or equal to zero, it disables the corresponding loss
+            function. Defaults to 5.
+    """
+
+    model: str = "tacotron2"

TTS/tts/configs/tacotron_config.py

@@ -0,0 +1,176 @@
+from dataclasses import dataclass
+from typing import List
+
+from TTS.tts.configs.shared_configs import BaseTTSConfig, GSTConfig
+
+
+@dataclass
+class TacotronConfig(BaseTTSConfig):
+    """Defines parameters for Tacotron based models.
+
+    Example:
+
+        >>> from TTS.tts.configs import TacotronConfig
+        >>> config = TacotronConfig()
+
+    Args:
+        model (str):
+            Model name used to select the right model class to initilize. Defaults to `Tacotron`.
+        use_gst (bool):
+            enable / disable the use of Global Style Token modules. Defaults to False.
+        gst (GSTConfig):
+            Instance of `GSTConfig` class.
+        gst_style_input (str):
+            Path to the wav file used at inference to set the speech style through GST. If `GST` is enabled and
+            this is not defined, the model uses a zero vector as an input. Defaults to None.
+        r (int):
+            Initial number of output frames that the decoder computed per iteration. Larger values makes training and inference
+            faster but reduces the quality of the output frames. This must be equal to the largest `r` value used in
+            `gradual_training` schedule. Defaults to 1.
+        gradual_training (List[List]):
+            Parameters for the gradual training schedule. It is in the form `[[a, b, c], [d ,e ,f] ..]` where `a` is
+            the step number to start using the rest of the values, `b` is the `r` value and `c` is the batch size.
+            If sets None, no gradual training is used. Defaults to None.
+        memory_size (int):
+            Defines the number of previous frames used by the Prenet. If set to < 0, then it uses only the last frame.
+            Defaults to -1.
+        prenet_type (str):
+            `original` or `bn`. `original` sets the default Prenet and `bn` uses Batch Normalization version of the
+            Prenet. Defaults to `original`.
+        prenet_dropout (bool):
+            enables / disables the use of dropout in the Prenet. Defaults to True.
+        prenet_dropout_at_inference (bool):
+            enable / disable the use of dropout in the Prenet at the inference time. Defaults to False.
+        stopnet (bool):
+            enable /disable the Stopnet that predicts the end of the decoder sequence. Defaults to True.
+        stopnet_pos_weight (float):
+            Weight that is applied to over-weight positive instances in the Stopnet loss. Use larger values with
+            datasets with longer sentences. Defaults to 10.
+        separate_stopnet (bool):
+            Use a distinct Stopnet which is trained separately from the rest of the model. Defaults to True.
+        attention_type (str):
+            attention type. Check ```TTS.tts.layers.attentions.init_attn```. Defaults to 'original'.
+        attention_heads (int):
+            Number of attention heads for GMM attention. Defaults to 5.
+        windowing (bool):
+            It especially useful at inference to keep attention alignment diagonal. Defaults to False.
+        use_forward_attn (bool):
+            It is only valid if ```attn_type``` is ```original```.  Defaults to False.
+        forward_attn_mask (bool):
+            enable/disable extra masking over forward attention. It is useful at inference to prevent
+            possible attention failures. Defaults to False.
+        transition_agent (bool):
+            enable/disable transition agent in forward attention. Defaults to False.
+        location_attn (bool):
+            enable/disable location sensitive attention as in the original Tacotron2 paper.
+            It is only valid if ```attn_type``` is ```original```. Defaults to True.
+        bidirectional_decoder (bool):
+            enable/disable bidirectional decoding. Defaults to False.
+        double_decoder_consistency (bool):
+            enable/disable double decoder consistency. Defaults to False.
+        ddc_r (int):
+            reduction rate used by the coarse decoder when `double_decoder_consistency` is in use. Set this
+            as a multiple of the `r` value. Defaults to 6.
+        use_speaker_embedding (bool):
+            enable / disable using speaker embeddings for multi-speaker models. If set True, the model is
+            in the multi-speaker mode. Defaults to False.
+        use_external_speaker_embedding_file (bool):
+            enable /disable using external speaker embeddings in place of the learned embeddings. Defaults to False.
+        external_speaker_embedding_file (str):
+            Path to the file including pre-computed speaker embeddings. Defaults to None.
+        noam_schedule (bool):
+            enable / disable the use of Noam LR scheduler. Defaults to False.
+        warmup_steps (int):
+            Number of warm-up steps for the Noam scheduler. Defaults 4000.
+        lr (float):
+            Initial learning rate. Defaults to `1e-4`.
+        wd (float):
+            Weight decay coefficient. Defaults to `1e-6`.
+        grad_clip (float):
+            Gradient clipping threshold. Defaults to `5`.
+        seq_len_notm (bool):
+            enable / disable the sequnce length normalization in the loss functions. If set True, loss of a sample
+            is divided by the sequence length. Defaults to False.
+        loss_masking (bool):
+            enable / disable masking the paddings of the samples in loss computation. Defaults to True.
+        decoder_loss_alpha (float):
+            Weight for the decoder loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        postnet_loss_alpha (float):
+            Weight for the postnet loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        postnet_diff_spec_alpha (float):
+            Weight for the postnet differential loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        decoder_diff_spec_alpha (float):
+            Weight for the decoder differential loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        decoder_ssim_alpha (float):
+            Weight for the decoder SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        postnet_ssim_alpha (float):
+            Weight for the postnet SSIM loss of the Tacotron model. If set less than or equal to zero, it disables the
+            corresponding loss function. Defaults to 0.25
+        ga_alpha (float):
+            Weight for the guided attention loss. If set less than or equal to zero, it disables the corresponding loss
+            function. Defaults to 5.
+    """
+
+    model: str = "tacotron"
+    use_gst: bool = False
+    gst: GSTConfig = None
+    gst_style_input: str = None
+    # model specific params
+    r: int = 2
+    gradual_training: List[List] = None
+    memory_size: int = -1
+    prenet_type: str = "original"
+    prenet_dropout: bool = True
+    prenet_dropout_at_inference: bool = False
+    stopnet: bool = True
+    separate_stopnet: bool = True
+    stopnet_pos_weight: float = 10.0
+
+    # attention layers
+    attention_type: str = "original"
+    attention_heads: int = None
+    attention_norm: str = "sigmoid"
+    windowing: bool = False
+    use_forward_attn: bool = False
+    forward_attn_mask: bool = False
+    transition_agent: bool = False
+    location_attn: bool = True
+
+    # advance methods
+    bidirectional_decoder: bool = False
+    double_decoder_consistency: bool = False
+    ddc_r: int = 6
+
+    # multi-speaker settings
+    use_speaker_embedding: bool = False
+    use_external_speaker_embedding_file: bool = False
+    external_speaker_embedding_file: str = False
+
+    # optimizer parameters
+    noam_schedule: bool = False
+    warmup_steps: int = 4000
+    lr: float = 1e-4
+    wd: float = 1e-6
+    grad_clip: float = 5.0
+    seq_len_norm: bool = False
+    loss_masking: bool = True
+
+    # loss params
+    decoder_loss_alpha: float = 0.25
+    postnet_loss_alpha: float = 0.25
+    postnet_diff_spec_alpha: float = 0.25
+    decoder_diff_spec_alpha: float = 0.25
+    decoder_ssim_alpha: float = 0.25
+    postnet_ssim_alpha: float = 0.25
+    ga_alpha: float = 5.0
+
+    def check_values(self):
+        if self.gradual_training:
+            assert (
+                self.gradual_training[0][1] == self.r
+            ), f"[!] the first scheduled gradual training `r` must be equal to the model's `r` value. {self.gradual_training[0][1]} vs {self.r}"

TTS/tts/datasets/TTSDataset.py

@@ -25,7 +25,7 @@ class MyDataset(Dataset):
         batch_group_size=0,
         min_seq_len=0,
         max_seq_len=float("inf"),
-        use_phonemes=True,
+        use_phonemes=False,
         phoneme_cache_path=None,
         phoneme_language="en-us",
         enable_eos_bos=False,

TTS/tts/datasets/preprocess.py

@@ -2,19 +2,41 @@ import os
 import re
 import sys
 import xml.etree.ElementTree as ET
+from collections import Counter
 from glob import glob
 from pathlib import Path
 from typing import List
 
+import numpy as np
 from tqdm import tqdm
 
-from TTS.tts.utils.generic_utils import split_dataset
-
 ####################
 # UTILITIES
 ####################
 
 
+def split_dataset(items):
+    speakers = [item[-1] for item in items]
+    is_multi_speaker = len(set(speakers)) > 1
+    eval_split_size = min(500, int(len(items) * 0.01))
+    assert eval_split_size > 0, " [!] You do not have enough samples to train. You need at least 100 samples."
+    np.random.seed(0)
+    np.random.shuffle(items)
+    if is_multi_speaker:
+        items_eval = []
+        speakers = [item[-1] for item in items]
+        speaker_counter = Counter(speakers)
+        while len(items_eval) < eval_split_size:
+            item_idx = np.random.randint(0, len(items))
+            speaker_to_be_removed = items[item_idx][-1]
+            if speaker_counter[speaker_to_be_removed] > 1:
+                items_eval.append(items[item_idx])
+                speaker_counter[speaker_to_be_removed] -= 1
+                del items[item_idx]
+        return items_eval, items
+    return items[:eval_split_size], items[eval_split_size:]
+
+
 def load_meta_data(datasets, eval_split=True):
     meta_data_train_all = []
     meta_data_eval_all = [] if eval_split else None
@@ -30,19 +52,19 @@ def load_meta_data(datasets, eval_split=True):
         print(f" | > Found {len(meta_data_train)} files in {Path(root_path).resolve()}")
         # load evaluation split if set
         if eval_split:
-            if meta_file_val is None:
-                meta_data_eval, meta_data_train = split_dataset(meta_data_train)
-            else:
+            if meta_file_val:
                 meta_data_eval = preprocessor(root_path, meta_file_val)
+            else:
+                meta_data_eval, meta_data_train = split_dataset(meta_data_train)
             meta_data_eval_all += meta_data_eval
         meta_data_train_all += meta_data_train
         # load attention masks for duration predictor training
-        if "meta_file_attn_mask" in dataset and dataset["meta_file_attn_mask"] is not None:
+        if dataset.meta_file_attn_mask:
             meta_data = dict(load_attention_mask_meta_data(dataset["meta_file_attn_mask"]))
             for idx, ins in enumerate(meta_data_train_all):
                 attn_file = meta_data[ins[1]].strip()
                 meta_data_train_all[idx].append(attn_file)
-            if meta_data_eval_all is not None:
+            if meta_data_eval_all:
                 for idx, ins in enumerate(meta_data_eval_all):
                     attn_file = meta_data[ins[1]].strip()
                     meta_data_eval_all[idx].append(attn_file)

TTS/tts/layers/tacotron/tacotron.py

@@ -463,7 +463,6 @@ class Decoder(nn.Module):
         stop_tokens = []
         t = 0
         self._init_states(inputs)
-        self.attention.init_win_idx()
         self.attention.init_states(inputs)
         while True:
             if t > 0:

TTS/tts/layers/tacotron/tacotron2.py

@@ -375,7 +375,6 @@ class Decoder(nn.Module):
         else:
             self._init_states(inputs, mask=None, keep_states=True)
 
-        self.attention.init_win_idx()
         self.attention.init_states(inputs)
         outputs, stop_tokens, alignments, t = [], [], [], 0
         while True:

TTS/tts/models/glow_tts.py

@@ -179,6 +179,81 @@ class GlowTTS(nn.Module):
         attn = attn.squeeze(1).permute(0, 2, 1)
         return z, logdet, y_mean, y_log_scale, attn, o_dur_log, o_attn_dur
 
+    @torch.no_grad()
+    def inference_with_MAS(self, x, x_lengths, y=None, y_lengths=None, attn=None, g=None):
+        """
+        It's similar to the teacher forcing in Tacotron.
+        It was proposed in: https://arxiv.org/abs/2104.05557
+        Shapes:
+            x: [B, T]
+            x_lenghts: B
+            y: [B, C, T]
+            y_lengths: B
+            g: [B, C] or B
+        """
+        y_max_length = y.size(2)
+        # norm speaker embeddings
+        if g is not None:
+            if self.external_speaker_embedding_dim:
+                g = F.normalize(g).unsqueeze(-1)
+            else:
+                g = F.normalize(self.emb_g(g)).unsqueeze(-1)  # [b, h, 1]
+
+        # embedding pass
+        o_mean, o_log_scale, o_dur_log, x_mask = self.encoder(x, x_lengths, g=g)
+        # drop redisual frames wrt num_squeeze and set y_lengths.
+        y, y_lengths, y_max_length, attn = self.preprocess(y, y_lengths, y_max_length, None)
+        # create masks
+        y_mask = torch.unsqueeze(sequence_mask(y_lengths, y_max_length), 1).to(x_mask.dtype)
+        attn_mask = torch.unsqueeze(x_mask, -1) * torch.unsqueeze(y_mask, 2)
+        # decoder pass
+        z, logdet = self.decoder(y, y_mask, g=g, reverse=False)
+        # find the alignment path between z and encoder output
+        o_scale = torch.exp(-2 * o_log_scale)
+        logp1 = torch.sum(-0.5 * math.log(2 * math.pi) - o_log_scale, [1]).unsqueeze(-1)  # [b, t, 1]
+        logp2 = torch.matmul(o_scale.transpose(1, 2), -0.5 * (z ** 2))  # [b, t, d] x [b, d, t'] = [b, t, t']
+        logp3 = torch.matmul((o_mean * o_scale).transpose(1, 2), z)  # [b, t, d] x [b, d, t'] = [b, t, t']
+        logp4 = torch.sum(-0.5 * (o_mean ** 2) * o_scale, [1]).unsqueeze(-1)  # [b, t, 1]
+        logp = logp1 + logp2 + logp3 + logp4  # [b, t, t']
+        attn = maximum_path(logp, attn_mask.squeeze(1)).unsqueeze(1).detach()
+
+        y_mean, y_log_scale, o_attn_dur = self.compute_outputs(attn, o_mean, o_log_scale, x_mask)
+        attn = attn.squeeze(1).permute(0, 2, 1)
+
+        # get predited aligned distribution
+        z = y_mean * y_mask
+
+        # reverse the decoder and predict using the aligned distribution
+        y, logdet = self.decoder(z, y_mask, g=g, reverse=True)
+
+        return y, logdet, y_mean, y_log_scale, attn, o_dur_log, o_attn_dur
+
+    @torch.no_grad()
+    def decoder_inference(self, y, y_lengths=None, g=None):
+        """
+        Shapes:
+            y: [B, C, T]
+            y_lengths: B
+            g: [B, C] or B
+        """
+        y_max_length = y.size(2)
+        # norm speaker embeddings
+        if g is not None:
+            if self.external_speaker_embedding_dim:
+                g = F.normalize(g).unsqueeze(-1)
+            else:
+                g = F.normalize(self.emb_g(g)).unsqueeze(-1)  # [b, h, 1]
+
+        y_mask = torch.unsqueeze(sequence_mask(y_lengths, y_max_length), 1).to(y.dtype)
+
+        # decoder pass
+        z, logdet = self.decoder(y, y_mask, g=g, reverse=False)
+
+        # reverse decoder and predict
+        y, logdet = self.decoder(z, y_mask, g=g, reverse=True)
+
+        return y, logdet
+
     @torch.no_grad()
     def inference(self, x, x_lengths, g=None):
         if g is not None:

TTS/tts/models/tacotron.py

@@ -41,11 +41,8 @@ class Tacotron(TacotronAbstract):
         encoder_in_features (int, optional): input channels for the encoder. Defaults to 512.
         decoder_in_features (int, optional): input channels for the decoder. Defaults to 512.
         speaker_embedding_dim (int, optional): external speaker conditioning vector channels. Defaults to None.
-        gst (bool, optional): enable/disable global style token learning. Defaults to False.
-        gst_embedding_dim (int, optional): size of channels for GST vectors. Defaults to 512.
-        gst_num_heads (int, optional): number of attention heads for GST. Defaults to 4.
-        gst_style_tokens (int, optional): number of GST tokens. Defaults to 10.
-        gst_use_speaker_embedding (bool, optional): enable/disable inputing speaker embedding to GST. Defaults to False.
+        use_gst (bool, optional): enable/disable Global style token module.
+        gst (Coqpit, optional): Coqpit to initialize the GST module. If `None`, GST is disabled. Defaults to None.
         memory_size (int, optional): size of the history queue fed to the prenet. Model feeds the last ```memory_size```
             output frames to the prenet.
     """
@@ -75,12 +72,9 @@ class Tacotron(TacotronAbstract):
         encoder_in_features=256,
         decoder_in_features=256,
         speaker_embedding_dim=None,
-        gst=False,
-        gst_embedding_dim=256,
-        gst_num_heads=4,
-        gst_style_tokens=10,
+        use_gst=False,
+        gst=None,
         memory_size=5,
-        gst_use_speaker_embedding=False,
     ):
         super().__init__(
             num_chars,
@@ -106,11 +100,8 @@ class Tacotron(TacotronAbstract):
             encoder_in_features,
             decoder_in_features,
             speaker_embedding_dim,
+            use_gst,
             gst,
-            gst_embedding_dim,
-            gst_num_heads,
-            gst_style_tokens,
-            gst_use_speaker_embedding,
         )
 
         # speaker embedding layers
@@ -154,15 +145,13 @@ class Tacotron(TacotronAbstract):
         self.decoder.prenet.dropout_at_inference = prenet_dropout_at_inference
 
         # global style token layers
-        if self.gst:
+        if self.gst and self.use_gst:
             self.gst_layer = GST(
-                num_mel=80,
-                num_heads=gst_num_heads,
-                num_style_tokens=gst_style_tokens,
-                gst_embedding_dim=self.gst_embedding_dim,
-                speaker_embedding_dim=speaker_embedding_dim
-                if self.embeddings_per_sample and self.gst_use_speaker_embedding
-                else None,
+                num_mel=decoder_output_dim,
+                speaker_embedding_dim=speaker_embedding_dim,
+                num_heads=gst.gst_num_heads,
+                num_style_tokens=gst.gst_num_style_tokens,
+                gst_embedding_dim=gst.gst_embedding_dim,
             )
         # backward pass decoder
         if self.bidirectional_decoder:
@@ -205,11 +194,9 @@ class Tacotron(TacotronAbstract):
         # sequence masking
         encoder_outputs = encoder_outputs * input_mask.unsqueeze(2).expand_as(encoder_outputs)
         # global style token
-        if self.gst:
+        if self.gst and self.use_gst:
             # B x gst_dim
-            encoder_outputs = self.compute_gst(
-                encoder_outputs, mel_specs, speaker_embeddings if self.gst_use_speaker_embedding else None
-            )
+            encoder_outputs = self.compute_gst(encoder_outputs, mel_specs, speaker_embeddings)
         # speaker embedding
         if self.num_speakers > 1:
             if not self.embeddings_per_sample:
@@ -263,11 +250,9 @@ class Tacotron(TacotronAbstract):
     def inference(self, characters, speaker_ids=None, style_mel=None, speaker_embeddings=None):
         inputs = self.embedding(characters)
         encoder_outputs = self.encoder(inputs)
-        if self.gst:
+        if self.gst and self.use_gst:
             # B x gst_dim
-            encoder_outputs = self.compute_gst(
-                encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None
-            )
+            encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings)
         if self.num_speakers > 1:
             if not self.embeddings_per_sample:
                 # B x 1 x speaker_embed_dim

TTS/tts/models/tacotron2.py

@@ -41,11 +41,8 @@ class Tacotron2(TacotronAbstract):
         encoder_in_features (int, optional): input channels for the encoder. Defaults to 512.
         decoder_in_features (int, optional): input channels for the decoder. Defaults to 512.
         speaker_embedding_dim (int, optional): external speaker conditioning vector channels. Defaults to None.
-        gst (bool, optional): enable/disable global style token learning. Defaults to False.
-        gst_embedding_dim (int, optional): size of channels for GST vectors. Defaults to 512.
-        gst_num_heads (int, optional): number of attention heads for GST. Defaults to 4.
-        gst_style_tokens (int, optional): number of GST tokens. Defaults to 10.
-        gst_use_speaker_embedding (bool, optional): enable/disable inputing speaker embedding to GST. Defaults to False.
+        use_gst (bool, optional): enable/disable Global style token module.
+        gst (Coqpit, optional): Coqpit to initialize the GST module. If `None`, GST is disabled. Defaults to None.
     """
 
     def __init__(
@@ -73,11 +70,8 @@ class Tacotron2(TacotronAbstract):
         encoder_in_features=512,
         decoder_in_features=512,
         speaker_embedding_dim=None,
-        gst=False,
-        gst_embedding_dim=512,
-        gst_num_heads=4,
-        gst_style_tokens=10,
-        gst_use_speaker_embedding=False,
+        use_gst=False,
+        gst=None,
     ):
         super().__init__(
             num_chars,
@@ -103,11 +97,8 @@ class Tacotron2(TacotronAbstract):
             encoder_in_features,
             decoder_in_features,
             speaker_embedding_dim,
+            use_gst,
             gst,
-            gst_embedding_dim,
-            gst_num_heads,
-            gst_style_tokens,
-            gst_use_speaker_embedding,
         )
 
         # speaker embedding layer
@@ -148,16 +139,15 @@ class Tacotron2(TacotronAbstract):
         self.decoder.prenet.dropout_at_inference = prenet_dropout_at_inference
 
         # global style token layers
-        if self.gst:
+        if self.gst and use_gst:
             self.gst_layer = GST(
-                num_mel=80,
-                num_heads=self.gst_num_heads,
-                num_style_tokens=self.gst_style_tokens,
-                gst_embedding_dim=self.gst_embedding_dim,
-                speaker_embedding_dim=speaker_embedding_dim
-                if self.embeddings_per_sample and self.gst_use_speaker_embedding
-                else None,
+                num_mel=decoder_output_dim,
+                speaker_embedding_dim=speaker_embedding_dim,
+                num_heads=gst.gst_num_heads,
+                num_style_tokens=gst.gst_num_style_tokens,
+                gst_embedding_dim=gst.gst_embedding_dim,
             )
+
         # backward pass decoder
         if self.bidirectional_decoder:
             self._init_backward_decoder()
@@ -203,11 +193,9 @@ class Tacotron2(TacotronAbstract):
         embedded_inputs = self.embedding(text).transpose(1, 2)
         # B x T_in_max x D_en
         encoder_outputs = self.encoder(embedded_inputs, text_lengths)
-        if self.gst:
+        if self.gst and self.use_gst:
             # B x gst_dim
-            encoder_outputs = self.compute_gst(
-                encoder_outputs, mel_specs, speaker_embeddings if self.gst_use_speaker_embedding else None
-            )
+            encoder_outputs = self.compute_gst(encoder_outputs, mel_specs, speaker_embeddings)
         if self.num_speakers > 1:
             if not self.embeddings_per_sample:
                 # B x 1 x speaker_embed_dim
@@ -261,11 +249,9 @@ class Tacotron2(TacotronAbstract):
         embedded_inputs = self.embedding(text).transpose(1, 2)
         encoder_outputs = self.encoder.inference(embedded_inputs)
 
-        if self.gst:
+        if self.gst and self.use_gst:
             # B x gst_dim
-            encoder_outputs = self.compute_gst(
-                encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None
-            )
+            encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings)
         if self.num_speakers > 1:
             if not self.embeddings_per_sample:
                 speaker_embeddings = self.speaker_embedding(speaker_ids)[:, None]
@@ -286,9 +272,7 @@ class Tacotron2(TacotronAbstract):
 
         if self.gst:
             # B x gst_dim
-            encoder_outputs = self.compute_gst(
-                encoder_outputs, style_mel, speaker_embeddings if self.gst_use_speaker_embedding else None
-            )
+            encoder_outputs = self.compute_gst(encoder_outputs, style_mel, speaker_embeddings)
 
         if self.num_speakers > 1:
             if not self.embeddings_per_sample:

TTS/tts/models/tacotron_abstract.py

@@ -33,11 +33,8 @@ class TacotronAbstract(ABC, nn.Module):
         encoder_in_features=512,
         decoder_in_features=512,
         speaker_embedding_dim=None,
-        gst=False,
-        gst_embedding_dim=512,
-        gst_num_heads=4,
-        gst_style_tokens=10,
-        gst_use_speaker_embedding=False,
+        use_gst=False,
+        gst=None,
     ):
         """Abstract Tacotron class"""
         super().__init__()
@@ -45,11 +42,8 @@ class TacotronAbstract(ABC, nn.Module):
         self.r = r
         self.decoder_output_dim = decoder_output_dim
         self.postnet_output_dim = postnet_output_dim
+        self.use_gst = use_gst
         self.gst = gst
-        self.gst_embedding_dim = gst_embedding_dim
-        self.gst_num_heads = gst_num_heads
-        self.gst_style_tokens = gst_style_tokens
-        self.gst_use_speaker_embedding = gst_use_speaker_embedding
         self.num_speakers = num_speakers
         self.bidirectional_decoder = bidirectional_decoder
         self.double_decoder_consistency = double_decoder_consistency
@@ -85,8 +79,8 @@ class TacotronAbstract(ABC, nn.Module):
             self.embeddings_per_sample = True
 
         # global style token
-        if self.gst:
-            self.decoder_in_features += gst_embedding_dim  # add gst embedding dim
+        if self.gst and use_gst:
+            self.decoder_in_features += self.gst.gst_embedding_dim  # add gst embedding dim
             self.gst_layer = None
 
         # model states
@@ -194,18 +188,18 @@ class TacotronAbstract(ABC, nn.Module):
         """Compute global style token"""
         device = inputs.device
         if isinstance(style_input, dict):
-            query = torch.zeros(1, 1, self.gst_embedding_dim // 2).to(device)
+            query = torch.zeros(1, 1, self.gst.gst_embedding_dim // 2).to(device)
             if speaker_embedding is not None:
                 query = torch.cat([query, speaker_embedding.reshape(1, 1, -1)], dim=-1)
 
             _GST = torch.tanh(self.gst_layer.style_token_layer.style_tokens)
-            gst_outputs = torch.zeros(1, 1, self.gst_embedding_dim).to(device)
+            gst_outputs = torch.zeros(1, 1, self.gst.gst_embedding_dim).to(device)
             for k_token, v_amplifier in style_input.items():
                 key = _GST[int(k_token)].unsqueeze(0).expand(1, -1, -1)
                 gst_outputs_att = self.gst_layer.style_token_layer.attention(query, key)
                 gst_outputs = gst_outputs + gst_outputs_att * v_amplifier
         elif style_input is None:
-            gst_outputs = torch.zeros(1, 1, self.gst_embedding_dim).to(device)
+            gst_outputs = torch.zeros(1, 1, self.gst.gst_embedding_dim).to(device)
         else:
             gst_outputs = self.gst_layer(style_input, speaker_embedding)  # pylint: disable=not-callable
         inputs = self._concat_speaker_embedding(inputs, gst_outputs)

TTS/tts/utils/generic_utils.py

@@ -1,33 +1,6 @@
-import importlib
-import re
-from collections import Counter
-
-import numpy as np
 import torch
 
-from TTS.utils.generic_utils import check_argument
-
-
-def split_dataset(items):
-    speakers = [item[-1] for item in items]
-    is_multi_speaker = len(set(speakers)) > 1
-    eval_split_size = min(500, int(len(items) * 0.01))
-    assert eval_split_size > 0, " [!] You do not have enough samples to train. You need at least 100 samples."
-    np.random.seed(0)
-    np.random.shuffle(items)
-    if is_multi_speaker:
-        items_eval = []
-        speakers = [item[-1] for item in items]
-        speaker_counter = Counter(speakers)
-        while len(items_eval) < eval_split_size:
-            item_idx = np.random.randint(0, len(items))
-            speaker_to_be_removed = items[item_idx][-1]
-            if speaker_counter[speaker_to_be_removed] > 1:
-                items_eval.append(items[item_idx])
-                speaker_counter[speaker_to_be_removed] -= 1
-                del items[item_idx]
-        return items_eval, items
-    return items[:eval_split_size], items[eval_split_size:]
+from TTS.utils.generic_utils import find_module
 
 
 # from https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1
@@ -39,17 +12,9 @@ def sequence_mask(sequence_length, max_len=None):
     return seq_range.unsqueeze(0) < sequence_length.unsqueeze(1)
 
 
-def to_camel(text):
-    text = text.capitalize()
-    text = re.sub(r"(?!^)_([a-zA-Z])", lambda m: m.group(1).upper(), text)
-    text = text.replace("Tts", "TTS")
-    return text
-
-
 def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None):
     print(" > Using model: {}".format(c.model))
-    MyModel = importlib.import_module("TTS.tts.models." + c.model.lower())
-    MyModel = getattr(MyModel, to_camel(c.model))
+    MyModel = find_module("TTS.tts.models", c.model.lower())
     if c.model.lower() in "tacotron":
         model = MyModel(
             num_chars=num_chars + getattr(c, "add_blank", False),
@@ -57,18 +22,15 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None):
             r=c.r,
             postnet_output_dim=int(c.audio["fft_size"] / 2 + 1),
             decoder_output_dim=c.audio["num_mels"],
-            gst=c.use_gst,
-            gst_embedding_dim=c.gst["gst_embedding_dim"],
-            gst_num_heads=c.gst["gst_num_heads"],
-            gst_style_tokens=c.gst["gst_style_tokens"],
-            gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"],
+            use_gst=c.use_gst,
+            gst=c.gst,
             memory_size=c.memory_size,
             attn_type=c.attention_type,
             attn_win=c.windowing,
             attn_norm=c.attention_norm,
             prenet_type=c.prenet_type,
             prenet_dropout=c.prenet_dropout,
-            prenet_dropout_at_inference=c.prenet_dropout_at_inference if "prenet_dropout_at_inference" in c else False,
+            prenet_dropout_at_inference=c.prenet_dropout_at_inference,
             forward_attn=c.use_forward_attn,
             trans_agent=c.transition_agent,
             forward_attn_mask=c.forward_attn_mask,
@@ -87,17 +49,14 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None):
             r=c.r,
             postnet_output_dim=c.audio["num_mels"],
             decoder_output_dim=c.audio["num_mels"],
-            gst=c.use_gst,
-            gst_embedding_dim=c.gst["gst_embedding_dim"],
-            gst_num_heads=c.gst["gst_num_heads"],
-            gst_style_tokens=c.gst["gst_style_tokens"],
-            gst_use_speaker_embedding=c.gst["gst_use_speaker_embedding"],
+            use_gst=c.use_gst,
+            gst=c.gst,
             attn_type=c.attention_type,
             attn_win=c.windowing,
             attn_norm=c.attention_norm,
             prenet_type=c.prenet_type,
             prenet_dropout=c.prenet_dropout,
-            prenet_dropout_at_inference=c.prenet_dropout_at_inference if "prenet_dropout_at_inference" in c else False,
+            prenet_dropout_at_inference=c.prenet_dropout_at_inference,
             forward_attn=c.use_forward_attn,
             trans_agent=c.transition_agent,
             forward_attn_mask=c.forward_attn_mask,
@@ -119,7 +78,7 @@ def setup_model(num_chars, num_speakers, c, speaker_embedding_dim=None):
             encoder_type=c.encoder_type,
             encoder_params=c.encoder_params,
             use_encoder_prenet=c["use_encoder_prenet"],
-            inference_noise_scale=c.get("inference_noise_scale", 0.33),
+            inference_noise_scale=c.inference_noise_scale,
             num_flow_blocks_dec=12,
             kernel_size_dec=5,
             dilation_rate=1,
@@ -164,202 +123,156 @@ def is_tacotron(c):
     return "tacotron" in c["model"].lower()
 
 
-def check_config_tts(c):
-    check_argument(
-        "model",
-        c,
-        enum_list=["tacotron", "tacotron2", "glow_tts", "speedy_speech", "align_tts"],
-        restricted=True,
-        val_type=str,
-    )
-    check_argument("run_name", c, restricted=True, val_type=str)
-    check_argument("run_description", c, val_type=str)
+# def check_config_tts(c):
+#     check_argument('model', c, enum_list=['tacotron', 'tacotron2', 'glow_tts', 'speedy_speech', 'align_tts'], restricted=True, val_type=str)
+#     check_argument('run_name', c, restricted=True, val_type=str)
+#     check_argument('run_description', c, val_type=str)
 
-    # AUDIO
-    check_argument("audio", c, restricted=True, val_type=dict)
+#     # AUDIO
+#     # check_argument('audio', c, restricted=True, val_type=dict)
 
-    # audio processing parameters
-    check_argument("num_mels", c["audio"], restricted=True, val_type=int, min_val=10, max_val=2056)
-    check_argument("fft_size", c["audio"], restricted=True, val_type=int, min_val=128, max_val=4058)
-    check_argument("sample_rate", c["audio"], restricted=True, val_type=int, min_val=512, max_val=100000)
-    check_argument(
-        "frame_length_ms",
-        c["audio"],
-        restricted=True,
-        val_type=float,
-        min_val=10,
-        max_val=1000,
-        alternative="win_length",
-    )
-    check_argument(
-        "frame_shift_ms", c["audio"], restricted=True, val_type=float, min_val=1, max_val=1000, alternative="hop_length"
-    )
-    check_argument("preemphasis", c["audio"], restricted=True, val_type=float, min_val=0, max_val=1)
-    check_argument("min_level_db", c["audio"], restricted=True, val_type=int, min_val=-1000, max_val=10)
-    check_argument("ref_level_db", c["audio"], restricted=True, val_type=int, min_val=0, max_val=1000)
-    check_argument("power", c["audio"], restricted=True, val_type=float, min_val=1, max_val=5)
-    check_argument("griffin_lim_iters", c["audio"], restricted=True, val_type=int, min_val=10, max_val=1000)
+#     # audio processing parameters
+#     # check_argument('num_mels', c['audio'], restricted=True, val_type=int, min_val=10, max_val=2056)
+#     # check_argument('fft_size', c['audio'], restricted=True, val_type=int, min_val=128, max_val=4058)
+#     # check_argument('sample_rate', c['audio'], restricted=True, val_type=int, min_val=512, max_val=100000)
+#     # check_argument('frame_length_ms', c['audio'], restricted=True, val_type=float, min_val=10, max_val=1000, alternative='win_length')
+#     # check_argument('frame_shift_ms', c['audio'], restricted=True, val_type=float, min_val=1, max_val=1000, alternative='hop_length')
+#     # check_argument('preemphasis', c['audio'], restricted=True, val_type=float, min_val=0, max_val=1)
+#     # check_argument('min_level_db', c['audio'], restricted=True, val_type=int, min_val=-1000, max_val=10)
+#     # check_argument('ref_level_db', c['audio'], restricted=True, val_type=int, min_val=0, max_val=1000)
+#     # check_argument('power', c['audio'], restricted=True, val_type=float, min_val=1, max_val=5)
+#     # check_argument('griffin_lim_iters', c['audio'], restricted=True, val_type=int, min_val=10, max_val=1000)
 
-    # vocabulary parameters
-    check_argument("characters", c, restricted=False, val_type=dict)
-    check_argument(
-        "pad", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str
-    )
-    check_argument(
-        "eos", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str
-    )
-    check_argument(
-        "bos", c["characters"] if "characters" in c.keys() else {}, restricted="characters" in c.keys(), val_type=str
-    )
-    check_argument(
-        "characters",
-        c["characters"] if "characters" in c.keys() else {},
-        restricted="characters" in c.keys(),
-        val_type=str,
-    )
-    check_argument(
-        "phonemes",
-        c["characters"] if "characters" in c.keys() else {},
-        restricted="characters" in c.keys() and c["use_phonemes"],
-        val_type=str,
-    )
-    check_argument(
-        "punctuations",
-        c["characters"] if "characters" in c.keys() else {},
-        restricted="characters" in c.keys(),
-        val_type=str,
-    )
+#     # vocabulary parameters
+#     check_argument('characters', c, restricted=False, val_type=dict)
+#     check_argument('pad', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str)
+#     check_argument('eos', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str)
+#     check_argument('bos', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str)
+#     check_argument('characters', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str)
+#     check_argument('phonemes', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys() and c['use_phonemes'], val_type=str)
+#     check_argument('punctuations', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str)
 
-    # normalization parameters
-    check_argument("signal_norm", c["audio"], restricted=True, val_type=bool)
-    check_argument("symmetric_norm", c["audio"], restricted=True, val_type=bool)
-    check_argument("max_norm", c["audio"], restricted=True, val_type=float, min_val=0.1, max_val=1000)
-    check_argument("clip_norm", c["audio"], restricted=True, val_type=bool)
-    check_argument("mel_fmin", c["audio"], restricted=True, val_type=float, min_val=0.0, max_val=1000)
-    check_argument("mel_fmax", c["audio"], restricted=True, val_type=float, min_val=500.0)
-    check_argument("spec_gain", c["audio"], restricted=True, val_type=[int, float], min_val=1, max_val=100)
-    check_argument("do_trim_silence", c["audio"], restricted=True, val_type=bool)
-    check_argument("trim_db", c["audio"], restricted=True, val_type=int)
+#     # normalization parameters
+#     # check_argument('signal_norm', c['audio'], restricted=True, val_type=bool)
+#     # check_argument('symmetric_norm', c['audio'], restricted=True, val_type=bool)
+#     # check_argument('max_norm', c['audio'], restricted=True, val_type=float, min_val=0.1, max_val=1000)
+#     # check_argument('clip_norm', c['audio'], restricted=True, val_type=bool)
+#     # check_argument('mel_fmin', c['audio'], restricted=True, val_type=float, min_val=0.0, max_val=1000)
+#     # check_argument('mel_fmax', c['audio'], restricted=True, val_type=float, min_val=500.0)
+#     # check_argument('spec_gain', c['audio'], restricted=True, val_type=[int, float], min_val=1, max_val=100)
+#     # check_argument('do_trim_silence', c['audio'], restricted=True, val_type=bool)
+#     # check_argument('trim_db', c['audio'], restricted=True, val_type=int)
 
-    # training parameters
-    check_argument("batch_size", c, restricted=True, val_type=int, min_val=1)
-    check_argument("eval_batch_size", c, restricted=True, val_type=int, min_val=1)
-    check_argument("r", c, restricted=True, val_type=int, min_val=1)
-    check_argument("gradual_training", c, restricted=False, val_type=list)
-    check_argument("mixed_precision", c, restricted=False, val_type=bool)
-    # check_argument('grad_accum', c, restricted=True, val_type=int, min_val=1, max_val=100)
+#     # training parameters
+#     # check_argument('batch_size', c, restricted=True, val_type=int, min_val=1)
+#     # check_argument('eval_batch_size', c, restricted=True, val_type=int, min_val=1)
+#     # check_argument('r', c, restricted=True, val_type=int, min_val=1)
+#     # check_argument('gradual_training', c, restricted=False, val_type=list)
+#     # check_argument('mixed_precision', c, restricted=False, val_type=bool)
+#     # check_argument('grad_accum', c, restricted=True, val_type=int, min_val=1, max_val=100)
 
-    # loss parameters
-    check_argument("loss_masking", c, restricted=True, val_type=bool)
-    if c["model"].lower() in ["tacotron", "tacotron2"]:
-        check_argument("decoder_loss_alpha", c, restricted=True, val_type=float, min_val=0)
-        check_argument("postnet_loss_alpha", c, restricted=True, val_type=float, min_val=0)
-        check_argument("postnet_diff_spec_alpha", c, restricted=True, val_type=float, min_val=0)
-        check_argument("decoder_diff_spec_alpha", c, restricted=True, val_type=float, min_val=0)
-        check_argument("decoder_ssim_alpha", c, restricted=True, val_type=float, min_val=0)
-        check_argument("postnet_ssim_alpha", c, restricted=True, val_type=float, min_val=0)
-        check_argument("ga_alpha", c, restricted=True, val_type=float, min_val=0)
-    if c["model"].lower in ["speedy_speech", "align_tts"]:
-        check_argument("ssim_alpha", c, restricted=True, val_type=float, min_val=0)
-        check_argument("l1_alpha", c, restricted=True, val_type=float, min_val=0)
-        check_argument("huber_alpha", c, restricted=True, val_type=float, min_val=0)
+#     # loss parameters
+#     # check_argument('loss_masking', c, restricted=True, val_type=bool)
+#     # if c['model'].lower() in ['tacotron', 'tacotron2']:
+#     #     check_argument('decoder_loss_alpha', c, restricted=True, val_type=float, min_val=0)
+#     #     check_argument('postnet_loss_alpha', c, restricted=True, val_type=float, min_val=0)
+#     #     check_argument('postnet_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0)
+#     #     check_argument('decoder_diff_spec_alpha', c, restricted=True, val_type=float, min_val=0)
+#     #     check_argument('decoder_ssim_alpha', c, restricted=True, val_type=float, min_val=0)
+#     #     check_argument('postnet_ssim_alpha', c, restricted=True, val_type=float, min_val=0)
+#     #     check_argument('ga_alpha', c, restricted=True, val_type=float, min_val=0)
+#     if c['model'].lower in ["speedy_speech", "align_tts"]:
+#         check_argument('ssim_alpha', c, restricted=True, val_type=float, min_val=0)
+#         check_argument('l1_alpha', c, restricted=True, val_type=float, min_val=0)
+#         check_argument('huber_alpha', c, restricted=True, val_type=float, min_val=0)
 
-    # validation parameters
-    check_argument("run_eval", c, restricted=True, val_type=bool)
-    check_argument("test_delay_epochs", c, restricted=True, val_type=int, min_val=0)
-    check_argument("test_sentences_file", c, restricted=False, val_type=str)
+#     # validation parameters
+#     # check_argument('run_eval', c, restricted=True, val_type=bool)
+#     # check_argument('test_delay_epochs', c, restricted=True, val_type=int, min_val=0)
+#     # check_argument('test_sentences_file', c, restricted=False, val_type=str)
 
-    # optimizer
-    check_argument("noam_schedule", c, restricted=False, val_type=bool)
-    check_argument("grad_clip", c, restricted=True, val_type=float, min_val=0.0)
-    check_argument("epochs", c, restricted=True, val_type=int, min_val=1)
-    check_argument("lr", c, restricted=True, val_type=float, min_val=0)
-    check_argument("wd", c, restricted=is_tacotron(c), val_type=float, min_val=0)
-    check_argument("warmup_steps", c, restricted=True, val_type=int, min_val=0)
-    check_argument("seq_len_norm", c, restricted=is_tacotron(c), val_type=bool)
+#     # optimizer
+#     check_argument('noam_schedule', c, restricted=False, val_type=bool)
+#     check_argument('grad_clip', c, restricted=True, val_type=float, min_val=0.0)
+#     check_argument('epochs', c, restricted=True, val_type=int, min_val=1)
+#     check_argument('lr', c, restricted=True, val_type=float, min_val=0)
+#     check_argument('wd', c, restricted=is_tacotron(c), val_type=float, min_val=0)
+#     check_argument('warmup_steps', c, restricted=True, val_type=int, min_val=0)
+#     check_argument('seq_len_norm', c, restricted=is_tacotron(c), val_type=bool)
 
-    # tacotron prenet
-    check_argument("memory_size", c, restricted=is_tacotron(c), val_type=int, min_val=-1)
-    check_argument("prenet_type", c, restricted=is_tacotron(c), val_type=str, enum_list=["original", "bn"])
-    check_argument("prenet_dropout", c, restricted=is_tacotron(c), val_type=bool)
+#     # tacotron prenet
+#     # check_argument('memory_size', c, restricted=is_tacotron(c), val_type=int, min_val=-1)
+#     # check_argument('prenet_type', c, restricted=is_tacotron(c), val_type=str, enum_list=['original', 'bn'])
+#     # check_argument('prenet_dropout', c, restricted=is_tacotron(c), val_type=bool)
 
-    # attention
-    check_argument(
-        "attention_type",
-        c,
-        restricted=is_tacotron(c),
-        val_type=str,
-        enum_list=["graves", "original", "dynamic_convolution"],
-    )
-    check_argument("attention_heads", c, restricted=is_tacotron(c), val_type=int)
-    check_argument("attention_norm", c, restricted=is_tacotron(c), val_type=str, enum_list=["sigmoid", "softmax"])
-    check_argument("windowing", c, restricted=is_tacotron(c), val_type=bool)
-    check_argument("use_forward_attn", c, restricted=is_tacotron(c), val_type=bool)
-    check_argument("forward_attn_mask", c, restricted=is_tacotron(c), val_type=bool)
-    check_argument("transition_agent", c, restricted=is_tacotron(c), val_type=bool)
-    check_argument("transition_agent", c, restricted=is_tacotron(c), val_type=bool)
-    check_argument("location_attn", c, restricted=is_tacotron(c), val_type=bool)
-    check_argument("bidirectional_decoder", c, restricted=is_tacotron(c), val_type=bool)
-    check_argument("double_decoder_consistency", c, restricted=is_tacotron(c), val_type=bool)
-    check_argument("ddc_r", c, restricted="double_decoder_consistency" in c.keys(), min_val=1, max_val=7, val_type=int)
+#     # attention
+#     check_argument('attention_type', c, restricted=is_tacotron(c), val_type=str, enum_list=['graves', 'original', 'dynamic_convolution'])
+#     check_argument('attention_heads', c, restricted=is_tacotron(c), val_type=int)
+#     check_argument('attention_norm', c, restricted=is_tacotron(c), val_type=str, enum_list=['sigmoid', 'softmax'])
+#     check_argument('windowing', c, restricted=is_tacotron(c), val_type=bool)
+#     check_argument('use_forward_attn', c, restricted=is_tacotron(c), val_type=bool)
+#     check_argument('forward_attn_mask', c, restricted=is_tacotron(c), val_type=bool)
+#     check_argument('transition_agent', c, restricted=is_tacotron(c), val_type=bool)
+#     check_argument('transition_agent', c, restricted=is_tacotron(c), val_type=bool)
+#     check_argument('location_attn', c, restricted=is_tacotron(c), val_type=bool)
+#     check_argument('bidirectional_decoder', c, restricted=is_tacotron(c), val_type=bool)
+#     check_argument('double_decoder_consistency', c, restricted=is_tacotron(c), val_type=bool)
+#     check_argument('ddc_r', c, restricted='double_decoder_consistency' in c.keys(), min_val=1, max_val=7, val_type=int)
 
-    if c["model"].lower() in ["tacotron", "tacotron2"]:
-        # stopnet
-        check_argument("stopnet", c, restricted=is_tacotron(c), val_type=bool)
-        check_argument("separate_stopnet", c, restricted=is_tacotron(c), val_type=bool)
+#     if c['model'].lower() in ['tacotron', 'tacotron2']:
+#         # stopnet
+#         # check_argument('stopnet', c, restricted=is_tacotron(c), val_type=bool)
+#         # check_argument('separate_stopnet', c, restricted=is_tacotron(c), val_type=bool)
 
-    # Model Parameters for non-tacotron models
-    if c["model"].lower in ["speedy_speech", "align_tts"]:
-        check_argument("positional_encoding", c, restricted=True, val_type=type)
-        check_argument("encoder_type", c, restricted=True, val_type=str)
-        check_argument("encoder_params", c, restricted=True, val_type=dict)
-        check_argument("decoder_residual_conv_bn_params", c, restricted=True, val_type=dict)
+#     # Model Parameters for non-tacotron models
+#     if c['model'].lower in ["speedy_speech", "align_tts"]:
+#         check_argument('positional_encoding', c, restricted=True, val_type=type)
+#         check_argument('encoder_type', c, restricted=True, val_type=str)
+#         check_argument('encoder_params', c, restricted=True, val_type=dict)
+#         check_argument('decoder_residual_conv_bn_params', c, restricted=True, val_type=dict)
 
-    # GlowTTS parameters
-    check_argument("encoder_type", c, restricted=not is_tacotron(c), val_type=str)
+#     # GlowTTS parameters
+#     check_argument('encoder_type', c, restricted=not is_tacotron(c), val_type=str)
 
-    # tensorboard
-    check_argument("print_step", c, restricted=True, val_type=int, min_val=1)
-    check_argument("tb_plot_step", c, restricted=True, val_type=int, min_val=1)
-    check_argument("save_step", c, restricted=True, val_type=int, min_val=1)
-    check_argument("checkpoint", c, restricted=True, val_type=bool)
-    check_argument("tb_model_param_stats", c, restricted=True, val_type=bool)
+#     # tensorboard
+#     # check_argument('print_step', c, restricted=True, val_type=int, min_val=1)
+#     # check_argument('tb_plot_step', c, restricted=True, val_type=int, min_val=1)
+#     # check_argument('save_step', c, restricted=True, val_type=int, min_val=1)
+#     # check_argument('checkpoint', c, restricted=True, val_type=bool)
+#     # check_argument('tb_model_param_stats', c, restricted=True, val_type=bool)
 
-    # dataloading
-    # pylint: disable=import-outside-toplevel
-    from TTS.tts.utils.text import cleaners
+#     # dataloading
+#     # pylint: disable=import-outside-toplevel
+#     from TTS.tts.utils.text import cleaners
+#     # check_argument('text_cleaner', c, restricted=True, val_type=str, enum_list=dir(cleaners))
+#     # check_argument('enable_eos_bos_chars', c, restricted=True, val_type=bool)
+#     # check_argument('num_loader_workers', c, restricted=True, val_type=int, min_val=0)
+#     # check_argument('num_val_loader_workers', c, restricted=True, val_type=int, min_val=0)
+#     # check_argument('batch_group_size', c, restricted=True, val_type=int, min_val=0)
+#     # check_argument('min_seq_len', c, restricted=True, val_type=int, min_val=0)
+#     # check_argument('max_seq_len', c, restricted=True, val_type=int, min_val=10)
+#     # check_argument('compute_input_seq_cache', c, restricted=True, val_type=bool)
 
-    check_argument("text_cleaner", c, restricted=True, val_type=str, enum_list=dir(cleaners))
-    check_argument("enable_eos_bos_chars", c, restricted=True, val_type=bool)
-    check_argument("num_loader_workers", c, restricted=True, val_type=int, min_val=0)
-    check_argument("num_val_loader_workers", c, restricted=True, val_type=int, min_val=0)
-    check_argument("batch_group_size", c, restricted=True, val_type=int, min_val=0)
-    check_argument("min_seq_len", c, restricted=True, val_type=int, min_val=0)
-    check_argument("max_seq_len", c, restricted=True, val_type=int, min_val=10)
-    check_argument("compute_input_seq_cache", c, restricted=True, val_type=bool)
+#     # paths
+#     # check_argument('output_path', c, restricted=True, val_type=str)
 
-    # paths
-    check_argument("output_path", c, restricted=True, val_type=str)
+#     # multi-speaker and gst
+#     # check_argument('use_speaker_embedding', c, restricted=True, val_type=bool)
+#     # check_argument('use_external_speaker_embedding_file', c, restricted=c['use_speaker_embedding'], val_type=bool)
+#     # check_argument('external_speaker_embedding_file', c, restricted=c['use_external_speaker_embedding_file'], val_type=str)
+#     if c['model'].lower() in ['tacotron', 'tacotron2'] and c['use_gst']:
+#         # check_argument('use_gst', c, restricted=is_tacotron(c), val_type=bool)
+#         # check_argument('gst', c, restricted=is_tacotron(c), val_type=dict)
+#         # check_argument('gst_style_input', c['gst'], restricted=is_tacotron(c), val_type=[str, dict])
+#         # check_argument('gst_embedding_dim', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=0, max_val=1000)
+#         # check_argument('gst_use_speaker_embedding', c['gst'], restricted=is_tacotron(c), val_type=bool)
+#         # check_argument('gst_num_heads', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=2, max_val=10)
+#         # check_argument('gst_num_style_tokens', c['gst'], restricted=is_tacotron(c), val_type=int, min_val=1, max_val=1000)
 
-    # multi-speaker and gst
-    check_argument("use_speaker_embedding", c, restricted=True, val_type=bool)
-    check_argument("use_external_speaker_embedding_file", c, restricted=c["use_speaker_embedding"], val_type=bool)
-    check_argument(
-        "external_speaker_embedding_file", c, restricted=c["use_external_speaker_embedding_file"], val_type=str
-    )
-    if c["model"].lower() in ["tacotron", "tacotron2"] and c["use_gst"]:
-        check_argument("use_gst", c, restricted=is_tacotron(c), val_type=bool)
-        check_argument("gst", c, restricted=is_tacotron(c), val_type=dict)
-        check_argument("gst_style_input", c["gst"], restricted=is_tacotron(c), val_type=[str, dict])
-        check_argument("gst_embedding_dim", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=0, max_val=1000)
-        check_argument("gst_use_speaker_embedding", c["gst"], restricted=is_tacotron(c), val_type=bool)
-        check_argument("gst_num_heads", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=2, max_val=10)
-        check_argument("gst_style_tokens", c["gst"], restricted=is_tacotron(c), val_type=int, min_val=1, max_val=1000)
-
-    # datasets - checking only the first entry
-    check_argument("datasets", c, restricted=True, val_type=list)
-    for dataset_entry in c["datasets"]:
-        check_argument("name", dataset_entry, restricted=True, val_type=str)
-        check_argument("path", dataset_entry, restricted=True, val_type=str)
-        check_argument("meta_file_train", dataset_entry, restricted=True, val_type=[str, list])
-        check_argument("meta_file_val", dataset_entry, restricted=True, val_type=str)
+#     # datasets - checking only the first entry
+#     # check_argument('datasets', c, restricted=True, val_type=list)
+#     # for dataset_entry in c['datasets']:
+#     #     check_argument('name', dataset_entry, restricted=True, val_type=str)
+#     #     check_argument('path', dataset_entry, restricted=True, val_type=str)
+#     #     check_argument('meta_file_train', dataset_entry, restricted=True, val_type=[str, list])
+#     #     check_argument('meta_file_val', dataset_entry, restricted=True, val_type=str)

TTS/tts/utils/speakers.py

@@ -6,9 +6,9 @@ from typing import Union
 import numpy as np
 import torch
 
+from TTS.config import load_config
 from TTS.speaker_encoder.utils.generic_utils import setup_model
 from TTS.utils.audio import AudioProcessor
-from TTS.utils.io import load_config
 
 
 def make_speakers_json_path(out_path):
@@ -28,9 +28,10 @@ def load_speaker_mapping(out_path):
 
 def save_speaker_mapping(out_path, speaker_mapping):
     """Saves speaker mapping if not yet present."""
-    speakers_json_path = make_speakers_json_path(out_path)
-    with open(speakers_json_path, "w") as f:
-        json.dump(speaker_mapping, f, indent=4)
+    if out_path is not None:
+        speakers_json_path = make_speakers_json_path(out_path)
+        with open(speakers_json_path, "w") as f:
+            json.dump(speaker_mapping, f, indent=4)
 
 
 def get_speakers(items):

TTS/tts/utils/synthesis.py

@@ -23,8 +23,8 @@ def text_to_seqvec(text, CONFIG):
                 text_cleaner,
                 CONFIG.phoneme_language,
                 CONFIG.enable_eos_bos_chars,
-                tp=CONFIG.characters if "characters" in CONFIG.keys() else None,
-                add_blank=CONFIG["add_blank"] if "add_blank" in CONFIG.keys() else False,
+                tp=CONFIG.characters,
+                add_blank=CONFIG.add_blank,
             ),
             dtype=np.int32,
         )
@@ -33,8 +33,8 @@ def text_to_seqvec(text, CONFIG):
             text_to_sequence(
                 text,
                 text_cleaner,
-                tp=CONFIG.characters if "characters" in CONFIG.keys() else None,
-                add_blank=CONFIG["add_blank"] if "add_blank" in CONFIG.keys() else False,
+                tp=CONFIG.characters,
+                add_blank=CONFIG.add_blank,
             ),
             dtype=np.int32,
         )
@@ -65,28 +65,31 @@ def compute_style_mel(style_wav, ap, cuda=False):
 
 
 def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None, speaker_embeddings=None):
-    speaker_embedding_g = speaker_id if speaker_id is not None else speaker_embeddings
     if "tacotron" in CONFIG.model.lower():
-        if not CONFIG.use_gst:
-            style_mel = None
-
-        if truncated:
-            decoder_output, postnet_output, alignments, stop_tokens = model.inference_truncated(
-                inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings
-            )
-        else:
+        if CONFIG.gst:
             decoder_output, postnet_output, alignments, stop_tokens = model.inference(
                 inputs, style_mel=style_mel, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings
             )
+        else:
+            if truncated:
+                decoder_output, postnet_output, alignments, stop_tokens = model.inference_truncated(
+                    inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings
+                )
+            else:
+                decoder_output, postnet_output, alignments, stop_tokens = model.inference(
+                    inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings
+                )
     elif "glow" in CONFIG.model.lower():
         inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device)  # pylint: disable=not-callable
         if hasattr(model, "module"):
             # distributed model
             postnet_output, _, _, _, alignments, _, _ = model.module.inference(
-                inputs, inputs_lengths, g=speaker_embedding_g
+                inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings
             )
         else:
-            postnet_output, _, _, _, alignments, _, _ = model.inference(inputs, inputs_lengths, g=speaker_embedding_g)
+            postnet_output, _, _, _, alignments, _, _ = model.inference(
+                inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings
+            )
         postnet_output = postnet_output.permute(0, 2, 1)
         # these only belong to tacotron models.
         decoder_output = None
@@ -95,9 +98,13 @@ def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel
         inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device)  # pylint: disable=not-callable
         if hasattr(model, "module"):
             # distributed model
-            postnet_output, alignments = model.module.inference(inputs, inputs_lengths, g=speaker_embedding_g)
+            postnet_output, alignments = model.module.inference(
+                inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings
+            )
         else:
-            postnet_output, alignments = model.inference(inputs, inputs_lengths, g=speaker_embedding_g)
+            postnet_output, alignments = model.inference(
+                inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings
+            )
         postnet_output = postnet_output.permute(0, 2, 1)
         # these only belong to tacotron models.
         decoder_output = None
@@ -108,7 +115,7 @@ def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel
 
 
 def run_model_tf(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None):
-    if CONFIG.use_gst and style_mel is not None:
+    if CONFIG.gst and style_mel is not None:
         raise NotImplementedError(" [!] GST inference not implemented for TF")
     if truncated:
         raise NotImplementedError(" [!] Truncated inference not implemented for TF")
@@ -120,7 +127,7 @@ def run_model_tf(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=No
 
 
 def run_model_tflite(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None):
-    if CONFIG.use_gst and style_mel is not None:
+    if CONFIG.gst and style_mel is not None:
         raise NotImplementedError(" [!] GST inference not implemented for TfLite")
     if truncated:
         raise NotImplementedError(" [!] Truncated inference not implemented for TfLite")
@@ -249,7 +256,7 @@ def synthesis(
     """
     # GST processing
     style_mel = None
-    if "use_gst" in CONFIG.keys() and CONFIG.use_gst and style_wav is not None:
+    if CONFIG.has("gst") and CONFIG.gst and style_wav is not None:
         if isinstance(style_wav, dict):
             style_mel = style_wav
         else:

TTS/tts/utils/text/__init__.py

@@ -2,12 +2,10 @@
 
 import re
 
-import phonemizer
 from packaging import version
-from phonemizer.phonemize import phonemize
 
-from TTS.tts.utils.chinese_mandarin.phonemizer import chinese_text_to_phonemes
 from TTS.tts.utils.text import cleaners
+from TTS.tts.utils.text.chinese_mandarin.phonemizer import chinese_text_to_phonemes
 from TTS.tts.utils.text.symbols import _bos, _eos, _punctuations, make_symbols, phonemes, symbols
 
 # pylint: disable=unnecessary-comprehension
@@ -28,62 +26,20 @@ PHONEME_PUNCTUATION_PATTERN = r"[" + _punctuations.replace(" ", "") + "]+"
 
 
 def text2phone(text, language):
-    """Convert graphemes to phonemes. For most of the languages, it calls
-    the phonemizer python library that calls espeak/espeak-ng. For chinese
-    mandarin, it calls pypinyin + custom function for phonemizing
-        Parameters:
-                text (str): text to phonemize
-                language (str): language of the text
-        Returns:
-                ph (str): phonemes as a string seperated by "|"
-                        ph = "ɪ|g|ˈ|z|æ|m|p|ə|l"
+    """Convert graphemes to phonemes.
+    Parameters:
+            text (str): text to phonemize
+            language (str): language of the text
+    Returns:
+            ph (str): phonemes as a string seperated by "|"
+                    ph = "ɪ|g|ˈ|z|æ|m|p|ə|l"
     """
 
     # TO REVIEW : How to have a good implementation for this?
     if language == "zh-CN":
         ph = chinese_text_to_phonemes(text)
         return ph
-
-    seperator = phonemizer.separator.Separator(" |", "", "|")
-    # try:
-    punctuations = re.findall(PHONEME_PUNCTUATION_PATTERN, text)
-    if version.parse(phonemizer.__version__) < version.parse("2.1"):
-        ph = phonemize(text, separator=seperator, strip=False, njobs=1, backend="espeak", language=language)
-        ph = ph[:-1].strip()  # skip the last empty character
-        # phonemizer does not tackle punctuations. Here we do.
-        # Replace \n with matching punctuations.
-        if punctuations:
-            # if text ends with a punctuation.
-            if text[-1] == punctuations[-1]:
-                for punct in punctuations[:-1]:
-                    ph = ph.replace("| |\n", "|" + punct + "| |", 1)
-                    ph = ph + punctuations[-1]
-            else:
-                for punct in punctuations:
-                    ph = ph.replace("| |\n", "|" + punct + "| |", 1)
-    elif version.parse(phonemizer.__version__) >= version.parse("2.1"):
-        ph = phonemize(
-            text,
-            separator=seperator,
-            strip=False,
-            njobs=1,
-            backend="espeak",
-            language=language,
-            preserve_punctuation=True,
-            language_switch="remove-flags",
-        )
-        # this is a simple fix for phonemizer.
-        # https://github.com/bootphon/phonemizer/issues/32
-        if punctuations:
-            for punctuation in punctuations:
-                ph = ph.replace(f"| |{punctuation} ", f"|{punctuation}| |").replace(
-                    f"| |{punctuation}", f"|{punctuation}| |"
-                )
-            ph = ph[:-3]
-    else:
-        raise RuntimeError(" [!] Use 'phonemizer' version 2.1 or older.")
-
-    return ph
+    raise ValueError(f" [!] Language {language} is not supported for phonemization.")
 
 
 def intersperse(sequence, token):

TTS/tts/utils/text/cleaners.py

@@ -14,7 +14,7 @@ import re
 
 from unidecode import unidecode
 
-from TTS.tts.utils.chinese_mandarin.numbers import replace_numbers_to_characters_in_text
+from TTS.tts.utils.text.chinese_mandarin.numbers import replace_numbers_to_characters_in_text
 
 from .abbreviations import abbreviations_en, abbreviations_fr
 from .number_norm import normalize_numbers

TTS/utils/arguments.py

@@ -4,20 +4,20 @@
 
 import argparse
 import glob
-import json
 import os
 import re
 
 import torch
 
+from TTS.config import load_config
 from TTS.tts.utils.text.symbols import parse_symbols
 from TTS.utils.console_logger import ConsoleLogger
 from TTS.utils.generic_utils import create_experiment_folder, get_git_branch
-from TTS.utils.io import copy_model_files, load_config
+from TTS.utils.io import copy_model_files
 from TTS.utils.tensorboard_logger import TensorboardLogger
 
 
-def parse_arguments(argv):
+def init_arguments(argv):
     """Parse command line arguments of training scripts.
 
     Args:
@@ -56,7 +56,7 @@ def parse_arguments(argv):
     parser.add_argument("--rank", type=int, default=0, help="DISTRIBUTED: process rank for distributed training.")
     parser.add_argument("--group_id", type=str, default="", help="DISTRIBUTED: process group id.")
 
-    return parser.parse_args()
+    return parser
 
 
 def get_last_checkpoint(path):
@@ -117,16 +117,11 @@ def get_last_checkpoint(path):
     return last_models["checkpoint"], last_models["best_model"]
 
 
-def process_args(args, model_class):
-    """Process parsed comand line arguments based on model class (tts or vocoder).
+def process_args(args):
+    """Process parsed comand line arguments.
 
     Args:
         args (argparse.Namespace or dict like): Parsed input arguments.
-        model_type (str): Model type used to check config parameters and setup
-            the TensorBoard logger. One of ['tts', 'vocoder'].
-
-    Raises:
-        ValueError: If `model_type` is not one of implemented choices.
 
     Returns:
         c (TTS.utils.io.AttrDict): Config paramaters.
@@ -137,29 +132,26 @@ def process_args(args, model_class):
         tb_logger (TTS.utils.tensorboard.TensorboardLogger): Class that does
             the TensorBoard loggind.
     """
+    if isinstance(args, tuple):
+        args, coqpit_overrides = args
     if args.continue_path:
-        args.output_path = args.continue_path
+        # continue a previous training from its output folder
+        experiment_path = args.continue_path
         args.config_path = os.path.join(args.continue_path, "config.json")
         args.restore_path, best_model = get_last_checkpoint(args.continue_path)
         if not args.best_path:
             args.best_path = best_model
-
     # setup output paths and read configs
-    c = load_config(args.config_path)
-    _ = os.path.dirname(os.path.realpath(__file__))
-
-    if "mixed_precision" in c and c.mixed_precision:
+    config = load_config(args.config_path)
+    # override values from command-line args
+    config.parse_known_args(coqpit_overrides, relaxed_parser=True)
+    if config.mixed_precision:
         print("   >  Mixed precision mode is ON")
-
-    out_path = args.continue_path
-    if not out_path:
-        out_path = create_experiment_folder(c.output_path, c.run_name, args.debug)
-
-    audio_path = os.path.join(out_path, "test_audios")
-
-    c_logger = ConsoleLogger()
-    tb_logger = None
-
+    experiment_path = args.continue_path
+    if not experiment_path:
+        experiment_path = create_experiment_folder(config.output_path, config.run_name, args.debug)
+    audio_path = os.path.join(experiment_path, "test_audios")
+    # setup rank 0 process in distributed training
     if args.rank == 0:
         os.makedirs(audio_path, exist_ok=True)
         new_fields = {}
@@ -169,18 +161,22 @@ def process_args(args, model_class):
         # if model characters are not set in the config file
         # save the default set to the config file for future
         # compatibility.
-        if model_class == "tts" and "characters" not in c:
+        if config.has("characters_config"):
             used_characters = parse_symbols()
             new_fields["characters"] = used_characters
-        copy_model_files(c, args.config_path, out_path, new_fields)
+        copy_model_files(config, experiment_path, new_fields)
         os.chmod(audio_path, 0o775)
-        os.chmod(out_path, 0o775)
+        os.chmod(experiment_path, 0o775)
+        tb_logger = TensorboardLogger(experiment_path, model_name=config.model)
+        # write model desc to tensorboard
+        tb_logger.tb_add_text("model-config", f"<pre>{config.to_json()}</pre>", 0)
+    c_logger = ConsoleLogger()
+    return config, experiment_path, audio_path, c_logger, tb_logger
 
-        log_path = out_path
 
-        tb_logger = TensorboardLogger(log_path, model_name=model_class.upper())
-
-        # write model config to tensorboard
-        tb_logger.tb_add_text("model-config", f"<pre>{json.dumps(c, indent=4)}</pre>", 0)
-
-    return c, out_path, audio_path, c_logger, tb_logger
+def init_training(argv):
+    """Initialization of a training run."""
+    parser = init_arguments(argv)
+    args = parser.parse_known_args()
+    config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger = process_args(args)
+    return args[0], config, OUT_PATH, AUDIO_PATH, c_logger, tb_logger

TTS/utils/audio.py

@@ -21,6 +21,7 @@ class AudioProcessor(object):
         sample_rate (int, optional): target audio sampling rate. Defaults to None.
         resample (bool, optional): enable/disable resampling of the audio clips when the target sampling rate does not match the original sampling rate. Defaults to False.
         num_mels (int, optional): number of melspectrogram dimensions. Defaults to None.
+        log_func (int, optional): log exponent used for converting spectrogram aplitude to DB.
         min_level_db (int, optional): minimum db threshold for the computed melspectrograms. Defaults to None.
         frame_shift_ms (int, optional): milliseconds of frames between STFT columns. Defaults to None.
         frame_length_ms (int, optional): milliseconds of STFT window length. Defaults to None.

TTS/utils/generic_utils.py

@@ -1,11 +1,22 @@
+# -*- coding: utf-8 -*-
 import datetime
 import glob
+import importlib
 import os
+import re
 import shutil
 import subprocess
 import sys
 from pathlib import Path
 
+import torch
+
+
+def get_cuda():
+    use_cuda = torch.cuda.is_available()
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    return use_cuda, device
+
 
 def get_git_branch():
     try:
@@ -66,6 +77,20 @@ def count_parameters(model):
     return sum(p.numel() for p in model.parameters() if p.requires_grad)
 
 
+def to_camel(text):
+    text = text.capitalize()
+    text = re.sub(r"(?!^)_([a-zA-Z])", lambda m: m.group(1).upper(), text)
+    text = text.replace("Tts", "TTS")
+    return text
+
+
+def find_module(module_path: str, module_name: str) -> object:
+    module_name = module_name.lower()
+    module = importlib.import_module(module_path + "." + module_name)
+    class_name = to_camel(module_name)
+    return getattr(module, class_name)
+
+
 def get_user_data_dir(appname):
     if sys.platform == "win32":
         import winreg  # pylint: disable=import-outside-toplevel
@@ -92,7 +117,7 @@ def set_init_dict(model_dict, checkpoint_state, c):
     # 2. filter out different size layers
     pretrained_dict = {k: v for k, v in pretrained_dict.items() if v.numel() == model_dict[k].numel()}
     # 3. skip reinit layers
-    if c.reinit_layers is not None:
+    if c.has("reinit_layers") and c.reinit_layers is not None:
         for reinit_layer_name in c.reinit_layers:
             pretrained_dict = {k: v for k, v in pretrained_dict.items() if reinit_layer_name not in k}
     # 4. overwrite entries in the existing state dict
@@ -137,29 +162,3 @@ class KeepAverage:
     def update_values(self, value_dict):
         for key, value in value_dict.items():
             self.update_value(key, value)
-
-
-def check_argument(
-    name, c, enum_list=None, max_val=None, min_val=None, restricted=False, val_type=None, alternative=None
-):
-    if alternative in c.keys() and c[alternative] is not None:
-        return
-    if restricted:
-        assert name in c.keys(), f" [!] {name} not defined in config.json"
-    if name in c.keys():
-        if max_val:
-            assert c[name] <= max_val, f" [!] {name} is larger than max value {max_val}"
-        if min_val:
-            assert c[name] >= min_val, f" [!] {name} is smaller than min value {min_val}"
-        if enum_list:
-            assert c[name].lower() in enum_list, f" [!] {name} is not a valid value"
-        if isinstance(val_type, list):
-            is_valid = False
-            for typ in val_type:
-                if isinstance(c[name], typ):
-                    is_valid = True
-            assert is_valid or c[name] is None, f" [!] {name} has wrong type - {type(c[name])} vs {val_type}"
-        elif val_type:
-            assert (
-                isinstance(c[name], val_type) or c[name] is None
-            ), f" [!] {name} has wrong type - {type(c[name])} vs {val_type}"

TTS/utils/io.py

@@ -1,11 +1,7 @@
-import json
 import os
 import pickle as pickle_tts
-import re
 from shutil import copyfile
 
-import yaml
-
 
 class RenamingUnpickler(pickle_tts.Unpickler):
     """Overload default pickler to solve module renaming problem"""
@@ -23,64 +19,25 @@ class AttrDict(dict):
         self.__dict__ = self
 
 
-def read_json_with_comments(json_path):
-    # fallback to json
-    with open(json_path, "r", encoding="utf-8") as f:
-        input_str = f.read()
-    # handle comments
-    input_str = re.sub(r"\\\n", "", input_str)
-    input_str = re.sub(r"//.*\n", "\n", input_str)
-    data = json.loads(input_str)
-    return data
-
-
-def load_config(config_path: str) -> AttrDict:
-    """Load config files and discard comments
-
-    Args:
-        config_path (str): path to config file.
-    """
-    config = AttrDict()
-
-    ext = os.path.splitext(config_path)[1]
-    if ext in (".yml", ".yaml"):
-        with open(config_path, "r", encoding="utf-8") as f:
-            data = yaml.safe_load(f)
-    else:
-        data = read_json_with_comments(config_path)
-    config.update(data)
-    return config
-
-
-def copy_model_files(c, config_file, out_path, new_fields):
+def copy_model_files(config, out_path, new_fields):
     """Copy config.json and other model files to training folder and add
     new fields.
 
     Args:
-        c (dict): model config from config.json.
-        config_file (str): path to config file.
+        config (Coqpit): Coqpit config defining the training run.
         out_path (str): output path to copy the file.
         new_fields (dict): new fileds to be added or edited
             in the config file.
     """
-    # copy config.json
     copy_config_path = os.path.join(out_path, "config.json")
-    config_lines = open(config_file, "r", encoding="utf-8").readlines()
     # add extra information fields
-    for key, value in new_fields.items():
-        if isinstance(value, str):
-            new_line = '"{}":"{}",\n'.format(key, value)
-        else:
-            new_line = '"{}":{},\n'.format(key, json.dumps(value, ensure_ascii=False))
-        config_lines.insert(1, new_line)
-    config_out_file = open(copy_config_path, "w", encoding="utf-8")
-    config_out_file.writelines(config_lines)
-    config_out_file.close()
+    config.update(new_fields, allow_new=True)
+    config.save_json(copy_config_path)
     # copy model stats file if available
-    if c.audio["stats_path"] is not None:
+    if config.audio.stats_path is not None:
         copy_stats_path = os.path.join(out_path, "scale_stats.npy")
         if not os.path.exists(copy_stats_path):
             copyfile(
-                c.audio["stats_path"],
+                config.audio.stats_path,
                 copy_stats_path,
             )

TTS/utils/manage.py

@@ -8,8 +8,8 @@ from shutil import copyfile
 import gdown
 import requests
 
+from TTS.config import load_config
 from TTS.utils.generic_utils import get_user_data_dir
-from TTS.utils.io import load_config
 
 
 class ModelManager(object):
@@ -101,6 +101,11 @@ class ModelManager(object):
         output_path = os.path.join(self.output_prefix, model_full_name)
         output_model_path = os.path.join(output_path, "model_file.pth.tar")
         output_config_path = os.path.join(output_path, "config.json")
+        # NOTE : band-aid for removing phoneme support
+        if "needs_phonemizer" in model_item and model_item["needs_phonemizer"]:
+            raise RuntimeError(
+                " [!] Use 🐸TTS <= v0.0.13 for this model. Current version does not support phoneme based models."
+            )
         if os.path.exists(output_path):
             print(f" > {model_name} is already downloaded.")
         else:
@@ -125,17 +130,15 @@ class ModelManager(object):
                 # set scale stats path in config.json
                 config_path = output_config_path
                 config = load_config(config_path)
-                config["audio"]["stats_path"] = output_stats_path
-                with open(config_path, "w") as jf:
-                    json.dump(config, jf)
+                config.audio.stats_path = output_stats_path
+                config.save_json(config_path)
             # update the speakers.json file path in the model config.json to the current path
             if os.path.exists(output_speakers_path):
                 # set scale stats path in config.json
                 config_path = output_config_path
                 config = load_config(config_path)
-                config["external_speaker_embedding_file"] = output_speakers_path
-                with open(config_path, "w") as jf:
-                    json.dump(config, jf)
+                config.external_speaker_embedding_file = output_speakers_path
+                config.save_json(config_path)
         return output_model_path, output_config_path, model_item
 
     def _download_gdrive_file(self, gdrive_idx, output):

TTS/utils/synthesizer.py

@@ -5,6 +5,7 @@ import numpy as np
 import pysbd
 import torch
 
+from TTS.config import load_config
 from TTS.tts.utils.generic_utils import setup_model
 from TTS.tts.utils.speakers import SpeakerManager
 
@@ -13,7 +14,6 @@ from TTS.tts.utils.speakers import SpeakerManager
 from TTS.tts.utils.synthesis import synthesis, trim_silence
 from TTS.tts.utils.text import make_symbols, phonemes, symbols
 from TTS.utils.audio import AudioProcessor
-from TTS.utils.io import load_config
 from TTS.vocoder.utils.generic_utils import interpolate_vocoder_input, setup_generator
 
 
@@ -113,12 +113,11 @@ class Synthesizer(object):
         # pylint: disable=global-statement
 
         global symbols, phonemes
-
         self.tts_config = load_config(tts_config_path)
         self.use_phonemes = self.tts_config.use_phonemes
         self.ap = AudioProcessor(verbose=False, **self.tts_config.audio)
 
-        if "characters" in self.tts_config.keys():
+        if self.tts_config.has("characters") and self.tts_config.characters:
             symbols, phonemes = make_symbols(**self.tts_config.characters)
 
         if self.use_phonemes:
@@ -151,7 +150,7 @@ class Synthesizer(object):
             use_cuda (bool): enable/disable CUDA use.
         """
         self.vocoder_config = load_config(model_config)
-        self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config["audio"])
+        self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config.audio)
         self.vocoder_model = setup_generator(self.vocoder_config)
         self.vocoder_model.load_checkpoint(self.vocoder_config, model_file, eval=True)
         if use_cuda:

TTS/vocoder/configs/__init__.py

@@ -0,0 +1,17 @@
+import importlib
+import os
+from inspect import isclass
+
+# import all files under configs/
+configs_dir = os.path.dirname(__file__)
+for file in os.listdir(configs_dir):
+    path = os.path.join(configs_dir, file)
+    if not file.startswith("_") and not file.startswith(".") and (file.endswith(".py") or os.path.isdir(path)):
+        config_name = file[: file.find(".py")] if file.endswith(".py") else file
+        module = importlib.import_module("TTS.vocoder.configs." + config_name)
+        for attribute_name in dir(module):
+            attribute = getattr(module, attribute_name)
+
+            if isclass(attribute):
+                # Add the class to this package's variables
+                globals()[attribute_name] = attribute

TTS/vocoder/configs/fullband_melgan_config.py

@@ -0,0 +1,106 @@
+from dataclasses import dataclass, field
+
+from .shared_configs import BaseGANVocoderConfig
+
+
+@dataclass
+class FullbandMelganConfig(BaseGANVocoderConfig):
+    """Defines parameters for FullBand MelGAN vocoder.
+
+    Example:
+
+        >>> from TTS.vocoder.configs import FullbandMelganConfig
+        >>> config = FullbandMelganConfig()
+
+    Args:
+        model (str):
+            Model name used for selecting the right model at initialization. Defaults to `melgan`.
+        discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to
+            'melgan_multiscale_discriminator`.
+        discriminator_model_params (dict): The discriminator model parameters. Defaults to
+            '{"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]}`
+        generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is
+            considered as a generator too. Defaults to `melgan_generator`.
+        batch_size (int):
+            Batch size used at training. Larger values use more memory. Defaults to 16.
+        seq_len (int):
+            Audio segment length used at training. Larger values use more memory. Defaults to 8192.
+        pad_short (int):
+            Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0.
+        use_noise_augment (bool):
+            enable / disable random noise added to the input waveform. The noise is added after computing the
+            features. Defaults to True.
+        use_cache (bool):
+            enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is
+            not large enough. Defaults to True.
+        use_stft_loss (bool):
+            enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True.
+        use_subband_stft (bool):
+            enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True.
+        use_mse_gan_loss (bool):
+            enable / disable using Mean Squeare Error GAN loss. Defaults to True.
+        use_hinge_gan_loss (bool):
+            enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models.
+            Defaults to False.
+        use_feat_match_loss (bool):
+            enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True.
+        use_l1_spec_loss (bool):
+            enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False.
+        stft_loss_params (dict): STFT loss parameters. Default to
+        `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}`
+        stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total
+            model loss. Defaults to 0.5.
+        subband_stft_loss_weight (float):
+            Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        mse_G_loss_weight (float):
+            MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5.
+        hinge_G_loss_weight (float):
+            Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        feat_match_loss_weight (float):
+            Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108.
+        l1_spec_loss_weight (float):
+            L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+    """
+
+    model: str = "melgan"
+
+    # Model specific params
+    discriminator_model: str = "melgan_multiscale_discriminator"
+    discriminator_model_params: dict = field(
+        default_factory=lambda: {"base_channels": 16, "max_channels": 512, "downsample_factors": [4, 4, 4]}
+    )
+    generator_model: str = "melgan_generator"
+    generator_model_params: dict = field(
+        default_factory=lambda: {"upsample_factors": [8, 8, 2, 2], "num_res_blocks": 4}
+    )
+
+    # Training - overrides
+    batch_size: int = 16
+    seq_len: int = 8192
+    pad_short: int = 2000
+    use_noise_augment: bool = True
+    use_cache: bool = True
+
+    # LOSS PARAMETERS - overrides
+    use_stft_loss: bool = True
+    use_subband_stft_loss: bool = False
+    use_mse_gan_loss: bool = True
+    use_hinge_gan_loss: bool = False
+    use_feat_match_loss: bool = True  # requires MelGAN Discriminators (MelGAN and HifiGAN)
+    use_l1_spec_loss: bool = False
+
+    stft_loss_params: dict = field(
+        default_factory=lambda: {
+            "n_ffts": [1024, 2048, 512],
+            "hop_lengths": [120, 240, 50],
+            "win_lengths": [600, 1200, 240],
+        }
+    )
+
+    # loss weights - overrides
+    stft_loss_weight: float = 0.5
+    subband_stft_loss_weight: float = 0
+    mse_G_loss_weight: float = 2.5
+    hinge_G_loss_weight: float = 0
+    feat_match_loss_weight: float = 108
+    l1_spec_loss_weight: float = 0.0

TTS/vocoder/configs/hifigan_config.py

@@ -0,0 +1,138 @@
+from dataclasses import dataclass, field
+
+from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig
+
+
+@dataclass
+class HifiganConfig(BaseGANVocoderConfig):
+    """Defines parameters for FullBand MelGAN vocoder.
+
+    Example:
+
+        >>> from TTS.vocoder.configs import HifiganConfig
+        >>> config = HifiganConfig()
+
+    Args:
+        model (str):
+            Model name used for selecting the right model at initialization. Defaults to `hifigan`.
+        discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to
+            'hifigan_discriminator`.
+        generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is
+            considered as a generator too. Defaults to `hifigan_generator`.
+        generator_model_params (dict): Parameters of the generator model. Defaults to
+            `
+            {
+                "use_mel": True,
+                "sample_rate": 22050,
+                "n_fft": 1024,
+                "hop_length": 256,
+                "win_length": 1024,
+                "n_mels": 80,
+                "mel_fmin": 0.0,
+                "mel_fmax": None,
+            }
+            `
+        batch_size (int):
+            Batch size used at training. Larger values use more memory. Defaults to 16.
+        seq_len (int):
+            Audio segment length used at training. Larger values use more memory. Defaults to 8192.
+        pad_short (int):
+            Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0.
+        use_noise_augment (bool):
+            enable / disable random noise added to the input waveform. The noise is added after computing the
+            features. Defaults to True.
+        use_cache (bool):
+            enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is
+            not large enough. Defaults to True.
+        use_stft_loss (bool):
+            enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True.
+        use_subband_stft (bool):
+            enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True.
+        use_mse_gan_loss (bool):
+            enable / disable using Mean Squeare Error GAN loss. Defaults to True.
+        use_hinge_gan_loss (bool):
+            enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models.
+            Defaults to False.
+        use_feat_match_loss (bool):
+            enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True.
+        use_l1_spec_loss (bool):
+            enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False.
+        stft_loss_params (dict):
+            STFT loss parameters. Default to
+            `{
+                "n_ffts": [1024, 2048, 512],
+                "hop_lengths": [120, 240, 50],
+                "win_lengths": [600, 1200, 240]
+            }`
+        l1_spec_loss_params (dict):
+            L1 spectrogram loss parameters. Default to
+            `{
+                "use_mel": True,
+                "sample_rate": 22050,
+                "n_fft": 1024,
+                "hop_length": 256,
+                "win_length": 1024,
+                "n_mels": 80,
+                "mel_fmin": 0.0,
+                "mel_fmax": None,
+            }`
+        stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total
+            model loss. Defaults to 0.5.
+        subband_stft_loss_weight (float):
+            Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        mse_G_loss_weight (float):
+            MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5.
+        hinge_G_loss_weight (float):
+            Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        feat_match_loss_weight (float):
+            Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108.
+        l1_spec_loss_weight (float):
+            L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+    """
+
+    model: str = "hifigan"
+    # model specific params
+    discriminator_model: str = "hifigan_discriminator"
+    generator_model: str = "hifigan_generator"
+    generator_model_params: dict = field(
+        default_factory=lambda: {
+            "upsample_factors": [8, 8, 2, 2],
+            "upsample_kernel_sizes": [16, 16, 4, 4],
+            "upsample_initial_channel": 512,
+            "resblock_kernel_sizes": [3, 7, 11],
+            "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+            "resblock_type": "1",
+        }
+    )
+
+    # LOSS PARAMETERS - overrides
+    use_stft_loss: bool = False
+    use_subband_stft_loss: bool = False
+    use_mse_gan_loss: bool = True
+    use_hinge_gan_loss: bool = False
+    use_feat_match_loss: bool = True  # requires MelGAN Discriminators (MelGAN and HifiGAN)
+    use_l1_spec_loss: bool = True
+
+    # loss weights - overrides
+    stft_loss_weight: float = 0
+    subband_stft_loss_weight: float = 0
+    mse_G_loss_weight: float = 1
+    hinge_G_loss_weight: float = 0
+    feat_match_loss_weight: float = 108
+    l1_spec_loss_weight: float = 45
+    l1_spec_loss_params: dict = field(
+        default_factory=lambda: {
+            "use_mel": True,
+            "sample_rate": 22050,
+            "n_fft": 1024,
+            "hop_length": 256,
+            "win_length": 1024,
+            "n_mels": 80,
+            "mel_fmin": 0.0,
+            "mel_fmax": None,
+        }
+    )
+
+    # optimizer parameters
+    lr: float = 1e-4
+    wd: float = 1e-6

TTS/vocoder/configs/hifigan_ljspeech.json

@@ -1,164 +0,0 @@
-{
-    "run_name": "hifigan",
-    "run_description": "hifigan mean-var scaling",
-
-    // AUDIO PARAMETERS
-    "audio":{
-        "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 20,     // reference level db, theoretically 20db is the sound of air.
-        "log_func": "np.log10",
-        "do_sound_norm": true,
-
-        // Silence trimming
-        "do_trim_silence": false,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 1.0,         // scaler value appplied after log transform of spectrogram.
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": "/home/erogol/.local/share/tts/tts_models--en--ljspeech--speedy-speech-wn/scale_stats.npy"
-    },
-
-    // DISTRIBUTED TRAINING
-    "distributed":{
-        "backend": "nccl",
-        "url": "tcp:\/\/localhost:54324"
-    },
-
-    // MODEL PARAMETERS
-    "use_pqmf": false,
-
-    // LOSS PARAMETERS
-    "use_stft_loss": false,
-    "use_subband_stft_loss": false,
-    "use_mse_gan_loss": true,
-    "use_hinge_gan_loss": false,
-    "use_feat_match_loss": true,  // use only with melgan discriminators
-    "use_l1_spec_loss": true,
-
-    // loss weights
-    "stft_loss_weight": 0,
-    "subband_stft_loss_weight": 0,
-    "mse_G_loss_weight": 1,
-    "hinge_G_loss_weight": 0,
-    "feat_match_loss_weight": 10,
-    "l1_spec_loss_weight": 45,
-
-    // multiscale stft loss parameters
-    // "stft_loss_params": {
-    //     "n_ffts": [1024, 2048, 512],
-    //     "hop_lengths": [120, 240, 50],
-    //     "win_lengths": [600, 1200, 240]
-    // },
-
-    "l1_spec_loss_params": {
-        "use_mel": true,
-        "sample_rate": 22050,
-        "n_fft": 1024,
-        "hop_length": 256,
-        "win_length": 1024,
-        "n_mels": 80,
-        "mel_fmin": 0.0,
-        "mel_fmax": null
-    },
-
-    "target_loss": "avg_G_loss",  // loss value to pick the best model to save after each epoch
-
-    // DISCRIMINATOR
-    "discriminator_model": "hifigan_discriminator",
-    //"discriminator_model_params":{
-    //    "peroids": [2, 3, 5, 7, 11],
-    //    "base_channels": 16,
-    //    "max_channels":512,
-    //    "downsample_factors":[4, 4, 4]
-    //},
-    "steps_to_start_discriminator": 0,      // steps required to start GAN trainining.1
-    "diff_samples_for_G_and_D": false,   // draw a new sample from the dataset for the D pass.
-
-    // GENERATOR
-    "generator_model": "hifigan_generator",
-    "generator_model_params": {
-        "upsample_factors":[8,8,2,2],
-        "upsample_kernel_sizes": [16,16,4,4],
-        "upsample_initial_channel": 512,
-        "resblock_kernel_sizes": [3,7,11],
-        "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
-        "resblock_type": "1"
-    },
-
-    // DATASET
-    "data_path": "/home/erogol/gdrive/Datasets/LJSpeech-1.1/wavs/",
-    "feature_path": null,
-    // "feature_path": "/home/erogol/gdrive/Datasets/non-binary-voice-files/tacotron-DCA/",
-    "seq_len": 8192,
-    "pad_short": 2000,
-    "conv_pad": 0,
-    "use_noise_augment": false,
-    "use_cache": true,
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // TRAINING
-    "batch_size": 16,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": 10,  //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "epochs": 10000,                // total number of epochs to train.
-    "wd": 0.0,                // Weight decay weight.
-    "gen_clip_grad": -1,      // Generator gradient clipping threshold. Apply gradient clipping if > 0
-    "disc_clip_grad": -1,     // Discriminator gradient clipping threshold.
-    "lr_gen": 0.0002,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "lr_disc": 0.0002,
-    "optimizer": "AdamW",
-    "optimizer_params":{
-        "betas": [0.8, 0.99],
-        "weight_decay": 0.0
-    },
-    "lr_scheduler_gen": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_gen_params": {
-       "gamma": 0.999,
-        "last_epoch": -1
-    },
-    "lr_scheduler_disc": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_disc_params": {
-    	  "gamma": 0.999,
-        "last_epoch": -1
-    },
-
-    // TENSORBOARD and LOGGING
-    "print_step": 25,       // Number of steps to log traning on console.
-    "print_eval": false,     // If True, it prints loss values for each step in eval run.
-    "save_step": 25000,      // Number of training steps expected to plot training stats on TB and save model checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "tb_model_param_stats": true,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-    // DATA LOADING
-    "num_loader_workers": 8,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "eval_split_size": 10,
-
-    // PATHS
-    "output_path": "/home/erogol/gdrive/Trainings/LJSpeech/"
-}
-
-

TTS/vocoder/configs/melgan_config.py

@@ -0,0 +1,106 @@
+from dataclasses import dataclass, field
+
+from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig
+
+
+@dataclass
+class MelganConfig(BaseGANVocoderConfig):
+    """Defines parameters for MelGAN vocoder.
+
+    Example:
+
+        >>> from TTS.vocoder.configs import MelganConfig
+        >>> config = MelganConfig()
+
+    Args:
+        model (str):
+            Model name used for selecting the right model at initialization. Defaults to `melgan`.
+        discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to
+            'melgan_multiscale_discriminator`.
+        discriminator_model_params (dict): The discriminator model parameters. Defaults to
+            '{"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]}`
+        generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is
+            considered as a generator too. Defaults to `melgan_generator`.
+        batch_size (int):
+            Batch size used at training. Larger values use more memory. Defaults to 16.
+        seq_len (int):
+            Audio segment length used at training. Larger values use more memory. Defaults to 8192.
+        pad_short (int):
+            Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0.
+        use_noise_augment (bool):
+            enable / disable random noise added to the input waveform. The noise is added after computing the
+            features. Defaults to True.
+        use_cache (bool):
+            enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is
+            not large enough. Defaults to True.
+        use_stft_loss (bool):
+            enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True.
+        use_subband_stft (bool):
+            enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True.
+        use_mse_gan_loss (bool):
+            enable / disable using Mean Squeare Error GAN loss. Defaults to True.
+        use_hinge_gan_loss (bool):
+            enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models.
+            Defaults to False.
+        use_feat_match_loss (bool):
+            enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True.
+        use_l1_spec_loss (bool):
+            enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False.
+        stft_loss_params (dict): STFT loss parameters. Default to
+        `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}`
+        stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total
+            model loss. Defaults to 0.5.
+        subband_stft_loss_weight (float):
+            Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        mse_G_loss_weight (float):
+            MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5.
+        hinge_G_loss_weight (float):
+            Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        feat_match_loss_weight (float):
+            Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108.
+        l1_spec_loss_weight (float):
+            L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+    """
+
+    model: str = "melgan"
+
+    # Model specific params
+    discriminator_model: str = "melgan_multiscale_discriminator"
+    discriminator_model_params: dict = field(
+        default_factory=lambda: {"base_channels": 16, "max_channels": 1024, "downsample_factors": [4, 4, 4, 4]}
+    )
+    generator_model: str = "melgan_generator"
+    generator_model_params: dict = field(
+        default_factory=lambda: {"upsample_factors": [8, 8, 2, 2], "num_res_blocks": 3}
+    )
+
+    # Training - overrides
+    batch_size: int = 16
+    seq_len: int = 8192
+    pad_short: int = 2000
+    use_noise_augment: bool = True
+    use_cache: bool = True
+
+    # LOSS PARAMETERS - overrides
+    use_stft_loss: bool = True
+    use_subband_stft_loss: bool = False
+    use_mse_gan_loss: bool = True
+    use_hinge_gan_loss: bool = False
+    use_feat_match_loss: bool = True  # requires MelGAN Discriminators (MelGAN and HifiGAN)
+    use_l1_spec_loss: bool = False
+
+    stft_loss_params: dict = field(
+        default_factory=lambda: {
+            "n_ffts": [1024, 2048, 512],
+            "hop_lengths": [120, 240, 50],
+            "win_lengths": [600, 1200, 240],
+        }
+    )
+
+    # loss weights - overrides
+    stft_loss_weight: float = 0.5
+    subband_stft_loss_weight: float = 0
+    mse_G_loss_weight: float = 2.5
+    hinge_G_loss_weight: float = 0
+    feat_match_loss_weight: float = 108
+    l1_spec_loss_weight: float = 0

TTS/vocoder/configs/multiband-melgan_and_rwd_config.json

@@ -1,158 +0,0 @@
-{
-    "run_name": "multiband-melgan-rwd",
-    "run_description": "multiband melgan with random window discriminator from https://arxiv.org/pdf/1909.11646.pdf",
-
-    // AUDIO PARAMETERS
-    "audio":{
-        // stft parameters
-        "num_freq": 513,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 20,     // reference level db, theoretically 20db is the sound of air.
-
-        // Silence trimming
-        "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // Griffin-Lim
-        "power": 1.5,           // value to sharpen wav signals after GL algorithm.
-        "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 0.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 8000.0,     // maximum freq level for mel-spec. Tune for dataset!!
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": null    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // DISTRIBUTED TRAINING
-    // "distributed":{
-    //     "backend": "nccl",
-    //     "url": "tcp:\/\/localhost:54321"
-    // },
-
-    // MODEL PARAMETERS
-    "use_pqmf": true,
-
-    // LOSS PARAMETERS
-    "use_stft_loss": true,
-    "use_subband_stft_loss": true,
-    "use_mse_gan_loss": true,
-    "use_hinge_gan_loss": false,
-    "use_feat_match_loss": false,  // use only with melgan discriminators
-
-    // loss weights
-    "stft_loss_weight": 0.5,
-    "subband_stft_loss_weight": 0.5,
-    "mse_G_loss_weight": 2.5,
-    "hinge_G_loss_weight": 2.5,
-    "feat_match_loss_weight": 25,
-
-    // multiscale stft loss parameters
-    "stft_loss_params": {
-        "n_ffts": [1024, 2048, 512],
-        "hop_lengths": [120, 240, 50],
-        "win_lengths": [600, 1200, 240]
-    },
-
-    // subband multiscale stft loss parameters
-    "subband_stft_loss_params":{
-        "n_ffts": [384, 683, 171],
-        "hop_lengths": [30, 60, 10],
-        "win_lengths": [150, 300, 60]
-    },
-
-    "target_loss": "avg_G_loss",  // loss value to pick the best model to save after each epoch
-
-    // DISCRIMINATOR
-    "discriminator_model": "random_window_discriminator",
-    "discriminator_model_params":{
-        "uncond_disc_donwsample_factors": [8, 4],
-        "cond_disc_downsample_factors": [[8, 4, 2, 2, 2], [8, 4, 2, 2], [8, 4, 2], [8, 4], [4, 2, 2]],
-        "cond_disc_out_channels": [[128, 128, 256, 256], [128, 256, 256], [128, 256], [256], [128, 256]],
-        "window_sizes": [512, 1024, 2048, 4096, 8192]
-    },
-    "steps_to_start_discriminator": 200000,      // steps required to start GAN trainining.1
-
-    // GENERATOR
-    "generator_model": "multiband_melgan_generator",
-    "generator_model_params": {
-        "upsample_factors":[8, 4, 2],
-        "num_res_blocks": 4
-    },
-
-    // DATASET
-    "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/",
-    "seq_len": 16384,
-    "pad_short": 2000,
-    "conv_pad": 0,
-    "use_noise_augment": false,
-    "use_cache": true,
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // TRAINING
-    "batch_size": 64,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": 10,  //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "noam_schedule": false,        // use noam warmup and lr schedule.
-    "warmup_steps_gen": 4000,          // Noam decay steps to increase the learning rate from 0 to "lr"
-    "warmup_steps_disc": 4000,
-    "epochs": 10000,                // total number of epochs to train.
-    "wd": 0.0,                // Weight decay weight.
-    "gen_clip_grad": -1,      // Generator gradient clipping threshold. Apply gradient clipping if > 0
-    "disc_clip_grad": -1,     // Discriminator gradient clipping threshold.
-    "lr_gen": 0.0002,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "lr_disc": 0.0002,
-    "optimizer": "AdamW",
-    "optimizer_params":{
-        "betas": [0.8, 0.99],
-        "weight_decay": 0.0
-    },
-    "lr_scheduler_gen": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_gen_params": {
-       "gamma": 0.999,
-        "last_epoch": -1
-    },
-    "lr_scheduler_disc": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_disc_params": {
-    	  "gamma": 0.999,
-        "last_epoch": -1
-    },
-
-    // TENSORBOARD and LOGGING
-    "print_step": 25,       // Number of steps to log traning on console.
-    "print_eval": false,     // If True, it prints loss values for each step in eval run.
-    "save_step": 25000,      // Number of training steps expected to plot training stats on TB and save model checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-    // DATA LOADING
-    "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "eval_split_size": 10,
-
-    // PATHS
-    "output_path": "/home/erogol/Models/LJSpeech/"
-}
-

TTS/vocoder/configs/multiband_melgan_config.json

@@ -1,148 +0,0 @@
-{
-    "run_name": "multiband-melgan",
-    "run_description": "multiband melgan mean-var scaling",
-
-    // AUDIO PARAMETERS
-    "audio":{
-        "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 0,     // reference level db, theoretically 20db is the sound of air.
-
-        // Silence trimming
-        "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 1.0,         // scaler value appplied after log transform of spectrogram.
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy"    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // DISTRIBUTED TRAINING
-    // "distributed":{
-    //     "backend": "nccl",
-    //     "url": "tcp:\/\/localhost:54321"
-    // },
-
-    // LOSS PARAMETERS
-    "use_stft_loss": true,
-    "use_subband_stft_loss": true,  // use only with multi-band models.
-    "use_mse_gan_loss": true,
-    "use_hinge_gan_loss": false,
-    "use_feat_match_loss": false,  // use only with melgan discriminators
-
-    // loss weights
-    "stft_loss_weight": 0.5,
-    "subband_stft_loss_weight": 0.5,
-    "mse_G_loss_weight": 2.5,
-    "hinge_G_loss_weight": 2.5,
-    "feat_match_loss_weight": 25,
-
-    // multiscale stft loss parameters
-    "stft_loss_params": {
-        "n_ffts": [1024, 2048, 512],
-        "hop_lengths": [120, 240, 50],
-        "win_lengths": [600, 1200, 240]
-    },
-
-    // subband multiscale stft loss parameters
-    "subband_stft_loss_params":{
-        "n_ffts": [384, 683, 171],
-        "hop_lengths": [30, 60, 10],
-        "win_lengths": [150, 300, 60]
-    },
-
-    "target_loss": "avg_G_loss",  // loss value to pick the best model to save after each epoch
-
-    // DISCRIMINATOR
-    "discriminator_model": "melgan_multiscale_discriminator",
-    "discriminator_model_params":{
-        "base_channels": 16,
-        "max_channels":512,
-        "downsample_factors":[4, 4, 4]
-    },
-    "steps_to_start_discriminator": 200000,      // steps required to start GAN trainining.1
-
-    // GENERATOR
-    "generator_model": "multiband_melgan_generator",
-    "generator_model_params": {
-        "upsample_factors":[8, 4, 2],
-        "num_res_blocks": 4
-    },
-
-    // DATASET
-    "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/",
-    "feature_path": null,
-    "seq_len": 16384,
-    "pad_short": 2000,
-    "conv_pad": 0,
-    "use_noise_augment": false,
-    "use_cache": true,
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // TRAINING
-    "batch_size": 64,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": 10,  //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "epochs": 10000,                // total number of epochs to train.
-    "wd": 0.0,                // Weight decay weight.
-    "gen_clip_grad": -1,      // Generator gradient clipping threshold. Apply gradient clipping if > 0
-    "disc_clip_grad": -1,     // Discriminator gradient clipping threshold.
-    "lr_gen": 0.0002,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "lr_disc": 0.0002,
-    "optimizer": "AdamW",
-    "optimizer_params":{
-        "betas": [0.8, 0.99],
-        "weight_decay": 0.0
-    },
-    "lr_scheduler_gen": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_gen_params": {
-       "gamma": 0.999,
-        "last_epoch": -1
-    },
-    "lr_scheduler_disc": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_disc_params": {
-    	  "gamma": 0.999,
-        "last_epoch": -1
-    },
-
-    // TENSORBOARD and LOGGING
-    "print_step": 25,       // Number of steps to log traning on console.
-    "print_eval": false,     // If True, it prints loss values for each step in eval run.
-    "save_step": 25000,      // Number of training steps expected to plot training stats on TB and save model checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-    // DATA LOADING
-    "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "eval_split_size": 10,
-
-    // PATHS
-    "output_path": "/home/erogol/Models/LJSpeech/"
-}
-

TTS/vocoder/configs/multiband_melgan_config.py

@@ -0,0 +1,144 @@
+from dataclasses import dataclass, field
+
+from TTS.vocoder.configs.shared_configs import BaseGANVocoderConfig
+
+
+@dataclass
+class MultibandMelganConfig(BaseGANVocoderConfig):
+    """Defines parameters for MultiBandMelGAN vocoder.
+
+    Example:
+
+        >>> from TTS.vocoder.configs import MultibandMelganConfig
+        >>> config = MultibandMelganConfig()
+
+    Args:
+        model (str):
+            Model name used for selecting the right model at initialization. Defaults to `melgan`.
+        discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to
+            'melgan_multiscale_discriminator`.
+        discriminator_model_params (dict): The discriminator model parameters. Defaults to
+            '{
+                "base_channels": 16,
+                "max_channels": 512,
+                "downsample_factors": [4, 4, 4]
+            }`
+        generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is
+            considered as a generator too. Defaults to `melgan_generator`.
+        generator_model_param (dict):
+            The generator model parameters. Defaults to `{"upsample_factors": [8, 4, 2], "num_res_blocks": 4}`.
+        use_pqmf (bool):
+            enable / disable PQMF modulation for multi-band training. Defaults to True.
+        lr_gen (float):
+            Initial learning rate for the generator model. Defaults to 0.0001.
+        lr_disc (float):
+            Initial learning rate for the discriminator model. Defaults to 0.0001.
+        optimizer (torch.optim.Optimizer):
+            Optimizer used for the training. Defaults to `AdamW`.
+        optimizer_params (dict):
+            Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}`
+        lr_scheduler_gen (torch.optim.Scheduler):
+            Learning rate scheduler for the generator. Defaults to `MultiStepLR`.
+        lr_scheduler_gen_params (dict):
+            Parameters for the generator learning rate scheduler. Defaults to
+            `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}`.
+        lr_scheduler_disc (torch.optim.Scheduler):
+            Learning rate scheduler for the discriminator. Defaults to `MultiStepLR`.
+        lr_scheduler_dict_params (dict):
+            Parameters for the discriminator learning rate scheduler. Defaults to
+            `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}`.
+        batch_size (int):
+            Batch size used at training. Larger values use more memory. Defaults to 16.
+        seq_len (int):
+            Audio segment length used at training. Larger values use more memory. Defaults to 8192.
+        pad_short (int):
+            Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0.
+        use_noise_augment (bool):
+            enable / disable random noise added to the input waveform. The noise is added after computing the
+            features. Defaults to True.
+        use_cache (bool):
+            enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is
+            not large enough. Defaults to True.
+        steps_to_start_discriminator (int):
+            Number of steps required to start training the discriminator. Defaults to 0.
+        use_stft_loss (bool):`
+            enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True.
+        use_subband_stft (bool):
+            enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True.
+        use_mse_gan_loss (bool):
+            enable / disable using Mean Squeare Error GAN loss. Defaults to True.
+        use_hinge_gan_loss (bool):
+            enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models.
+            Defaults to False.
+        use_feat_match_loss (bool):
+            enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True.
+        use_l1_spec_loss (bool):
+            enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False.
+        stft_loss_params (dict): STFT loss parameters. Default to
+            `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}`
+        stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total
+            model loss. Defaults to 0.5.
+        subband_stft_loss_weight (float):
+            Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        mse_G_loss_weight (float):
+            MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5.
+        hinge_G_loss_weight (float):
+            Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        feat_match_loss_weight (float):
+            Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 108.
+        l1_spec_loss_weight (float):
+            L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+    """
+
+    model: str = "multiband_melgan"
+
+    # Model specific params
+    discriminator_model: str = "melgan_multiscale_discriminator"
+    discriminator_model_params: dict = field(
+        default_factory=lambda: {"base_channels": 16, "max_channels": 512, "downsample_factors": [4, 4, 4]}
+    )
+    generator_model: str = "multiband_melgan_generator"
+    generator_model_params: dict = field(default_factory=lambda: {"upsample_factors": [8, 4, 2], "num_res_blocks": 4})
+    use_pqmf: bool = True
+
+    # optimizer - overrides
+    lr_gen: float = 0.0001  # Initial learning rate.
+    lr_disc: float = 0.0001  # Initial learning rate.
+    optimizer: str = "AdamW"
+    optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0})
+    lr_scheduler_gen: str = "MultiStepLR"  # one of the schedulers from https:#pytorch.org/docs/stable/optim.html
+    lr_scheduler_gen_params: dict = field(
+        default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}
+    )
+    lr_scheduler_disc: str = "MultiStepLR"  # one of the schedulers from https:#pytorch.org/docs/stable/optim.html
+    lr_scheduler_disc_params: dict = field(
+        default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}
+    )
+
+    # Training - overrides
+    batch_size: int = 64
+    seq_len: int = 16384
+    pad_short: int = 2000
+    use_noise_augment: bool = False
+    use_cache: bool = True
+    steps_to_start_discriminator: bool = 200000
+
+    # LOSS PARAMETERS - overrides
+    use_stft_loss: bool = True
+    use_subband_stft_loss: bool = True
+    use_mse_gan_loss: bool = True
+    use_hinge_gan_loss: bool = False
+    use_feat_match_loss: bool = False  # requires MelGAN Discriminators (MelGAN and HifiGAN)
+    use_l1_spec_loss: bool = False
+
+    subband_stft_loss_params: dict = field(
+        default_factory=lambda: {"n_ffts": [384, 683, 171], "hop_lengths": [30, 60, 10], "win_lengths": [150, 300, 60]}
+    )
+
+    # loss weights - overrides
+    stft_loss_weight: float = 0.5
+    subband_stft_loss_weight: float = 0
+    mse_G_loss_weight: float = 2.5
+    hinge_G_loss_weight: float = 0
+    feat_match_loss_weight: float = 108
+    l1_spec_loss_weight: float = 0

TTS/vocoder/configs/parallel_wavegan_config.json

@@ -1,149 +0,0 @@
-{
-    "run_name": "pwgan",
-    "run_description": "parallel-wavegan training",
-
-    // AUDIO PARAMETERS
-    "audio":{
-        "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 0,     // reference level db, theoretically 20db is the sound of air.
-
-        // Silence trimming
-        "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 1.0,         // scaler value appplied after log transform of spectrogram.
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy"    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // DISTRIBUTED TRAINING
-    // "distributed":{
-    //     "backend": "nccl",
-    //     "url": "tcp:\/\/localhost:54321"
-    // },
-
-    // MODEL PARAMETERS
-    "use_pqmf": true,
-
-    // LOSS PARAMETERS
-    "use_stft_loss": true,
-    "use_subband_stft_loss": false,  // USE ONLY WITH MULTIBAND MODELS
-    "use_mse_gan_loss": true,
-    "use_hinge_gan_loss": false,
-    "use_feat_match_loss": false,  // use only with melgan discriminators
-
-    // loss weights
-    "stft_loss_weight": 0.5,
-    "subband_stft_loss_weight": 0.5,
-    "mse_G_loss_weight": 2.5,
-    "hinge_G_loss_weight": 2.5,
-    "feat_match_loss_weight": 25,
-
-    // multiscale stft loss parameters
-    "stft_loss_params": {
-        "n_ffts": [1024, 2048, 512],
-        "hop_lengths": [120, 240, 50],
-        "win_lengths": [600, 1200, 240]
-    },
-
-    // subband multiscale stft loss parameters
-    "subband_stft_loss_params":{
-        "n_ffts": [384, 683, 171],
-        "hop_lengths": [30, 60, 10],
-        "win_lengths": [150, 300, 60]
-    },
-
-    "target_loss": "avg_G_loss",  // loss value to pick the best model to save after each epoch
-
-    // DISCRIMINATOR
-    "discriminator_model": "parallel_wavegan_discriminator",
-    "discriminator_model_params":{
-        "num_layers": 10
-    },
-    "steps_to_start_discriminator": 200000,      // steps required to start GAN trainining.1
-
-    // GENERATOR
-    "generator_model": "parallel_wavegan_generator",
-    "generator_model_params": {
-        "upsample_factors":[4, 4, 4, 4],
-        "stacks": 3,
-        "num_res_blocks": 30
-    },
-
-    // DATASET
-    "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/",
-    "feature_path": null,
-    "seq_len": 25600,
-    "pad_short": 2000,
-    "conv_pad": 0,
-    "use_noise_augment": false,
-    "use_cache": true,
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // TRAINING
-    "batch_size": 6,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": 10,  //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "epochs": 10000,                // total number of epochs to train.
-    "wd": 0.0,                // Weight decay weight.
-    "gen_clip_grad": -1,      // Generator gradient clipping threshold. Apply gradient clipping if > 0
-    "disc_clip_grad": -1,     // Discriminator gradient clipping threshold.
-    "lr_gen": 0.0002,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "lr_disc": 0.0002,
-    "optimizer": "AdamW",
-    "optimizer_params":{
-        "betas": [0.8, 0.99],
-        "weight_decay": 0.0
-    },
-    "lr_scheduler_gen": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_gen_params": {
-       "gamma": 0.999,
-        "last_epoch": -1
-    },
-    "lr_scheduler_disc": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_disc_params": {
-    	  "gamma": 0.999,
-        "last_epoch": -1
-    },
-    // TENSORBOARD and LOGGING
-    "print_step": 25,       // Number of steps to log traning on console.
-    "print_eval": false,     // If True, it prints loss values for each step in eval run.
-    "save_step": 25000,      // Number of training steps expected to plot training stats on TB and save model checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-    // DATA LOADING
-    "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "eval_split_size": 10,
-
-    // PATHS
-    "output_path": "/home/erogol/Models/LJSpeech/"
-}
-

TTS/vocoder/configs/parallel_wavegan_config.py

@@ -0,0 +1,130 @@
+from dataclasses import dataclass, field
+
+from .shared_configs import BaseGANVocoderConfig
+
+
+@dataclass
+class ParallelWaveganConfig(BaseGANVocoderConfig):
+    """Defines parameters for ParallelWavegan vocoder.
+
+    Args:
+        model (str):
+            Model name used for selecting the right configuration at initialization. Defaults to `parallel_wavegan`.
+        discriminator_model (str): One of the discriminators from `TTS.vocoder.models.*_discriminator`. Defaults to
+            'parallel_wavegan_discriminator`.
+        discriminator_model_params (dict): The discriminator model kwargs. Defaults to
+            '{"num_layers": 10}`
+        generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is
+            considered as a generator too. Defaults to `parallel_wavegan_generator`.
+        generator_model_param (dict):
+            The generator model kwargs. Defaults to `{"upsample_factors": [4, 4, 4, 4], "stacks": 3, "num_res_blocks": 30}`.
+        batch_size (int):
+            Batch size used at training. Larger values use more memory. Defaults to 16.
+        seq_len (int):
+            Audio segment length used at training. Larger values use more memory. Defaults to 8192.
+        pad_short (int):
+            Additional padding applied to the audio samples shorter than `seq_len`. Defaults to 0.
+        use_noise_augment (bool):
+            enable / disable random noise added to the input waveform. The noise is added after computing the
+            features. Defaults to True.
+        use_cache (bool):
+            enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is
+            not large enough. Defaults to True.
+        steps_to_start_discriminator (int):
+            Number of steps required to start training the discriminator. Defaults to 0.
+        use_stft_loss (bool):`
+            enable / disable use of STFT loss originally used by ParallelWaveGAN model. Defaults to True.
+        use_subband_stft (bool):
+            enable / disable use of subband loss computation originally used by MultiBandMelgan model. Defaults to True.
+        use_mse_gan_loss (bool):
+            enable / disable using Mean Squeare Error GAN loss. Defaults to True.
+        use_hinge_gan_loss (bool):
+            enable / disable using Hinge GAN loss. You should choose either Hinge or MSE loss for training GAN models.
+            Defaults to False.
+        use_feat_match_loss (bool):
+            enable / disable using Feature Matching loss originally used by MelGAN model. Defaults to True.
+        use_l1_spec_loss (bool):
+            enable / disable using L1 spectrogram loss originally used by HifiGAN model. Defaults to False.
+        stft_loss_params (dict): STFT loss parameters. Default to
+            `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}`
+        stft_loss_weight (float): STFT loss weight that multiplies the computed loss before summing up the total
+            model loss. Defaults to 0.5.
+        subband_stft_loss_weight (float):
+            Subband STFT loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        mse_G_loss_weight (float):
+            MSE generator loss weight that multiplies the computed loss before summing up the total loss. faults to 2.5.
+        hinge_G_loss_weight (float):
+            Hinge generator loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        feat_match_loss_weight (float):
+            Feature matching loss weight that multiplies the computed loss before summing up the total loss. faults to 0.
+        l1_spec_loss_weight (float):
+            L1 spectrogram loss weight that multiplies the computed loss before summing up the total loss. Defaults to 0.
+        lr_gen (float):
+            Generator model initial learning rate. Defaults to 0.0002.
+        lr_disc (float):
+            Discriminator model initial learning rate. Defaults to 0.0002.
+        optimizer (torch.optim.Optimizer):
+            Optimizer used for the training. Defaults to `AdamW`.
+        optimizer_params (dict):
+            Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}`
+        lr_scheduler_gen (torch.optim.Scheduler):
+            Learning rate scheduler for the generator. Defaults to `ExponentialLR`.
+        lr_scheduler_gen_params (dict):
+            Parameters for the generator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`.
+        lr_scheduler_disc (torch.optim.Scheduler):
+            Learning rate scheduler for the discriminator. Defaults to `ExponentialLR`.
+        lr_scheduler_dict_params (dict):
+            Parameters for the discriminator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`.
+    """
+
+    model: str = "parallel_wavegan"
+
+    # Model specific params
+    discriminator_model: str = "parallel_wavegan_discriminator"
+    discriminator_model_params: dict = field(default_factory=lambda: {"num_layers": 10})
+    generator_model: str = "parallel_wavegan_generator"
+    generator_model_params: dict = field(
+        default_factory=lambda: {"upsample_factors": [4, 4, 4, 4], "stacks": 3, "num_res_blocks": 30}
+    )
+
+    # Training - overrides
+    batch_size: int = 6
+    seq_len: int = 25600
+    pad_short: int = 2000
+    use_noise_augment: bool = False
+    use_cache: bool = True
+    steps_to_start_discriminator: int = 200000
+
+    # LOSS PARAMETERS - overrides
+    use_stft_loss: bool = True
+    use_subband_stft_loss: bool = False
+    use_mse_gan_loss: bool = True
+    use_hinge_gan_loss: bool = False
+    use_feat_match_loss: bool = False  # requires MelGAN Discriminators (MelGAN and HifiGAN)
+    use_l1_spec_loss: bool = False
+
+    stft_loss_params: dict = field(
+        default_factory=lambda: {
+            "n_ffts": [1024, 2048, 512],
+            "hop_lengths": [120, 240, 50],
+            "win_lengths": [600, 1200, 240],
+        }
+    )
+
+    # loss weights - overrides
+    stft_loss_weight: float = 0.5
+    subband_stft_loss_weight: float = 0
+    mse_G_loss_weight: float = 2.5
+    hinge_G_loss_weight: float = 0
+    feat_match_loss_weight: float = 0
+    l1_spec_loss_weight: float = 0
+
+    # optimizer overrides
+    lr_gen: float = 0.0002  # Initial learning rate.
+    lr_disc: float = 0.0002  # Initial learning rate.
+    optimizer: str = "AdamW"
+    optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0})
+    lr_scheduler_gen: str = "ExponentialLR"  # one of the schedulers from https:#pytorch.org/docs/stable/optim.html
+    lr_scheduler_gen_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1})
+    lr_scheduler_disc: str = "ExponentialLR"  # one of the schedulers from https:#pytorch.org/docs/stable/optim.html
+    lr_scheduler_disc_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1})

TTS/vocoder/configs/shared_configs.py

@@ -0,0 +1,183 @@
+from dataclasses import dataclass, field
+
+from coqpit import MISSING
+
+from TTS.config import BaseAudioConfig, BaseTrainingConfig
+
+
+@dataclass
+class BaseVocoderConfig(BaseTrainingConfig):
+    """Shared parameters among all the vocoder models.
+    Args:
+        audio (BaseAudioConfig):
+            Audio processor config instance. Defaultsto `BaseAudioConfig()`.
+        use_noise_augment (bool):
+            Augment the input audio with random noise. Defaults to False/
+        eval_split_size (int):
+            Number of instances used for evaluation. Defaults to 10.
+        data_path (str):
+            Root path of the training data. All the audio files found recursively from this root path are used for
+            training. Defaults to MISSING.
+        feature_path (str):
+            Root path to the precomputed feature files. Defaults to None.
+        seq_len (int):
+            Length of the waveform segments used for training. Defaults to MISSING.
+        pad_short (int):
+            Extra padding for the waveforms shorter than `seq_len`. Defaults to 0.
+        conv_path (int):
+            Extra padding for the feature frames against convolution of the edge frames. Defaults to MISSING.
+            Defaults to 0.
+        use_cache (bool):
+            enable / disable in memory caching of the computed features. If the RAM is not enough, if may cause OOM.
+            Defaults to False.
+        epochs (int):
+            Number of training epochs to. Defaults to 10000.
+        wd (float):
+            Weight decay.
+    """
+
+    audio: BaseAudioConfig = field(default_factory=BaseAudioConfig)
+    # dataloading
+    use_noise_augment: bool = False  # enable/disable random noise augmentation in spectrograms.
+    eval_split_size: int = 10  # number of samples used for evaluation.
+    # dataset
+    data_path: str = MISSING  # root data path. It finds all wav files recursively from there.
+    feature_path: str = None  # if you use precomputed features
+    seq_len: int = MISSING  # signal length used in training.
+    pad_short: int = 0  # additional padding for short wavs
+    conv_pad: int = 0  # additional padding against convolutions applied to spectrograms
+    use_cache: bool = False  # use in memory cache to keep the computed features. This might cause OOM.
+    # OPTIMIZER
+    epochs: int = 10000  # total number of epochs to train.
+    wd: float = 0.0  # Weight decay weight.
+
+
+@dataclass
+class BaseGANVocoderConfig(BaseVocoderConfig):
+    """Base config class used among all the GAN based vocoders.
+    Args:
+        use_stft_loss (bool):
+            enable / disable the use of STFT loss. Defaults to True.
+        use_subband_stft_loss (bool):
+            enable / disable the use of Subband STFT loss. Defaults to True.
+        use_mse_gan_loss (bool):
+            enable / disable the use of Mean Squared Error based GAN loss. Defaults to True.
+        use_hinge_gan_loss (bool):
+            enable / disable the use of Hinge GAN loss. Defaults to True.
+        use_feat_match_loss (bool):
+            enable / disable feature matching loss. Defaults to True.
+        use_l1_spec_loss (bool):
+            enable / disable L1 spectrogram loss. Defaults to True.
+        stft_loss_weight (float):
+            Loss weight that multiplies the computed loss value. Defaults to 0.
+        subband_stft_loss_weight (float):
+            Loss weight that multiplies the computed loss value. Defaults to 0.
+        mse_G_loss_weight (float):
+            Loss weight that multiplies the computed loss value. Defaults to 1.
+        hinge_G_loss_weight (float):
+            Loss weight that multiplies the computed loss value. Defaults to 0.
+        feat_match_loss_weight (float):
+            Loss weight that multiplies the computed loss value. Defaults to 100.
+        l1_spec_loss_weight (float):
+            Loss weight that multiplies the computed loss value. Defaults to 45.
+        stft_loss_params (dict):
+            Parameters for the STFT loss. Defaults to `{"n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240]}`.
+        l1_spec_loss_params (dict):
+            Parameters for the L1 spectrogram loss. Defaults to
+            `{
+                "use_mel": True,
+                "sample_rate": 22050,
+                "n_fft": 1024,
+                "hop_length": 256,
+                "win_length": 1024,
+                "n_mels": 80,
+                "mel_fmin": 0.0,
+                "mel_fmax": None,
+            }`
+        target_loss (str):
+            Target loss name that defines the quality of the model. Defaults to `avg_G_loss`.
+        gen_clip_grad (float):
+            Gradient clipping threshold for the generator model. Any value less than 0 disables clipping.
+            Defaults to -1.
+        disc_clip_grad (float):
+            Gradient clipping threshold for the discriminator model. Any value less than 0 disables clipping.
+            Defaults to -1.
+        lr_gen (float):
+            Generator model initial learning rate. Defaults to 0.0002.
+        lr_disc (float):
+            Discriminator model initial learning rate. Defaults to 0.0002.
+        optimizer (torch.optim.Optimizer):
+            Optimizer used for the training. Defaults to `AdamW`.
+        optimizer_params (dict):
+            Optimizer kwargs. Defaults to `{"betas": [0.8, 0.99], "weight_decay": 0.0}`
+        lr_scheduler_gen (torch.optim.Scheduler):
+            Learning rate scheduler for the generator. Defaults to `ExponentialLR`.
+        lr_scheduler_gen_params (dict):
+            Parameters for the generator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`.
+        lr_scheduler_disc (torch.optim.Scheduler):
+            Learning rate scheduler for the discriminator. Defaults to `ExponentialLR`.
+        lr_scheduler_dict_params (dict):
+            Parameters for the discriminator learning rate scheduler. Defaults to `{"gamma": 0.999, "last_epoch": -1}`.
+        use_pqmf (bool):
+            enable / disable PQMF for subband approximation at training. Defaults to False.
+        steps_to_start_discriminator (int):
+            Number of steps required to start training the discriminator. Defaults to 0.
+        diff_samples_for_G_and_D (bool):
+            enable / disable use of different training samples for the generator and the discriminator iterations.
+            Enabling it results in slower iterations but faster convergance in some cases. Defaults to False.
+    """
+
+    # LOSS PARAMETERS
+    use_stft_loss: bool = True
+    use_subband_stft_loss: bool = True
+    use_mse_gan_loss: bool = True
+    use_hinge_gan_loss: bool = True
+    use_feat_match_loss: bool = True  # requires MelGAN Discriminators (MelGAN and HifiGAN)
+    use_l1_spec_loss: bool = True
+
+    # loss weights
+    stft_loss_weight: float = 0
+    subband_stft_loss_weight: float = 0
+    mse_G_loss_weight: float = 1
+    hinge_G_loss_weight: float = 0
+    feat_match_loss_weight: float = 100
+    l1_spec_loss_weight: float = 45
+
+    stft_loss_params: dict = field(
+        default_factory=lambda: {
+            "n_ffts": [1024, 2048, 512],
+            "hop_lengths": [120, 240, 50],
+            "win_lengths": [600, 1200, 240],
+        }
+    )
+
+    l1_spec_loss_params: dict = field(
+        default_factory=lambda: {
+            "use_mel": True,
+            "sample_rate": 22050,
+            "n_fft": 1024,
+            "hop_length": 256,
+            "win_length": 1024,
+            "n_mels": 80,
+            "mel_fmin": 0.0,
+            "mel_fmax": None,
+        }
+    )
+
+    target_loss: str = "avg_G_loss"  # loss value to pick the best model to save after each epoch
+
+    # optimizer
+    gen_clip_grad: float = -1  # Generator gradient clipping threshold. Apply gradient clipping if > 0
+    disc_clip_grad: float = -1  # Discriminator gradient clipping threshold.
+    lr_gen: float = 0.0002  # Initial learning rate.
+    lr_disc: float = 0.0002  # Initial learning rate.
+    optimizer: str = "AdamW"
+    optimizer_params: dict = field(default_factory=lambda: {"betas": [0.8, 0.99], "weight_decay": 0.0})
+    lr_scheduler_gen: str = "ExponentialLR"  # one of the schedulers from https:#pytorch.org/docs/stable/optim.html
+    lr_scheduler_gen_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1})
+    lr_scheduler_disc: str = "ExponentialLR"  # one of the schedulers from https:#pytorch.org/docs/stable/optim.html
+    lr_scheduler_disc_params: dict = field(default_factory=lambda: {"gamma": 0.999, "last_epoch": -1})
+
+    use_pqmf: bool = False  # enable/disable using pqmf for multi-band training. (Multi-band MelGAN)
+    steps_to_start_discriminator = 0  # start training the discriminator after this number of steps.
+    diff_samples_for_G_and_D: bool = False  # use different samples for G and D training steps.

TTS/vocoder/configs/universal_fullband_melgan.json

@@ -1,145 +0,0 @@
-{
-    "run_name": "fullband-melgan",
-    "run_description": "fullband melgan mean-var scaling",
-
-    // AUDIO PARAMETERS
-    "audio":{
-        "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 24000,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 0,     // reference level db, theoretically 20db is the sound of air.
-
-        // Silence trimming
-        "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 1.0,         // scaler value appplied after log transform of spectrogram.
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": "/home/erogol/Data/libritts/LibriTTS/scale_stats.npy"    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // DISTRIBUTED TRAINING
-    "distributed":{
-        "backend": "nccl",
-        "url": "tcp:\/\/localhost:54324"
-    },
-
-    // MODEL PARAMETERS
-    "use_pqmf": false,
-
-    // LOSS PARAMETERS
-    "use_stft_loss": true,
-    "use_subband_stft_loss": false,
-    "use_mse_gan_loss": true,
-    "use_hinge_gan_loss": false,
-    "use_feat_match_loss": false,  // use only with melgan discriminators
-
-    // loss weights
-    "stft_loss_weight": 0.5,
-    "subband_stft_loss_weight": 0.5,
-    "mse_G_loss_weight": 2.5,
-    "hinge_G_loss_weight": 2.5,
-    "feat_match_loss_weight": 25,
-
-    // multiscale stft loss parameters
-    "stft_loss_params": {
-        "n_ffts": [1024, 2048, 512],
-        "hop_lengths": [120, 240, 50],
-        "win_lengths": [600, 1200, 240]
-    },
-
-    "target_loss": "avg_G_loss",  // loss value to pick the best model to save after each epoch
-
-    // DISCRIMINATOR
-    "discriminator_model": "melgan_multiscale_discriminator",
-    "discriminator_model_params":{
-        "base_channels": 16,
-        "max_channels":512,
-        "downsample_factors":[4, 4, 4]
-    },
-    "steps_to_start_discriminator": 200000,      // steps required to start GAN trainining.1
-
-    // GENERATOR
-    "generator_model": "fullband_melgan_generator",
-    "generator_model_params": {
-        "upsample_factors":[8, 8, 4],
-        "num_res_blocks": 4
-    },
-
-    // DATASET
-    "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/",
-    "feature_path": null,
-    "seq_len": 16384,
-    "pad_short": 2000,
-    "conv_pad": 0,
-    "use_noise_augment": false,
-    "use_cache": true,
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // TRAINING
-    "batch_size": 48,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
-
-    // VALIDATION
-    "run_eval": true,
-    "test_delay_epochs": 10,  //Until attention is aligned, testing only wastes computation time.
-    "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
-
-    // OPTIMIZER
-    "epochs": 10000,                // total number of epochs to train.
-    "wd": 0.0,                // Weight decay weight.
-    "gen_clip_grad": -1,      // Generator gradient clipping threshold. Apply gradient clipping if > 0
-    "disc_clip_grad": -1,     // Discriminator gradient clipping threshold.
-    "lr_gen": 0.0002,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "lr_disc": 0.0002,
-    "optimizer": "AdamW",
-    "optimizer_params":{
-        "betas": [0.8, 0.99],
-        "weight_decay": 0.0
-    },
-    "lr_scheduler_gen": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_gen_params": {
-       "gamma": 0.999,
-        "last_epoch": -1
-    },
-    "lr_scheduler_disc": "ExponentialLR",   // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_disc_params": {
-    	  "gamma": 0.999,
-        "last_epoch": -1
-    },
-
-    // TENSORBOARD and LOGGING
-    "print_step": 25,       // Number of steps to log traning on console.
-    "print_eval": false,     // If True, it prints loss values for each step in eval run.
-    "save_step": 25000,      // Number of training steps expected to plot training stats on TB and save model checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-    // DATA LOADING
-    "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "eval_split_size": 10,
-
-    // PATHS
-    "output_path": "/home/erogol/Models/"
-}
-
-

TTS/vocoder/configs/wavegrad_config.py

@@ -0,0 +1,111 @@
+from dataclasses import dataclass, field
+
+from TTS.vocoder.configs.shared_configs import BaseVocoderConfig
+
+
+@dataclass
+class WavegradConfig(BaseVocoderConfig):
+    """Defines parameters for WaveGrad vocoder.
+    Example:
+
+        >>> from TTS.vocoder.configs import WavegradConfig
+        >>> config = WavegradConfig()
+
+    Args:
+        model (str):
+            Model name used for selecting the right model at initialization. Defaults to `wavegrad`.
+        generator_model (str): One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is
+            considered as a generator too. Defaults to `wavegrad`.
+        model_params (dict):
+            WaveGrad kwargs. Defaults to
+            `
+            {
+                "use_weight_norm": True,
+                "y_conv_channels": 32,
+                "x_conv_channels": 768,
+                "ublock_out_channels": [512, 512, 256, 128, 128],
+                "dblock_out_channels": [128, 128, 256, 512],
+                "upsample_factors": [4, 4, 4, 2, 2],
+                "upsample_dilations": [[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8], [1, 2, 4, 8], [1, 2, 4, 8]],
+            }
+            `
+        target_loss (str):
+            Target loss name that defines the quality of the model. Defaults to `avg_wavegrad_loss`.
+        epochs (int):
+            Number of epochs to traing the model. Defaults to 10000.
+        batch_size (int):
+            Batch size used at training. Larger values use more memory. Defaults to 96.
+        seq_len (int):
+            Audio segment length used at training. Larger values use more memory. Defaults to 6144.
+        use_cache (bool):
+            enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is
+            not large enough. Defaults to True.
+        mixed_precision (bool):
+            enable / disable mixed precision training. Default is True.
+        eval_split_size (int):
+            Number of samples used for evalutaion. Defaults to 50.
+        train_noise_schedule (dict):
+            Training noise schedule. Defaults to
+            `{"min_val": 1e-6, "max_val": 1e-2, "num_steps": 1000}`
+        test_noise_schedule (dict):
+            Inference noise schedule. For a better performance, you may need to use `bin/tune_wavegrad.py` to find a
+            better schedule. Defaults to
+            `
+            {
+                "min_val": 1e-6,
+                "max_val": 1e-2,
+                "num_steps": 50,
+            }
+            `
+        grad_clip (float):
+            Gradient clipping threshold. If <= 0.0, no clipping is applied. Defaults to 1.0
+        lr (float):
+            Initila leraning rate. Defaults to 1e-4.
+        lr_scheduler (str):
+            One of the learning rate schedulers from `torch.optim.scheduler.*`. Defaults to `MultiStepLR`.
+        lr_scheduler_params (dict):
+            kwargs for the scheduler. Defaults to `{"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}`
+    """
+
+    model: str = "wavegrad"
+    # Model specific params
+    generator_model: str = "wavegrad"
+    model_params: dict = field(
+        default_factory=lambda: {
+            "use_weight_norm": True,
+            "y_conv_channels": 32,
+            "x_conv_channels": 768,
+            "ublock_out_channels": [512, 512, 256, 128, 128],
+            "dblock_out_channels": [128, 128, 256, 512],
+            "upsample_factors": [4, 4, 4, 2, 2],
+            "upsample_dilations": [[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8], [1, 2, 4, 8], [1, 2, 4, 8]],
+        }
+    )
+    target_loss: str = "avg_wavegrad_loss"  # loss value to pick the best model to save after each epoch
+
+    # Training - overrides
+    epochs: int = 10000
+    batch_size: int = 96
+    seq_len: int = 6144
+    use_cache: bool = True
+    mixed_precision: bool = True
+    eval_split_size: int = 50
+
+    # NOISE SCHEDULE PARAMS
+    train_noise_schedule: dict = field(default_factory=lambda: {"min_val": 1e-6, "max_val": 1e-2, "num_steps": 1000})
+
+    test_noise_schedule: dict = field(
+        default_factory=lambda: {  # inference noise schedule. Try TTS/bin/tune_wavegrad.py to find the optimal values.
+            "min_val": 1e-6,
+            "max_val": 1e-2,
+            "num_steps": 50,
+        }
+    )
+
+    # optimizer overrides
+    grad_clip: float = 1.0
+    lr: float = 1e-4  # Initial learning rate.
+    lr_scheduler: str = "MultiStepLR"  # one of the schedulers from https:#pytorch.org/docs/stable/optim.html
+    lr_scheduler_params: dict = field(
+        default_factory=lambda: {"gamma": 0.5, "milestones": [100000, 200000, 300000, 400000, 500000, 600000]}
+    )

TTS/vocoder/configs/wavegrad_libritts.json

@@ -1,118 +0,0 @@
-{
-    "run_name": "wavegrad-libritts",
-    "run_description": "wavegrad libritts",
-
-    "audio":{
-        "fft_size": 1024,         // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024,      // stft window length in ms.
-        "hop_length": 256,       // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null,  // stft window hop-lengh in ms. If null, 'hop_length' is used.
-
-        // Audio processing parameters
-        "sample_rate": 24000,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "preemphasis": 0.0,     // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 0,     // reference level db, theoretically 20db is the sound of air.
-
-        // Silence trimming
-        "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60,          // threshold for timming silence. Set this according to your dataset.
-
-        // MelSpectrogram parameters
-        "num_mels": 80,         // size of the mel spec frame.
-        "mel_fmin": 50.0,        // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 7600.0,     // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 1.0,         // scaler value appplied after log transform of spectrogram.
-
-        // Normalization parameters
-        "signal_norm": true,    // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100,   // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0,        // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true,      // clip normalized values into the range.
-        "stats_path": "/home/erogol/Data/libritts/LibriTTS/scale_stats_wavegrad.npy"    // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-    // DISTRIBUTED TRAINING
-    "mixed_precision": true,     // enable torch mixed precision training (true, false)
-    "distributed":{
-        "backend": "nccl",
-        "url": "tcp:\/\/localhost:54322"
-    },
-
-    "target_loss": "avg_wavegrad_loss",  // loss value to pick the best model to save after each epoch
-
-    // MODEL PARAMETERS
-    "generator_model": "wavegrad",
-    "model_params":{
-        "use_weight_norm": true,
-        "y_conv_channels":32,
-        "x_conv_channels":768,
-        "ublock_out_channels": [512, 512, 256, 128, 128],
-        "dblock_out_channels": [128, 128, 256, 512],
-        "upsample_factors": [4, 4, 4, 2, 2],
-        "upsample_dilations": [
-            [1, 2, 1, 2],
-            [1, 2, 1, 2],
-            [1, 2, 4, 8],
-            [1, 2, 4, 8],
-            [1, 2, 4, 8]]
-    },
-
-    // DATASET
-    "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/",  // root data path. It finds all wav files recursively from there.
-    "feature_path": null,   // if you use precomputed features
-    "seq_len": 6144,        // 24 * hop_length
-    "pad_short": 0,      // additional padding for short wavs
-    "conv_pad": 0,          // additional padding against convolutions applied to spectrograms
-    "use_noise_augment": false,     // add noise to the audio signal for augmentation
-    "use_cache": false,      // use in memory cache to keep the computed features. This might cause OOM.
-
-    "reinit_layers": [],    // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
-
-    // TRAINING
-    "batch_size": 96,      // Batch size for training.
-
-    // NOISE SCHEDULE PARAMS - Only effective at training time.
-    "train_noise_schedule":{
-        "min_val": 1e-6,
-        "max_val": 1e-2,
-        "num_steps": 1000
-    },
-    "test_noise_schedule":{
-        "min_val": 1e-6,
-        "max_val": 1e-2,
-        "num_steps": 50
-    },
-
-    // VALIDATION
-    "run_eval": true,       // enable/disable evaluation run
-
-    // OPTIMIZER
-    "epochs": 10000,                // total number of epochs to train.
-    "clip_grad": 1.0,                 // Generator gradient clipping threshold. Apply gradient clipping if > 0
-    "lr_scheduler": "MultiStepLR",  // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_params": {
-        "gamma": 0.5,
-        "milestones": [100000, 200000, 300000, 400000, 500000, 600000]
-    },
-    "lr": 1e-4,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-
-    // TENSORBOARD and LOGGING
-    "print_step": 50,       // Number of steps to log traning on console.
-    "print_eval": false,     // If True, it prints loss values for each step in eval run.
-    "save_step": 5000,      // Number of training steps expected to plot training stats on TB and save model checkpoints.
-    "checkpoint": true,     // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": true,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-    // DATA LOADING
-    "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4,    // number of evaluation data loader processes.
-    "eval_split_size": 256,
-
-    // PATHS
-    "output_path": "/home/erogol/Models/LJSpeech/"
-}
-

TTS/vocoder/configs/wavernn_config.json

@@ -1,103 +0,0 @@
-{
-    "run_name": "wavernn_librittts",
-    "run_description": "wavernn libritts training from LJSpeech model",
-
-// AUDIO PARAMETERS
-    "audio": {
-        "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame.
-        "win_length": 1024, // stft window length in ms.
-        "hop_length": 256, // stft window hop-lengh in ms.
-        "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
-        "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used.
-        // Audio processing parameters
-        "sample_rate": 24000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
-        "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
-        "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air.
-        // Silence trimming
-        "do_trim_silence": false, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
-        "trim_db": 60, // threshold for timming silence. Set this according to your dataset.
-        // MelSpectrogram parameters
-        "num_mels": 80, // size of the mel spec frame.
-        "mel_fmin": 40.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
-        "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!!
-        "spec_gain": 20.0, // scaler value appplied after log transform of spectrogram.
-        // Normalization parameters
-        "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
-        "min_level_db": -100, // lower bound for normalization
-        "symmetric_norm": true, // move normalization to range [-1, 1]
-        "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
-        "clip_norm": true, // clip normalized values into the range.
-        "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
-    },
-
-// Generating / Synthesizing
-    "batched": true,
-    "target_samples": 11000, // target number of samples to be generated in each batch entry
-    "overlap_samples": 550, // number of samples for crossfading between batches
-    // DISTRIBUTED TRAINING
-    // "distributed":{
-    //     "backend": "nccl",
-    //     "url": "tcp:\/\/localhost:54321"
-    // },
-
-// MODEL MODE
-    "mode": "mold", // mold [string], gauss [string], bits [int]
-    "mulaw": true, // apply mulaw if mode is bits
-
-// MODEL PARAMETERS
-    "wavernn_model_params": {
-        "rnn_dims": 512,
-        "fc_dims": 512,
-        "compute_dims": 128,
-        "res_out_dims": 128,
-        "num_res_blocks": 10,
-        "use_aux_net": true,
-        "use_upsample_net": true,
-        "upsample_factors": [4, 8, 8] 	// this needs to correctly factorise hop_length
-    },
-
-// GENERATOR - for backward compatibility
-    "generator_model": "WaveRNN",
-
-// DATASET
-    //"use_gta": true,								// use computed gta features from the tts model
-    "data_path": "/home/erogol/Data/libritts/LibriTTS/train-clean-360/", // path containing training wav files
-    "feature_path": null, // path containing computed features from wav files if null compute them
-    "seq_len": 1280, // has to be devideable by hop_length
-    "padding": 2, // pad the input for resnet to see wider input length
-
-// TRAINING
-    "batch_size": 256, // Batch size for training.
-    "epochs": 10000, // total number of epochs to train.
-    "mixed_precision": true, // enable/ disable mixed precision training
-
-// VALIDATION
-    "run_eval": true,
-    "test_every_epochs": 10, // Test after set number of epochs (Test every 10 epochs for example)
-
-// OPTIMIZER
-    "grad_clip": 4, // apply gradient clipping if > 0
-    "lr_scheduler": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
-    "lr_scheduler_params": {
-        "gamma": 0.5,
-        "milestones": [200000, 400000, 600000]
-    },
-    "lr": 1e-4, // initial learning rate
-
-// TENSORBOARD and LOGGING
-    "print_step": 25, // Number of steps to log traning on console.
-    "print_eval": false, // If True, it prints loss values for each step in eval run.
-    "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints.
-    "checkpoint": true, // If true, it saves checkpoints per "save_step"
-    "keep_all_best": false,  // If true, keeps all best_models after keep_after steps
-    "keep_after": 10000,    // Global step after which to keep best models if keep_all_best is true
-    "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
-
-// DATA LOADING
-    "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values.
-    "num_val_loader_workers": 4, // number of evaluation data loader processes.
-    "eval_split_size": 50, // number of samples for testing
-
-// PATHS
-    "output_path": "/home/erogol/Models/LJSpeech/"
-}

TTS/vocoder/configs/wavernn_config.py

@@ -0,0 +1,115 @@
+from dataclasses import dataclass, field
+
+from TTS.vocoder.configs.shared_configs import BaseVocoderConfig
+
+
+@dataclass
+class WavernnConfig(BaseVocoderConfig):
+    """Defines parameters for Wavernn vocoder.
+    Example:
+
+        >>> from TTS.vocoder.configs import WavernnConfig
+        >>> config = WavernnConfig()
+
+    Args:
+        model (str):
+            Model name used for selecting the right model at initialization. Defaults to `wavernn`.
+        mode (str):
+            Output mode of the WaveRNN vocoder. `mold` for Mixture of Logistic Distribution, `gauss` for a single
+            Gaussian Distribution and `bits` for quantized bits as the model's output.
+        mulaw (bool):
+            enable / disable the use of Mulaw quantization for training. Only applicable if `mode == 'bits'`. Defaults
+            to `True`.
+        generator_model (str):
+            One of the generators from TTS.vocoder.models.*`. Every other non-GAN vocoder model is
+            considered as a generator too. Defaults to `WaveRNN`.
+        wavernn_model_params (dict):
+            kwargs for the WaveRNN model. Defaults to
+            `{
+                "rnn_dims": 512,
+                "fc_dims": 512,
+                "compute_dims": 128,
+                "res_out_dims": 128,
+                "num_res_blocks": 10,
+                "use_aux_net": True,
+                "use_upsample_net": True,
+                "upsample_factors": [4, 8, 8]
+            }`
+        batched (bool):
+            enable / disable the batched inference. It speeds up the inference by splitting the input into segments and
+            processing the segments in a batch. Then it merges the outputs with a certain overlap and smoothing. If
+            you set it False, without CUDA, it is too slow to be practical. Defaults to True.
+        target_samples (int):
+            Size of the segments in batched mode. Defaults to 11000.
+        overlap_sampels (int):
+            Size of the overlap between consecutive segments. Defaults to 550.
+        batch_size (int):
+            Batch size used at training. Larger values use more memory. Defaults to 256.
+        seq_len (int):
+            Audio segment length used at training. Larger values use more memory. Defaults to 1280.
+        padding (int):
+            Padding applied to the input feature frames against the convolution layers of the feature network.
+            Defaults to 2.
+        use_noise_augment (bool):
+            enable / disable random noise added to the input waveform. The noise is added after computing the
+            features. Defaults to True.
+        use_cache (bool):
+            enable / disable in memory caching of the computed features. It can cause OOM error if the system RAM is
+            not large enough. Defaults to True.
+        mixed_precision (bool):
+            enable / disable mixed precision training. Default is True.
+        eval_split_size (int):
+            Number of samples used for evalutaion. Defaults to 50.
+        test_every_epoch (int):
+            Number of epochs waited to run the next evalution. Since inference takes some time, it is better to
+            wait some number of epochs not ot waste training time. Defaults to 10.
+        grad_clip (float):
+            Gradient clipping threshold. If <= 0.0, no clipping is applied. Defaults to 4.0
+        lr (float):
+            Initila leraning rate. Defaults to 1e-4.
+        lr_scheduler (str):
+            One of the learning rate schedulers from `torch.optim.scheduler.*`. Defaults to `MultiStepLR`.
+        lr_scheduler_params (dict):
+            kwargs for the scheduler. Defaults to `{"gamma": 0.5, "milestones": [200000, 400000, 600000]}`
+    """
+
+    model: str = "wavernn"
+
+    # Model specific params
+    mode: str = "mold"  # mold [string], gauss [string], bits [int]
+    mulaw: bool = True  # apply mulaw if mode is bits
+    generator_model: str = "WaveRNN"
+    wavernn_model_params: dict = field(
+        default_factory=lambda: {
+            "rnn_dims": 512,
+            "fc_dims": 512,
+            "compute_dims": 128,
+            "res_out_dims": 128,
+            "num_res_blocks": 10,
+            "use_aux_net": True,
+            "use_upsample_net": True,
+            "upsample_factors": [4, 8, 8],  # this needs to correctly factorise hop_length
+        }
+    )
+
+    # Inference
+    batched: bool = True
+    target_samples: int = 11000
+    overlap_samples: int = 550
+
+    # Training - overrides
+    epochs: int = 10000
+    batch_size: int = 256
+    seq_len: int = 1280
+    padding: int = 2
+    use_noise_augment: bool = False
+    use_cache: bool = True
+    mixed_precision: bool = True
+    eval_split_size: int = 50
+    test_every_epochs: int = 10  # number of epochs to wait until the next test run (synthesizing a full audio clip).
+
+    # optimizer overrides
+    grad_clip: float = 4.0
+    lr: float = 1e-4  # Initial learning rate.
+    lr_scheduler: str = "MultiStepLR"  # one of the schedulers from https:#pytorch.org/docs/stable/optim.html
+    lr_scheduler_params: dict = field(default_factory=lambda: {"gamma": 0.5, "milestones": [200000, 400000, 600000]})

hubconf.py

@@ -1,6 +1,6 @@
 dependencies = [
     'torch', 'gdown', 'pysbd', 'phonemizer', 'unidecode', 'pypinyin'
-]  # apt install espeak-ng
+]
 import torch
 
 from TTS.utils.manage import ModelManager

notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb

@@ -1,387 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "6LWsNd3_M3MP"
-   },
-   "source": [
-    "# Mozilla TTS on CPU Real-Time Speech Synthesis with TFLite"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "FAqrSIWgLyP0"
-   },
-   "source": [
-    "**These models are converted from released [PyTorch models](https://colab.research.google.com/drive/1u_16ZzHjKYFn1HNVuA4Qf_i2MMFB9olY?usp=sharing) using our TF utilities provided in Mozilla TTS.**\n",
-    "\n",
-    "#### **Notebook Details**\n",
-    "These TFLite models support TF 2.3rc0 and for different versions you might need to regenerate them. \n",
-    "\n",
-    "TFLite optimizations degrades the TTS model performance and we do not apply\n",
-    "any optimization for the vocoder model due to the same reason. If you like to\n",
-    "keep the quality, consider to regenerate TFLite model accordingly.\n",
-    "\n",
-    "Models optimized with TFLite can be slow on a regular CPU since it is optimized\n",
-    "specifically for lower-end systems.\n",
-    "\n",
-    "---\n",
-    "\n",
-    "\n",
-    "\n",
-    "#### **Model Details** \n",
-    "We use Tacotron2 and MultiBand-Melgan models and LJSpeech dataset.\n",
-    "\n",
-    "Tacotron2 is trained using [Double Decoder Consistency](https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency/) (DDC) only for 130K steps (3 days) with a single GPU.\n",
-    "\n",
-    "MultiBand-Melgan is trained  1.45M steps with real spectrograms.\n",
-    "\n",
-    "Note that both model performances can be improved with more training.\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "Ku-dA4DKoeXk"
-   },
-   "source": [
-    "### Download TF Models and configs"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 162
-    },
-    "colab_type": "code",
-    "id": "jGIgnWhGsxU1",
-    "outputId": "57af701e-77ec-400d-fee5-64aa7603d357"
-   },
-   "outputs": [],
-   "source": [
-    "!gdown --id 17PYXCmTe0el_SLTwznrt3vOArNGMGo5v -O tts_model.tflite\n",
-    "!gdown --id 18CQ6G6tBEOfvCHlPqP8EBI4xWbrr9dBc -O config.json"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 235
-    },
-    "colab_type": "code",
-    "id": "4dnpE0-kvTsu",
-    "outputId": "6aab0622-9add-4ee4-b9f8-177d6ddc0e86"
-   },
-   "outputs": [],
-   "source": [
-    "!gdown --id 1aXveT-NjOM1mUr6tM4JfWjshq67GvVIO -O vocoder_model.tflite\n",
-    "!gdown --id 1Rd0R_nRCrbjEdpOwq6XwZAktvugiBvmu -O config_vocoder.json\n",
-    "!gdown --id 11oY3Tv0kQtxK_JPgxrfesa99maVXHNxU -O scale_stats.npy"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "_ZuDrj_ioqHE"
-   },
-   "source": [
-    "### Setup Libraries"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 964
-    },
-    "colab_type": "code",
-    "id": "X2axt5BYq7gv",
-    "outputId": "aa53986f-f218-4d17-8667-0d74bb90c927"
-   },
-   "outputs": [],
-   "source": [
-    "# need it for char to phoneme conversion\n",
-    "! sudo apt-get install espeak"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 144
-    },
-    "colab_type": "code",
-    "id": "ZduAf-qYYEIT",
-    "outputId": "c1fcac0d-b8f8-442c-d598-4f549c42b698"
-   },
-   "outputs": [],
-   "source": [
-    "!git clone https://github.com/mozilla/TTS"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 1000
-    },
-    "colab_type": "code",
-    "id": "ofPCvPyjZEcT",
-    "outputId": "f3d3ea73-eae5-473c-db19-276bd0e721cc"
-   },
-   "outputs": [],
-   "source": [
-    "%cd TTS\n",
-    "!git checkout c7296b3\n",
-    "!pip install -r requirements.txt\n",
-    "!python setup.py install\n",
-    "!pip install tensorflow==2.3.0rc0\n",
-    "%cd .."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "Zlgi8fPdpRF0"
-   },
-   "source": [
-    "### Define TTS function"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "f-Yc42nQZG5A"
-   },
-   "outputs": [],
-   "source": [
-    "def run_vocoder(mel_spec):\n",
-    "  vocoder_inputs = mel_spec[None, :, :]\n",
-    "  # get input and output details\n",
-    "  input_details = vocoder_model.get_input_details()\n",
-    "  # reshape input tensor for the new input shape\n",
-    "  vocoder_model.resize_tensor_input(input_details[0]['index'], vocoder_inputs.shape)\n",
-    "  vocoder_model.allocate_tensors()\n",
-    "  detail = input_details[0]\n",
-    "  vocoder_model.set_tensor(detail['index'], vocoder_inputs)\n",
-    "  # run the model\n",
-    "  vocoder_model.invoke()\n",
-    "  # collect outputs\n",
-    "  output_details = vocoder_model.get_output_details()\n",
-    "  waveform = vocoder_model.get_tensor(output_details[0]['index'])\n",
-    "  return waveform \n",
-    "\n",
-    "\n",
-    "def tts(model, text, CONFIG, p):\n",
-    "    t_1 = time.time()\n",
-    "    waveform, alignment, mel_spec, mel_postnet_spec, stop_tokens, inputs = synthesis(model, text, CONFIG, use_cuda, ap, speaker_id, style_wav=None,\n",
-    "                                                                             truncated=False, enable_eos_bos_chars=CONFIG.enable_eos_bos_chars,\n",
-    "                                                                             backend='tflite')\n",
-    "    waveform = run_vocoder(mel_postnet_spec.T)\n",
-    "    waveform = waveform[0, 0]\n",
-    "    rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n",
-    "    tps = (time.time() - t_1) / len(waveform)\n",
-    "    print(waveform.shape)\n",
-    "    print(\" > Run-time: {}\".format(time.time() - t_1))\n",
-    "    print(\" > Real-time factor: {}\".format(rtf))\n",
-    "    print(\" > Time per step: {}\".format(tps))\n",
-    "    IPython.display.display(IPython.display.Audio(waveform, rate=CONFIG.audio['sample_rate']))  \n",
-    "    return alignment, mel_postnet_spec, stop_tokens, waveform"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "ZksegYQepkFg"
-   },
-   "source": [
-    "### Load TF Models"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "oVa0kOamprgj"
-   },
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import torch\n",
-    "import time\n",
-    "import IPython\n",
-    "\n",
-    "from TTS.tf.utils.tflite import load_tflite_model\n",
-    "from TTS.tf.utils.io import load_checkpoint\n",
-    "from TTS.utils.io import load_config\n",
-    "from TTS.utils.text.symbols import symbols, phonemes\n",
-    "from TTS.utils.audio import AudioProcessor\n",
-    "from TTS.tts.utils.synthesis import synthesis"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "EY-sHVO8IFSH"
-   },
-   "outputs": [],
-   "source": [
-    "# runtime settings\n",
-    "use_cuda = False"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "_1aIUp2FpxOQ"
-   },
-   "outputs": [],
-   "source": [
-    "# model paths\n",
-    "TTS_MODEL = \"tts_model.tflite\"\n",
-    "TTS_CONFIG = \"config.json\"\n",
-    "VOCODER_MODEL = \"vocoder_model.tflite\"\n",
-    "VOCODER_CONFIG = \"config_vocoder.json\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "CpgmdBVQplbv"
-   },
-   "outputs": [],
-   "source": [
-    "# load configs\n",
-    "TTS_CONFIG = load_config(TTS_CONFIG)\n",
-    "VOCODER_CONFIG = load_config(VOCODER_CONFIG)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 471
-    },
-    "colab_type": "code",
-    "id": "zmrQxiozIUVE",
-    "outputId": "ca7e9016-4c28-4cef-efe7-0613d399aa4c"
-   },
-   "outputs": [],
-   "source": [
-    "# load the audio processor\n",
-    "ap = AudioProcessor(**TTS_CONFIG.audio)         "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "8fLoI4ipqMeS"
-   },
-   "outputs": [],
-   "source": [
-    "# LOAD TTS MODEL\n",
-    "# multi speaker \n",
-    "speaker_id = None\n",
-    "speakers = []\n",
-    "\n",
-    "# load the models\n",
-    "model = load_tflite_model(TTS_MODEL)\n",
-    "vocoder_model = load_tflite_model(VOCODER_MODEL)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "Ws_YkPKsLgo-"
-   },
-   "source": [
-    "## Run Inference"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/",
-     "height": 134
-    },
-    "colab_type": "code",
-    "id": "FuWxZ9Ey5Puj",
-    "outputId": "d1888ebd-3208-42a4-aaf9-78d0e3ec987d"
-   },
-   "outputs": [],
-   "source": [
-    "sentence =  \"Bill got in the habit of asking himself “Is that thought true?” and if he wasn’t absolutely certain it was, he just let it go.\"\n",
-    "align, spec, stop_tokens, wav = tts(model, sentence, TTS_CONFIG, ap)"
-   ]
-  }
- ],
- "metadata": {
-  "colab": {
-   "collapsed_sections": [],
-   "name": "DDC-TTS_and_MultiBand-MelGAN_TFLite_Example.ipynb",
-   "provenance": [],
-   "toc_visible": true
-  },
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018.ipynb

@@ -1,650 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "yZK6UdwSFnOO"
-   },
-   "source": [
-    "# **Download and install Coqui TTS**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "yvb0pX3WY6MN"
-   },
-   "outputs": [],
-   "source": [
-    "import os \n",
-    "!git clone https://github.com/Edresson/TTS -b dev-gst-embeddings"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "iB9nl2UEG3SY"
-   },
-   "outputs": [],
-   "source": [
-    "!apt-get install espeak\n",
-    "os.chdir('TTS')\n",
-    "!pip install -r requirements.txt\n",
-    "!python setup.py develop\n",
-    "os.chdir('..')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "w6Krn8k1inC_"
-   },
-   "source": [
-    "\n",
-    "\n",
-    "**Download Checkpoint**\n",
-    "\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "PiYHf3lKhi9z"
-   },
-   "outputs": [],
-   "source": [
-    "!wget -c -q --show-progress -O ./TTS-checkpoint.zip  https://github.com/Edresson/TTS/releases/download/v1.0.0/Checkpoints-TTS-MultiSpeaker-Jia-et-al-2018.zip\n",
-    "!unzip ./TTS-checkpoint.zip\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "MpYNgqrZcJKn"
-   },
-   "source": [
-    "**Utils Functions**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "4KZA4b_CbMqx"
-   },
-   "outputs": [],
-   "source": [
-    "%load_ext autoreload\n",
-    "%autoreload 2\n",
-    "import argparse\n",
-    "import json\n",
-    "# pylint: disable=redefined-outer-name, unused-argument\n",
-    "import os\n",
-    "import string\n",
-    "import time\n",
-    "import sys\n",
-    "import numpy as np\n",
-    "\n",
-    "TTS_PATH = \"../content/TTS\"\n",
-    "# add libraries into environment\n",
-    "sys.path.append(TTS_PATH) # set this if TTS is not installed globally\n",
-    "\n",
-    "import torch\n",
-    "\n",
-    "from TTS.tts.utils.generic_utils import setup_model\n",
-    "from TTS.tts.utils.synthesis import synthesis\n",
-    "from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols\n",
-    "from TTS.utils.audio import AudioProcessor\n",
-    "from TTS.utils.io import load_config\n",
-    "from TTS.vocoder.utils.generic_utils import setup_generator\n",
-    "\n",
-    "\n",
-    "def tts(model, vocoder_model, text, CONFIG, use_cuda, ap, use_gl, speaker_fileid, speaker_embedding=None):\n",
-    "    t_1 = time.time()\n",
-    "    waveform, _, _, mel_postnet_spec, _, _ = synthesis(model, text, CONFIG, use_cuda, ap, speaker_fileid, None, False, CONFIG.enable_eos_bos_chars, use_gl, speaker_embedding=speaker_embedding)\n",
-    "    if CONFIG.model == \"Tacotron\" and not use_gl:\n",
-    "        mel_postnet_spec = ap.out_linear_to_mel(mel_postnet_spec.T).T\n",
-    "    if not use_gl:\n",
-    "        waveform = vocoder_model.inference(torch.FloatTensor(mel_postnet_spec.T).unsqueeze(0))\n",
-    "    if use_cuda and not use_gl:\n",
-    "        waveform = waveform.cpu()\n",
-    "    if not use_gl:\n",
-    "        waveform = waveform.numpy()\n",
-    "    waveform = waveform.squeeze()\n",
-    "    rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n",
-    "    tps = (time.time() - t_1) / len(waveform)\n",
-    "    print(\" > Run-time: {}\".format(time.time() - t_1))\n",
-    "    print(\" > Real-time factor: {}\".format(rtf))\n",
-    "    print(\" > Time per step: {}\".format(tps))\n",
-    "    return waveform\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "ENA2OumIVeMA"
-   },
-   "source": [
-    "# **Vars definitions**\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "jPD0d_XpVXmY"
-   },
-   "outputs": [],
-   "source": [
-    "TEXT = ''\n",
-    "OUT_PATH = 'tests-audios/'\n",
-    "# create output path\n",
-    "os.makedirs(OUT_PATH, exist_ok=True)\n",
-    "\n",
-    "SPEAKER_FILEID = None # if None use the first embedding from speakers.json\n",
-    "\n",
-    "# model vars \n",
-    "MODEL_PATH = 'best_model.pth.tar'\n",
-    "CONFIG_PATH = 'config.json'\n",
-    "SPEAKER_JSON = 'speakers.json'\n",
-    "\n",
-    "# vocoder vars\n",
-    "VOCODER_PATH = ''\n",
-    "VOCODER_CONFIG_PATH = ''\n",
-    "\n",
-    "USE_CUDA = True"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "dV6cXXlfi72r"
-   },
-   "source": [
-    "# **Restore  TTS Model**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "x1WgLFauWUPe"
-   },
-   "outputs": [],
-   "source": [
-    "# load the config\n",
-    "C = load_config(CONFIG_PATH)\n",
-    "C.forward_attn_mask = True\n",
-    "\n",
-    "# load the audio processor\n",
-    "ap = AudioProcessor(**C.audio)\n",
-    "\n",
-    "# if the vocabulary was passed, replace the default\n",
-    "if 'characters' in C.keys():\n",
-    "    symbols, phonemes = make_symbols(**C.characters)\n",
-    "\n",
-    "speaker_embedding = None\n",
-    "speaker_embedding_dim = None\n",
-    "num_speakers = 0\n",
-    "# load speakers\n",
-    "if SPEAKER_JSON != '':\n",
-    "    speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n",
-    "    num_speakers = len(speaker_mapping)\n",
-    "    if C.use_external_speaker_embedding_file:\n",
-    "        if SPEAKER_FILEID is not None:\n",
-    "            speaker_embedding = speaker_mapping[SPEAKER_FILEID]['embedding']\n",
-    "        else: # if speaker_fileid is not specificated use the first sample in speakers.json\n",
-    "            choise_speaker = list(speaker_mapping.keys())[0]\n",
-    "            print(\" Speaker: \",choise_speaker.split('_')[0],'was chosen automatically', \"(this speaker seen in training)\")\n",
-    "            speaker_embedding = speaker_mapping[choise_speaker]['embedding']\n",
-    "        speaker_embedding_dim = len(speaker_embedding)\n",
-    "\n",
-    "# load the model\n",
-    "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n",
-    "model = setup_model(num_chars, num_speakers, C, speaker_embedding_dim)\n",
-    "cp = torch.load(MODEL_PATH, map_location=torch.device('cpu'))\n",
-    "model.load_state_dict(cp['model'])\n",
-    "model.eval()\n",
-    "\n",
-    "if USE_CUDA:\n",
-    "    model.cuda()\n",
-    "\n",
-    "model.decoder.set_r(cp['r'])\n",
-    "\n",
-    "# load vocoder model\n",
-    "if VOCODER_PATH!= \"\":\n",
-    "    VC = load_config(VOCODER_CONFIG_PATH)\n",
-    "    vocoder_model = setup_generator(VC)\n",
-    "    vocoder_model.load_state_dict(torch.load(VOCODER_PATH, map_location=\"cpu\")[\"model\"])\n",
-    "    vocoder_model.remove_weight_norm()\n",
-    "    if USE_CUDA:\n",
-    "        vocoder_model.cuda()\n",
-    "    vocoder_model.eval()\n",
-    "else:\n",
-    "    vocoder_model = None\n",
-    "    VC = None\n",
-    "\n",
-    "# synthesize voice\n",
-    "use_griffin_lim = VOCODER_PATH== \"\"\n",
-    "\n",
-    "if not C.use_external_speaker_embedding_file:\n",
-    "    if SPEAKER_FILEID.isdigit():\n",
-    "        SPEAKER_FILEID = int(SPEAKER_FILEID)\n",
-    "    else:\n",
-    "        SPEAKER_FILEID = None\n",
-    "else:\n",
-    "    SPEAKER_FILEID = None\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "tNvVEoE30qY6"
-   },
-   "source": [
-    "Synthesize sentence with Speaker\n",
-    "\n",
-    "> Stop running the cell to leave!\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "2o8fXkVSyXOa"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with Speaker: \",choise_speaker.split('_')[0], \"(this speaker seen in training)\")\n",
-    "while True:\n",
-    "  TEXT = input(\"Enter sentence: \")\n",
-    "  print(\" > Text: {}\".format(TEXT))\n",
-    "  wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n",
-    "  IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "  # save the results\n",
-    "  file_name = TEXT.replace(\" \", \"_\")\n",
-    "  file_name = file_name.translate(\n",
-    "      str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "  out_path = os.path.join(OUT_PATH, file_name)\n",
-    "  print(\" > Saving output to {}\".format(out_path))\n",
-    "  ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "vnV-FigfvsS2"
-   },
-   "source": [
-    "# **Select Speaker**\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "RuCGOnJ_fgDV"
-   },
-   "outputs": [],
-   "source": [
-    "\n",
-    "# VCTK speakers not seen in training (new speakers)\n",
-    "VCTK_test_Speakers = [\"p225\", \"p234\", \"p238\", \"p245\", \"p248\", \"p261\", \"p294\", \"p302\", \"p326\", \"p335\", \"p347\"]\n",
-    "\n",
-    "# VCTK speakers seen in training\n",
-    "VCTK_train_Speakers = ['p244', 'p300', 'p303', 'p273', 'p292', 'p252', 'p254', 'p269', 'p345', 'p274', 'p363', 'p285', 'p351', 'p361', 'p295', 'p266', 'p307', 'p230', 'p339', 'p253', 'p310', 'p241', 'p256', 'p323', 'p237', 'p229', 'p298', 'p336', 'p276', 'p305', 'p255', 'p278', 'p299', 'p265', 'p267', 'p280', 'p260', 'p272', 'p262', 'p334', 'p283', 'p247', 'p246', 'p374', 'p297', 'p249', 'p250', 'p304', 'p240', 'p236', 'p312', 'p286', 'p263', 'p258', 'p313', 'p376', 'p279', 'p340', 'p362', 'p284', 'p231', 'p308', 'p277', 'p275', 'p333', 'p314', 'p330', 'p264', 'p226', 'p288', 'p343', 'p239', 'p232', 'p268', 'p270', 'p329', 'p227', 'p271', 'p228', 'p311', 'p301', 'p293', 'p364', 'p251', 'p317', 'p360', 'p281', 'p243', 'p287', 'p233', 'p259', 'p316', 'p257', 'p282', 'p306', 'p341', 'p318']\n",
-    "\n",
-    "\n",
-    "num_samples_speaker = 2 # In theory the more samples of the speaker the more similar to the real voice it will be!\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "hkvv7gRcx4WV"
-   },
-   "source": [
-    "## **Example select  a VCTK seen speaker in training**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "BviNMI9UyCYz"
-   },
-   "outputs": [],
-   "source": [
-    "# get embedding\n",
-    "Speaker_choise = VCTK_train_Speakers[0] # choise one of training speakers\n",
-    "# load speakers\n",
-    "if SPEAKER_JSON != '':\n",
-    "    speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n",
-    "    if C.use_external_speaker_embedding_file:\n",
-    "        speaker_embeddings = []\n",
-    "        for key in list(speaker_mapping.keys()):\n",
-    "          if Speaker_choise in key:\n",
-    "            if len(speaker_embeddings) < num_samples_speaker:\n",
-    "              speaker_embeddings.append(speaker_mapping[key]['embedding'])\n",
-    "        # takes the average of the embedings samples of the announcers\n",
-    "        speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n",
-    "        "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "5e5_XnLsx3jg"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker seen in training)\")\n",
-    "while True:\n",
-    "  TEXT = input(\"Enter sentence: \")\n",
-    "  print(\" > Text: {}\".format(TEXT))\n",
-    "  wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n",
-    "  IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "  # save the results\n",
-    "  file_name = TEXT.replace(\" \", \"_\")\n",
-    "  file_name = file_name.translate(\n",
-    "      str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "  out_path = os.path.join(OUT_PATH, file_name)\n",
-    "  print(\" > Saving output to {}\".format(out_path))\n",
-    "  ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "QJ6VgT2a4vHW"
-   },
-   "source": [
-    "## **Example select  a VCTK not seen speaker in training (new Speakers)**\n",
-    "\n",
-    "\n",
-    "> Fitting new Speakers :)\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "SZS57ZK-4vHa"
-   },
-   "outputs": [],
-   "source": [
-    "# get embedding\n",
-    "Speaker_choise = VCTK_test_Speakers[0] # choise one of training speakers\n",
-    "# load speakers\n",
-    "if SPEAKER_JSON != '':\n",
-    "    speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n",
-    "    if C.use_external_speaker_embedding_file:\n",
-    "        speaker_embeddings = []\n",
-    "        for key in list(speaker_mapping.keys()):\n",
-    "          if Speaker_choise in key:\n",
-    "            if len(speaker_embeddings) < num_samples_speaker:\n",
-    "              speaker_embeddings.append(speaker_mapping[key]['embedding'])\n",
-    "        # takes the average of the embedings samples of the announcers\n",
-    "        speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n",
-    "        "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "bbs85vzz4vHo"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker  not seen in training (new speaker))\")\n",
-    "while True:\n",
-    "  TEXT = input(\"Enter sentence: \")\n",
-    "  print(\" > Text: {}\".format(TEXT))\n",
-    "  wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n",
-    "  IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "  # save the results\n",
-    "  file_name = TEXT.replace(\" \", \"_\")\n",
-    "  file_name = file_name.translate(\n",
-    "      str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "  out_path = os.path.join(OUT_PATH, file_name)\n",
-    "  print(\" > Saving output to {}\".format(out_path))\n",
-    "  ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "LEE6mQLh5Who"
-   },
-   "source": [
-    "# **Example Synthesizing with your own voice :)**\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "La70gSB65nrs"
-   },
-   "source": [
-    " Download and load GE2E Speaker Encoder "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "r0IEFZ0B5vQg"
-   },
-   "outputs": [],
-   "source": [
-    "!wget -c -q --show-progress -O ./SpeakerEncoder-checkpoint.zip  https://github.com/Edresson/TTS/releases/download/v1.0.0/GE2E-SpeakerEncoder-iter25k.zip\n",
-    "!unzip ./SpeakerEncoder-checkpoint.zip"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "jEH8HCTh5mF6"
-   },
-   "outputs": [],
-   "source": [
-    "SE_MODEL_RUN_PATH = \"GE2E-SpeakerEncoder/\"\n",
-    "SE_MODEL_PATH = os.path.join(SE_MODEL_RUN_PATH, \"best_model.pth.tar\")\n",
-    "SE_CONFIG_PATH =os.path.join(SE_MODEL_RUN_PATH, \"config.json\")\n",
-    "USE_CUDA = True"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "tOwkfQqT6-Qo"
-   },
-   "outputs": [],
-   "source": [
-    "from TTS.utils.audio import AudioProcessor\n",
-    "from TTS.speaker_encoder.model import SpeakerEncoder\n",
-    "se_config = load_config(SE_CONFIG_PATH)\n",
-    "se_ap = AudioProcessor(**se_config['audio'])\n",
-    "\n",
-    "se_model = SpeakerEncoder(**se_config.model)\n",
-    "se_model.load_state_dict(torch.load(SE_MODEL_PATH)['model'])\n",
-    "se_model.eval()\n",
-    "if USE_CUDA:\n",
-    "    se_model.cuda()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "0TLlbUFG8O36"
-   },
-   "source": [
-    "Upload a wav audio file in your voice.\n",
-    "\n",
-    "\n",
-    "> We recommend files longer than 3 seconds, the bigger the file the closer to your voice :)\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "_FWwHPjJ8NXl"
-   },
-   "outputs": [],
-   "source": [
-    "from google.colab import files\n",
-    "file_list = files.upload()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "WWOf6sgbBbGY"
-   },
-   "outputs": [],
-   "source": [
-    "# extract embedding from wav files\n",
-    "speaker_embeddings = []\n",
-    "for name in file_list.keys():\n",
-    "    if '.wav' in name:\n",
-    "      mel_spec = se_ap.melspectrogram(se_ap.load_wav(name, sr=se_ap.sample_rate)).T\n",
-    "      mel_spec = torch.FloatTensor(mel_spec[None, :, :])\n",
-    "      if USE_CUDA:\n",
-    "          mel_spec = mel_spec.cuda()\n",
-    "      embedd = se_model.compute_embedding(mel_spec).cpu().detach().numpy().reshape(-1)\n",
-    "      speaker_embeddings.append(embedd)\n",
-    "    else:\n",
-    "      print(\" You need upload Wav files, others files is not supported !!\")\n",
-    "\n",
-    "# takes the average of the embedings samples of the announcers\n",
-    "speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "xmItcGac5WiG"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n",
-    "while True:\n",
-    "  TEXT = input(\"Enter sentence: \")\n",
-    "  print(\" > Text: {}\".format(TEXT))\n",
-    "  wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding)\n",
-    "  IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "  # save the results\n",
-    "  file_name = TEXT.replace(\" \", \"_\")\n",
-    "  file_name = file_name.translate(\n",
-    "      str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "  out_path = os.path.join(OUT_PATH, file_name)\n",
-    "  print(\" > Saving output to {}\".format(out_path))\n",
-    "  ap.save_wav(wav, out_path)"
-   ]
-  }
- ],
- "metadata": {
-  "accelerator": "GPU",
-  "colab": {
-   "collapsed_sections": [
-    "vnV-FigfvsS2",
-    "hkvv7gRcx4WV",
-    "QJ6VgT2a4vHW"
-   ],
-   "name": "Demo-Mozilla-TTS-MultiSpeaker-jia-et-al-2018.ipynb",
-   "provenance": []
-  },
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

notebooks/Demo_Coqui_TTS_MultiSpeaker_jia_et_al_2018_With_GST.ipynb

@@ -1,847 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "yZK6UdwSFnOO"
-   },
-   "source": [
-    "# **Download and install Coqui TTS**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "yvb0pX3WY6MN"
-   },
-   "outputs": [],
-   "source": [
-    "import os \n",
-    "!git clone https://github.com/Edresson/TTS -b dev-gst-embeddings"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "iB9nl2UEG3SY"
-   },
-   "outputs": [],
-   "source": [
-    "!apt-get install espeak\n",
-    "os.chdir('TTS')\n",
-    "!pip install -r requirements.txt\n",
-    "!python setup.py develop\n",
-    "os.chdir('..')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "w6Krn8k1inC_"
-   },
-   "source": [
-    "\n",
-    "\n",
-    "**Download Checkpoint**\n",
-    "\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "PiYHf3lKhi9z"
-   },
-   "outputs": [],
-   "source": [
-    "!wget -c -q --show-progress -O ./TTS-checkpoint.zip https://github.com/Edresson/TTS/releases/download/v1.0.0/Checkpoints-TTS-MultiSpeaker-Jia-et-al-2018-with-GST.zip\n",
-    "!unzip ./TTS-checkpoint.zip\n",
-    "\n",
-    "# Download gst style example\n",
-    "!wget https://github.com/Edresson/TTS/releases/download/v1.0.0/gst-style-example.wav"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "MpYNgqrZcJKn"
-   },
-   "source": [
-    "**Utils Functions**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "4KZA4b_CbMqx"
-   },
-   "outputs": [],
-   "source": [
-    "%load_ext autoreload\n",
-    "%autoreload 2\n",
-    "import argparse\n",
-    "import json\n",
-    "# pylint: disable=redefined-outer-name, unused-argument\n",
-    "import os\n",
-    "import string\n",
-    "import time\n",
-    "import sys\n",
-    "import numpy as np\n",
-    "\n",
-    "TTS_PATH = \"../content/TTS\"\n",
-    "# add libraries into environment\n",
-    "sys.path.append(TTS_PATH) # set this if TTS is not installed globally\n",
-    "\n",
-    "import torch\n",
-    "\n",
-    "from TTS.tts.utils.generic_utils import setup_model\n",
-    "from TTS.tts.utils.synthesis import synthesis\n",
-    "from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols\n",
-    "from TTS.utils.audio import AudioProcessor\n",
-    "from TTS.utils.io import load_config\n",
-    "from TTS.vocoder.utils.generic_utils import setup_generator\n",
-    "\n",
-    "\n",
-    "def tts(model, vocoder_model, text, CONFIG, use_cuda, ap, use_gl, speaker_fileid, speaker_embedding=None, gst_style=None):\n",
-    "    t_1 = time.time()\n",
-    "    waveform, _, _, mel_postnet_spec, _, _ = synthesis(model, text, CONFIG, use_cuda, ap, speaker_fileid, gst_style, False, CONFIG.enable_eos_bos_chars, use_gl, speaker_embedding=speaker_embedding)\n",
-    "    if CONFIG.model == \"Tacotron\" and not use_gl:\n",
-    "        mel_postnet_spec = ap.out_linear_to_mel(mel_postnet_spec.T).T\n",
-    "    if not use_gl:\n",
-    "        waveform = vocoder_model.inference(torch.FloatTensor(mel_postnet_spec.T).unsqueeze(0))\n",
-    "    if use_cuda and not use_gl:\n",
-    "        waveform = waveform.cpu()\n",
-    "    if not use_gl:\n",
-    "        waveform = waveform.numpy()\n",
-    "    waveform = waveform.squeeze()\n",
-    "    rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n",
-    "    tps = (time.time() - t_1) / len(waveform)\n",
-    "    print(\" > Run-time: {}\".format(time.time() - t_1))\n",
-    "    print(\" > Real-time factor: {}\".format(rtf))\n",
-    "    print(\" > Time per step: {}\".format(tps))\n",
-    "    return waveform\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "ENA2OumIVeMA"
-   },
-   "source": [
-    "# **Vars definitions**\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "jPD0d_XpVXmY"
-   },
-   "outputs": [],
-   "source": [
-    "TEXT = ''\n",
-    "OUT_PATH = 'tests-audios/'\n",
-    "# create output path\n",
-    "os.makedirs(OUT_PATH, exist_ok=True)\n",
-    "\n",
-    "SPEAKER_FILEID = None # if None use the first embedding from speakers.json\n",
-    "\n",
-    "# model vars \n",
-    "MODEL_PATH = 'best_model.pth.tar'\n",
-    "CONFIG_PATH = 'config.json'\n",
-    "SPEAKER_JSON = 'speakers.json'\n",
-    "\n",
-    "# vocoder vars\n",
-    "VOCODER_PATH = ''\n",
-    "VOCODER_CONFIG_PATH = ''\n",
-    "\n",
-    "USE_CUDA = True"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "dV6cXXlfi72r"
-   },
-   "source": [
-    "# **Restore  TTS Model**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "x1WgLFauWUPe"
-   },
-   "outputs": [],
-   "source": [
-    "# load the config\n",
-    "C = load_config(CONFIG_PATH)\n",
-    "C.forward_attn_mask = True\n",
-    "\n",
-    "# load the audio processor\n",
-    "ap = AudioProcessor(**C.audio)\n",
-    "\n",
-    "# if the vocabulary was passed, replace the default\n",
-    "if 'characters' in C.keys():\n",
-    "    symbols, phonemes = make_symbols(**C.characters)\n",
-    "\n",
-    "speaker_embedding = None\n",
-    "speaker_embedding_dim = None\n",
-    "num_speakers = 0\n",
-    "# load speakers\n",
-    "if SPEAKER_JSON != '':\n",
-    "    speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n",
-    "    num_speakers = len(speaker_mapping)\n",
-    "    if C.use_external_speaker_embedding_file:\n",
-    "        if SPEAKER_FILEID is not None:\n",
-    "            speaker_embedding = speaker_mapping[SPEAKER_FILEID]['embedding']\n",
-    "        else: # if speaker_fileid is not specificated use the first sample in speakers.json\n",
-    "            choise_speaker = list(speaker_mapping.keys())[0]\n",
-    "            print(\" Speaker: \",choise_speaker.split('_')[0],'was chosen automatically', \"(this speaker seen in training)\")\n",
-    "            speaker_embedding = speaker_mapping[choise_speaker]['embedding']\n",
-    "        speaker_embedding_dim = len(speaker_embedding)\n",
-    "\n",
-    "# load the model\n",
-    "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n",
-    "model = setup_model(num_chars, num_speakers, C, speaker_embedding_dim)\n",
-    "cp = torch.load(MODEL_PATH, map_location=torch.device('cpu'))\n",
-    "model.load_state_dict(cp['model'])\n",
-    "model.eval()\n",
-    "\n",
-    "if USE_CUDA:\n",
-    "    model.cuda()\n",
-    "\n",
-    "model.decoder.set_r(cp['r'])\n",
-    "\n",
-    "# load vocoder model\n",
-    "if VOCODER_PATH!= \"\":\n",
-    "    VC = load_config(VOCODER_CONFIG_PATH)\n",
-    "    vocoder_model = setup_generator(VC)\n",
-    "    vocoder_model.load_state_dict(torch.load(VOCODER_PATH, map_location=\"cpu\")[\"model\"])\n",
-    "    vocoder_model.remove_weight_norm()\n",
-    "    if USE_CUDA:\n",
-    "        vocoder_model.cuda()\n",
-    "    vocoder_model.eval()\n",
-    "else:\n",
-    "    vocoder_model = None\n",
-    "    VC = None\n",
-    "\n",
-    "# synthesize voice\n",
-    "use_griffin_lim = VOCODER_PATH== \"\"\n",
-    "\n",
-    "if not C.use_external_speaker_embedding_file:\n",
-    "    if SPEAKER_FILEID.isdigit():\n",
-    "        SPEAKER_FILEID = int(SPEAKER_FILEID)\n",
-    "    else:\n",
-    "        SPEAKER_FILEID = None\n",
-    "else:\n",
-    "    SPEAKER_FILEID = None\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "tNvVEoE30qY6"
-   },
-   "source": [
-    "Synthesize sentence with Speaker\n",
-    "\n",
-    "> Stop running the cell to leave!\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "2o8fXkVSyXOa"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with Speaker: \",choise_speaker.split('_')[0], \"(this speaker seen in training)\")\n",
-    "gst_style = 'gst-style-example.wav'\n",
-    "while True:\n",
-    "  TEXT = input(\"Enter sentence: \")\n",
-    "  print(\" > Text: {}\".format(TEXT))\n",
-    "  wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "  IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "  # save the results\n",
-    "  file_name = TEXT.replace(\" \", \"_\")\n",
-    "  file_name = file_name.translate(\n",
-    "      str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "  out_path = os.path.join(OUT_PATH, file_name)\n",
-    "  print(\" > Saving output to {}\".format(out_path))\n",
-    "  ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "vnV-FigfvsS2"
-   },
-   "source": [
-    "# **Select Speaker**\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "RuCGOnJ_fgDV"
-   },
-   "outputs": [],
-   "source": [
-    "\n",
-    "# VCTK speakers not seen in training (new speakers)\n",
-    "VCTK_test_Speakers = [\"p225\", \"p234\", \"p238\", \"p245\", \"p248\", \"p261\", \"p294\", \"p302\", \"p326\", \"p335\", \"p347\"]\n",
-    "\n",
-    "# VCTK speakers seen in training\n",
-    "VCTK_train_Speakers = ['p244', 'p300', 'p303', 'p273', 'p292', 'p252', 'p254', 'p269', 'p345', 'p274', 'p363', 'p285', 'p351', 'p361', 'p295', 'p266', 'p307', 'p230', 'p339', 'p253', 'p310', 'p241', 'p256', 'p323', 'p237', 'p229', 'p298', 'p336', 'p276', 'p305', 'p255', 'p278', 'p299', 'p265', 'p267', 'p280', 'p260', 'p272', 'p262', 'p334', 'p283', 'p247', 'p246', 'p374', 'p297', 'p249', 'p250', 'p304', 'p240', 'p236', 'p312', 'p286', 'p263', 'p258', 'p313', 'p376', 'p279', 'p340', 'p362', 'p284', 'p231', 'p308', 'p277', 'p275', 'p333', 'p314', 'p330', 'p264', 'p226', 'p288', 'p343', 'p239', 'p232', 'p268', 'p270', 'p329', 'p227', 'p271', 'p228', 'p311', 'p301', 'p293', 'p364', 'p251', 'p317', 'p360', 'p281', 'p243', 'p287', 'p233', 'p259', 'p316', 'p257', 'p282', 'p306', 'p341', 'p318']\n",
-    "\n",
-    "\n",
-    "num_samples_speaker = 2 # In theory the more samples of the speaker the more similar to the real voice it will be!\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "hkvv7gRcx4WV"
-   },
-   "source": [
-    "## **Example select  a VCTK seen speaker in training**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "BviNMI9UyCYz"
-   },
-   "outputs": [],
-   "source": [
-    "# get embedding\n",
-    "Speaker_choise = VCTK_train_Speakers[0] # choise one of training speakers\n",
-    "# load speakers\n",
-    "if SPEAKER_JSON != '':\n",
-    "    speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n",
-    "    if C.use_external_speaker_embedding_file:\n",
-    "        speaker_embeddings = []\n",
-    "        for key in list(speaker_mapping.keys()):\n",
-    "          if Speaker_choise in key:\n",
-    "            if len(speaker_embeddings) < num_samples_speaker:\n",
-    "              speaker_embeddings.append(speaker_mapping[key]['embedding'])\n",
-    "        # takes the average of the embedings samples of the announcers\n",
-    "        speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n",
-    "        "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "5e5_XnLsx3jg"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker seen in training)\")\n",
-    "gst_style = 'gst-style-example.wav'\n",
-    "while True:\n",
-    "  TEXT = input(\"Enter sentence: \")\n",
-    "  print(\" > Text: {}\".format(TEXT))\n",
-    "  wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "  IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "  # save the results\n",
-    "  file_name = TEXT.replace(\" \", \"_\")\n",
-    "  file_name = file_name.translate(\n",
-    "      str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "  out_path = os.path.join(OUT_PATH, file_name)\n",
-    "  print(\" > Saving output to {}\".format(out_path))\n",
-    "  ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "QJ6VgT2a4vHW"
-   },
-   "source": [
-    "## **Example select  a VCTK not seen speaker in training (new Speakers)**\n",
-    "\n",
-    "\n",
-    "> Fitting new Speakers :)\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "SZS57ZK-4vHa"
-   },
-   "outputs": [],
-   "source": [
-    "# get embedding\n",
-    "Speaker_choise = VCTK_test_Speakers[0] # choise one of training speakers\n",
-    "# load speakers\n",
-    "if SPEAKER_JSON != '':\n",
-    "    speaker_mapping = json.load(open(SPEAKER_JSON, 'r'))\n",
-    "    if C.use_external_speaker_embedding_file:\n",
-    "        speaker_embeddings = []\n",
-    "        for key in list(speaker_mapping.keys()):\n",
-    "          if Speaker_choise in key:\n",
-    "            if len(speaker_embeddings) < num_samples_speaker:\n",
-    "              speaker_embeddings.append(speaker_mapping[key]['embedding'])\n",
-    "        # takes the average of the embedings samples of the announcers\n",
-    "        speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()\n",
-    "        "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "bbs85vzz4vHo"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker not seen in training (new speaker))\")\n",
-    "gst_style = 'gst-style-example.wav'\n",
-    "while True:\n",
-    "  TEXT = input(\"Enter sentence: \")\n",
-    "  print(\" > Text: {}\".format(TEXT))\n",
-    "  wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "  IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "  # save the results\n",
-    "  file_name = TEXT.replace(\" \", \"_\")\n",
-    "  file_name = file_name.translate(\n",
-    "      str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "  out_path = os.path.join(OUT_PATH, file_name)\n",
-    "  print(\" > Saving output to {}\".format(out_path))\n",
-    "  ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "g_G_HweN04W-"
-   },
-   "source": [
-    "# **Changing GST tokens manually (without wav reference)**"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "jyFP5syW2bjt"
-   },
-   "source": [
-    "You can define tokens manually, this way you can increase/decrease the function of a given GST token. For example a token is responsible for the length of the speaker's pauses, if you increase the value of that token you will have longer pauses and if you decrease it you will have shorter pauses."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "SpwjDjCM2a3Y"
-   },
-   "outputs": [],
-   "source": [
-    "# set gst tokens, in this model we have 5 tokens\n",
-    "gst_style = {\"0\": 0, \"1\": 0, \"3\": 0, \"4\": 0}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "qWChMbI_0z5X"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker  not seen in training (new speaker))\")\n",
-    "TEXT = input(\"Enter sentence: \")\n",
-    "print(\" > Text: {}\".format(TEXT))\n",
-    "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "# save the results\n",
-    "file_name = TEXT.replace(\" \", \"_\")\n",
-    "file_name = file_name.translate(\n",
-    "    str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "out_path = os.path.join(OUT_PATH, file_name)\n",
-    "print(\" > Saving output to {}\".format(out_path))\n",
-    "ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "uFjUi9xQ3mG3"
-   },
-   "outputs": [],
-   "source": [
-    "gst_style = {\"0\": 0.9, \"1\": 0, \"3\": 0, \"4\": 0}\n",
-    "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker  not seen in training (new speaker))\")\n",
-    "TEXT = input(\"Enter sentence: \")\n",
-    "print(\" > Text: {}\".format(TEXT))\n",
-    "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "# save the results\n",
-    "file_name = TEXT.replace(\" \", \"_\")\n",
-    "file_name = file_name.translate(\n",
-    "    str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "out_path = os.path.join(OUT_PATH, file_name)\n",
-    "print(\" > Saving output to {}\".format(out_path))\n",
-    "ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "Uw0d6gWg4L27"
-   },
-   "outputs": [],
-   "source": [
-    "gst_style = {\"0\": -0.9, \"1\": 0, \"3\": 0, \"4\": 0}\n",
-    "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker  not seen in training (new speaker))\")\n",
-    "TEXT = input(\"Enter sentence: \")\n",
-    "print(\" > Text: {}\".format(TEXT))\n",
-    "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "# save the results\n",
-    "file_name = TEXT.replace(\" \", \"_\")\n",
-    "file_name = file_name.translate(\n",
-    "    str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "out_path = os.path.join(OUT_PATH, file_name)\n",
-    "print(\" > Saving output to {}\".format(out_path))\n",
-    "ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "V9izw4-54-Tl"
-   },
-   "outputs": [],
-   "source": [
-    "gst_style = {\"0\": 0, \"1\": 0.9, \"3\": 0, \"4\": 0}\n",
-    "print(\"Synthesize sentence with Speaker: \",Speaker_choise.split('_')[0], \"(this speaker  not seen in training (new speaker))\")\n",
-    "TEXT = input(\"Enter sentence: \")\n",
-    "print(\" > Text: {}\".format(TEXT))\n",
-    "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "# save the results\n",
-    "file_name = TEXT.replace(\" \", \"_\")\n",
-    "file_name = file_name.translate(\n",
-    "    str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "out_path = os.path.join(OUT_PATH, file_name)\n",
-    "print(\" > Saving output to {}\".format(out_path))\n",
-    "ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "LEE6mQLh5Who"
-   },
-   "source": [
-    "# **Example Synthesizing with your own voice :)**\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "La70gSB65nrs"
-   },
-   "source": [
-    " Download and load GE2E Speaker Encoder "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "r0IEFZ0B5vQg"
-   },
-   "outputs": [],
-   "source": [
-    "!wget -c -q --show-progress -O ./SpeakerEncoder-checkpoint.zip  https://github.com/Edresson/TTS/releases/download/v1.0.0/GE2E-SpeakerEncoder-iter25k.zip\n",
-    "!unzip ./SpeakerEncoder-checkpoint.zip"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "jEH8HCTh5mF6"
-   },
-   "outputs": [],
-   "source": [
-    "SE_MODEL_RUN_PATH = \"GE2E-SpeakerEncoder/\"\n",
-    "SE_MODEL_PATH = os.path.join(SE_MODEL_RUN_PATH, \"best_model.pth.tar\")\n",
-    "SE_CONFIG_PATH =os.path.join(SE_MODEL_RUN_PATH, \"config.json\")\n",
-    "USE_CUDA = True"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "tOwkfQqT6-Qo"
-   },
-   "outputs": [],
-   "source": [
-    "from TTS.utils.audio import AudioProcessor\n",
-    "from TTS.speaker_encoder.model import SpeakerEncoder\n",
-    "se_config = load_config(SE_CONFIG_PATH)\n",
-    "se_ap = AudioProcessor(**se_config['audio'])\n",
-    "\n",
-    "se_model = SpeakerEncoder(**se_config.model)\n",
-    "se_model.load_state_dict(torch.load(SE_MODEL_PATH)['model'])\n",
-    "se_model.eval()\n",
-    "if USE_CUDA:\n",
-    "    se_model.cuda()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "0TLlbUFG8O36"
-   },
-   "source": [
-    "Upload one or more wav audio files in your voice.\n",
-    "\n",
-    "\n",
-    "> We recommend files longer than 3 seconds, the bigger the file the closer to your voice :)\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "_FWwHPjJ8NXl"
-   },
-   "outputs": [],
-   "source": [
-    "# select one or more wav files\n",
-    "from google.colab import files\n",
-    "file_list = files.upload()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "WWOf6sgbBbGY"
-   },
-   "outputs": [],
-   "source": [
-    "# extract embedding from wav files\n",
-    "speaker_embeddings = []\n",
-    "for name in file_list.keys():\n",
-    "    if '.wav' in name:\n",
-    "      mel_spec = se_ap.melspectrogram(se_ap.load_wav(name, sr=se_ap.sample_rate)).T\n",
-    "      mel_spec = torch.FloatTensor(mel_spec[None, :, :])\n",
-    "      if USE_CUDA:\n",
-    "          mel_spec = mel_spec.cuda()\n",
-    "      embedd = se_model.compute_embedding(mel_spec).cpu().detach().numpy().reshape(-1)\n",
-    "      speaker_embeddings.append(embedd)\n",
-    "    else:\n",
-    "      print(\"You need upload Wav files, others files is not supported !!\")\n",
-    "\n",
-    "# takes the average of the embedings samples of the announcers\n",
-    "speaker_embedding = np.mean(np.array(speaker_embeddings), axis=0).tolist()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "AQ7eP31d9yzq"
-   },
-   "outputs": [],
-   "source": [
-    "import IPython\n",
-    "from IPython.display import Audio\n",
-    "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n",
-    "gst_style = {\"0\": 0, \"1\": 0.0, \"3\": 0, \"4\": 0}\n",
-    "gst_style = 'gst-style-example.wav'\n",
-    "TEXT = input(\"Enter sentence: \")\n",
-    "print(\" > Text: {}\".format(TEXT))\n",
-    "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "# save the results\n",
-    "file_name = TEXT.replace(\" \", \"_\")\n",
-    "file_name = file_name.translate(\n",
-    "    str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "out_path = os.path.join(OUT_PATH, file_name)\n",
-    "print(\" > Saving output to {}\".format(out_path))\n",
-    "ap.save_wav(wav, out_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "11i10yE1-LMJ"
-   },
-   "source": [
-    "Uploading your own GST reference wav file"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "eKohSQG1-KkT"
-   },
-   "outputs": [],
-   "source": [
-    "# select one wav file for GST reference\n",
-    "from google.colab import files\n",
-    "file_list = files.upload()\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "xmItcGac5WiG"
-   },
-   "outputs": [],
-   "source": [
-    "print(\"Synthesize sentence with New Speaker using files: \",file_list.keys(), \"(this speaker not seen in training (new speaker))\")\n",
-    "gst_style = list(file_list.keys())[0]\n",
-    "TEXT = input(\"Enter sentence: \")\n",
-    "print(\" > Text: {}\".format(TEXT))\n",
-    "wav = tts(model, vocoder_model, TEXT, C, USE_CUDA, ap, use_griffin_lim, SPEAKER_FILEID, speaker_embedding=speaker_embedding, gst_style=gst_style)\n",
-    "IPython.display.display(Audio(wav, rate=ap.sample_rate))\n",
-    "# save the results\n",
-    "file_name = TEXT.replace(\" \", \"_\")\n",
-    "file_name = file_name.translate(\n",
-    "    str.maketrans('', '', string.punctuation.replace('_', ''))) + '.wav'\n",
-    "out_path = os.path.join(OUT_PATH, file_name)\n",
-    "print(\" > Saving output to {}\".format(out_path))\n",
-    "ap.save_wav(wav, out_path)"
-   ]
-  }
- ],
- "metadata": {
-  "accelerator": "GPU",
-  "colab": {
-   "collapsed_sections": [
-    "yZK6UdwSFnOO",
-    "ENA2OumIVeMA",
-    "dV6cXXlfi72r",
-    "vnV-FigfvsS2",
-    "g_G_HweN04W-",
-    "LEE6mQLh5Who"
-   ],
-   "name": "Demo-Mozilla-TTS-MultiSpeaker-jia-et-al-2018-With-GST.ipynb",
-   "provenance": [],
-   "toc_visible": true
-  },
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

recipes/README.md

@@ -0,0 +1,13 @@
+# 🐸💬 TTS Training Recipes
+
+TTS recipes intended to host bash scripts running all the necessary steps to train a TTS model with a particular dataset.
+
+Run each script from the root TTS folder as follows
+
+```console
+$ bash ./recipes/<dataset>/<model>/run.sh
+```
+
+All the outputs are held under the recipe directory unless you change the paths in the bash script.
+
+If you train a new model using TTS, feel free to share your training to expand the list of recipes.

recipes/ljspeech/tacotron2-DDC/run.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# take the scripts's parent's directory to prefix all the output paths.
+RUN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+echo $RUN_DIR
+# download LJSpeech dataset
+wget http://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2
+# extract
+tar -xjf LJSpeech-1.1.tar.bz2
+# create train-val splits
+shuf LJSpeech-1.1/metadata.csv > LJSpeech-1.1/metadata_shuf.csv
+head -n 12000 LJSpeech-1.1/metadata_shuf.csv > LJSpeech-1.1/metadata_train.csv
+tail -n 1100 LJSpeech-1.1/metadata_shuf.csv > LJSpeech-1.1/metadata_val.csv
+mv LJSpeech-1.1 $RUN_DIR/
+rm LJSpeech-1.1.tar.bz2
+# compute dataset mean and variance for normalization
+python TTS/bin/compute_statistics.py $RUN_DIR/tacotron2-DDC.json $RUN_DIR/scale_stats.npy --data_path $RUN_DIR/LJSpeech-1.1/wavs/
+# training ....
+# change the GPU id if needed
+CUDA_VISIBLE_DEVICES="0" python TTS/bin/train_tacotron.py --config_path $RUN_DIR/tacotron2-DDC.json \
+                                                          --coqpit.output_path $RUN_DIR  \
+                                                          --coqpit.datasets.0.path $RUN_DIR/LJSpeech-1.1/    \
+                                                          --coqpit.audio.stats_path $RUN_DIR/scale_stats.npy \

recipes/ljspeech/tacotron2-DDC/tacotron2-DDC.json

@@ -0,0 +1,91 @@
+{
+    "datasets": [
+        {
+            "name": "ljspeech",
+            "path": "DEFINE THIS",
+            "meta_file_train": "metadata.csv",
+            "meta_file_val": null
+        }
+    ],
+    "audio": {
+        "fft_size": 1024,
+        "win_length": 1024,
+        "hop_length": 256,
+        "frame_length_ms": null,
+        "frame_shift_ms": null,
+        "sample_rate": 22050,
+        "preemphasis": 0.0,
+        "ref_level_db": 20,
+        "do_trim_silence": true,
+        "trim_db": 60,
+        "power": 1.5,
+        "griffin_lim_iters": 60,
+        "num_mels": 80,
+        "mel_fmin": 50.0,
+        "mel_fmax": 7600.0,
+        "spec_gain": 1,
+        "signal_norm": true,
+        "min_level_db": -100,
+        "symmetric_norm": true,
+        "max_norm": 4.0,
+        "clip_norm": true,
+        "stats_path": "scale_stats.npy"
+    },
+    "gst":{
+        "gst_embedding_dim": 256,
+        "gst_num_heads": 4,
+        "gst_num_style_tokens": 10
+    },
+    "model": "Tacotron2",
+    "run_name": "ljspeech-ddc",
+    "run_description": "tacotron2 with double decoder consistency.",
+    "batch_size": 64,
+    "eval_batch_size": 16,
+    "mixed_precision": true,
+    "loss_masking": true,
+    "decoder_loss_alpha": 0.25,
+    "postnet_loss_alpha": 0.25,
+    "postnet_diff_spec_alpha": 0.25,
+    "decoder_diff_spec_alpha": 0.25,
+    "decoder_ssim_alpha": 0.25,
+    "postnet_ssim_alpha": 0.25,
+    "ga_alpha": 5.0,
+    "stopnet_pos_weight": 15.0,
+    "run_eval": true,
+    "test_delay_epochs": 10,
+    "test_sentences_file": null,
+    "noam_schedule": true,
+    "grad_clip": 0.05,
+    "epochs": 1000,
+    "lr": 0.001,
+    "wd": 1e-06,
+    "warmup_steps": 4000,
+    "memory_size": -1,
+    "prenet_type": "original",
+    "prenet_dropout": true,
+    "attention_type": "original",
+    "location_attn": true,
+    "double_decoder_consistency": true,
+    "ddc_r": 6,
+    "attention_norm": "sigmoid",
+    "r": 6,
+    "gradual_training": [[0, 6, 64], [10000, 4, 32], [50000, 3, 32], [100000, 2, 32]],
+    "stopnet": true,
+    "separate_stopnet": true,
+    "print_step": 25,
+    "tb_plot_step": 100,
+    "print_eval": false,
+    "save_step": 10000,
+    "checkpoint": true,
+    "text_cleaner": "phoneme_cleaners",
+    "num_loader_workers": 4,
+    "num_val_loader_workers": 4,
+    "batch_group_size": 4,
+    "min_seq_len": 6,
+    "max_seq_len": 180,
+    "compute_input_seq_cache": true,
+    "output_path": "DEFINE THIS",
+    "phoneme_cache_path": "DEFINE THIS",
+    "use_phonemes": false,
+    "phoneme_language": "en-us"
+}

requirements.txt

@@ -7,7 +7,6 @@ librosa==0.8.0
 matplotlib
 numpy==1.18.5
 pandas
-phonemizer>=2.2.0
 pypinyin
 pysbd
 pyyaml
@@ -16,5 +15,7 @@ soundfile
 tensorboardX
 torch>=1.7
 tqdm
+numba==0.52
 umap-learn==0.4.6
 unidecode==0.4.20
+coqpit

run_bash_tests.sh

@@ -2,13 +2,7 @@ set -e
 TF_CPP_MIN_LOG_LEVEL=3
 
 # runtime bash based tests
+# TODO: move these to python
 ./tests/bash_tests/test_demo_server.sh && \
 ./tests/bash_tests/test_resample.sh && \
-./tests/bash_tests/test_tacotron_train.sh && \
-./tests/bash_tests/test_glow-tts_train.sh && \
-./tests/bash_tests/test_vocoder_gan_train.sh && \
-./tests/bash_tests/test_vocoder_wavernn_train.sh && \
-./tests/bash_tests/test_vocoder_wavegrad_train.sh && \
-./tests/bash_tests/test_speedy_speech_train.sh && \
-./tests/bash_tests/test_aligntts_train.sh && \
 ./tests/bash_tests/test_compute_statistics.sh

setup.py

@@ -4,6 +4,7 @@ import os
 import subprocess
 import sys
 from distutils.version import LooseVersion
+from TTS._version import __version__
 
 import numpy
 import setuptools.command.build_py
@@ -18,7 +19,7 @@ if LooseVersion(sys.version) < LooseVersion("3.6") or LooseVersion(sys.version)
     )
 
 
-version = '0.0.13.2'
+version = __version__
 cwd = os.path.dirname(os.path.abspath(__file__))
 
 class build_py(setuptools.command.build_py.build_py):  # pylint: disable=too-many-ancestors

tests/__init__.py

@@ -1,5 +1,16 @@
 import os
 
+from TTS.utils.generic_utils import get_cuda
+
+
+def get_device_id():
+    use_cuda, _ = get_cuda()
+    if use_cuda:
+        GPU_ID = "0"
+    else:
+        GPU_ID = ""
+    return GPU_ID
+
 
 def get_tests_path():
     """Returns the path to the test directory."""

tests/bash_tests/test_aligntts_train.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -xe
-BASEDIR=$(dirname "$0")
-echo "$BASEDIR"
-# run training
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_align_tts.py --config_path $BASEDIR/../inputs/test_align_tts.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_align_tts.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/

tests/bash_tests/test_glow-tts_train.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -xe
-BASEDIR=$(dirname "$0")
-echo "$BASEDIR"
-# run training
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_glow_tts.py --config_path $BASEDIR/../inputs/test_glow_tts.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_glow_tts.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/

tests/bash_tests/test_speedy_speech_train.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -xe
-BASEDIR=$(dirname "$0")
-echo "$BASEDIR"
-# run training
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_speedy_speech.py --config_path $BASEDIR/../inputs/test_speedy_speech.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_speedy_speech.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/

tests/bash_tests/test_tacotron_train.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-set -xe
-BASEDIR=$(dirname "$0")
-echo "$BASEDIR"
-
-# run training
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron_config.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/
-
-# run Tacotron bi-directional decoder
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron_bd_config.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/
-
-# Tacotron2
-# run training
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/../inputs/test_tacotron2_config.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_tacotron.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/
-

tests/bash_tests/test_vocoder_gan_train.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-set -xe
-BASEDIR=$(dirname "$0")
-echo "$BASEDIR"
-# create run dir
-mkdir $BASEDIR/../train_outputs
-# run training
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_gan.py --config_path $BASEDIR/../inputs/test_vocoder_multiband_melgan_config.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_vocoder_gan.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER

tests/bash_tests/test_vocoder_wavegrad_train.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-set -xe
-BASEDIR=$(dirname "$0")
-echo "$BASEDIR"
-# create run dir
-mkdir -p $BASEDIR/../train_outputs
-# run training
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavegrad.py --config_path $BASEDIR/../inputs/test_vocoder_wavegrad.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_vocoder_wavegrad.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER

tests/bash_tests/test_vocoder_wavernn_train.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-set -xe
-BASEDIR=$(dirname "$0")
-echo "$BASEDIR"
-# create run dir
-mkdir -p $BASEDIR/../train_outputs
-# run training
-CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavernn.py --config_path $BASEDIR/../inputs/test_vocoder_wavernn_config.json
-# find the training folder
-LATEST_FOLDER=$(ls $BASEDIR/../train_outputs/| sort | tail -1)
-echo $LATEST_FOLDER
-# continue the previous training
-CUDA_VISIBLE_DEVICES=""  python TTS/bin/train_vocoder_wavernn.py --continue_path $BASEDIR/../train_outputs/$LATEST_FOLDER
-# remove all the outputs
-rm -rf $BASEDIR/../train_outputs/$LATEST_FOLDER