diff --git a/config_tacotron.json b/config_tacotron.json index 61435ee7..ee648b6f 100644 --- a/config_tacotron.json +++ b/config_tacotron.json @@ -40,11 +40,12 @@ "windowing": false, // Enables attention windowing. Used only in eval mode. "memory_size": 5, // ONLY TACOTRON - memory queue size used to queue network predictions to feed autoregressive connection. Useful if r < 5. "attention_norm": "sigmoid", // softmax or sigmoid. Suggested to use softmax for Tacotron2 and sigmoid for Tacotron. - "prenet_type": "bn", // ONLY TACOTRON2 - "original" or "bn". - "prenet_dropout": true, // ONLY TACOTRON2 - enable/disable dropout at prenet. - "use_forward_attn": true, // ONLY TACOTRON2 - if it uses forward attention. In general, it aligns faster. - "transition_agent": true, // ONLY TACOTRON2 - enable/disable transition agent of forward attention. - "location_attn": false, // ONLY TACOTRON2 - enable_disable location sensitive attention. It is enabled for TACOTRON by default. + "prenet_type": "original", // "original" or "bn". + "prenet_dropout": true, // enable/disable dropout at prenet. + "use_forward_attn": true, // if it uses forward attention. In general, it aligns faster. + "forward_attn_mask": false, // Apply forward attention mask af inference to prevent bad modes. Try it if your model does not align well. + "transition_agent": true, // enable/disable transition agent of forward attention. + "location_attn": false, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. "loss_masking": true, // enable / disable loss masking against the sequence padding. "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. "stopnet": true, // Train stopnet predicting the end of synthesis. diff --git a/config_cluster.json b/config_tacotron2.json similarity index 97% rename from config_cluster.json rename to config_tacotron2.json index 420b03ed..e07f79c3 100644 --- a/config_cluster.json +++ b/config_tacotron2.json @@ -42,6 +42,7 @@ "prenet_type": "original", // ONLY TACOTRON2 - "original" or "bn". "prenet_dropout": true, // ONLY TACOTRON2 - enable/disable dropout at prenet. "use_forward_attn": true, // ONLY TACOTRON2 - if it uses forward attention. In general, it aligns faster. + "forward_attn_mask": false, // Apply forward attention mask af inference to prevent bad modes. Try it if your model does not align well. "transition_agent": false, // ONLY TACOTRON2 - enable/disable transition agent of forward attention. "location_attn": false, // ONLY TACOTRON2 - enable_disable location sensitive attention. It is enabled for TACOTRON by default. "loss_masking": true, // enable / disable loss masking against the sequence padding. diff --git a/config_tacotron_gst.json b/config_tacotron_gst.json new file mode 100644 index 00000000..84924f69 --- /dev/null +++ b/config_tacotron_gst.json @@ -0,0 +1,81 @@ +{ + "run_name": "mozilla-tacotron-gst", + "run_description": "", + + "audio":{ + // Audio processing parameters + "num_mels": 80, // size of the mel spec frame. + "num_freq": 1025, // number of stft frequency levels. Size of the linear spectogram frame. + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "frame_length_ms": 50, // stft window length in ms. + "frame_shift_ms": 12.5, // stft window hop-lengh in ms. + "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "min_level_db": -100, // normalization range + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + // Normalization parameters + "signal_norm": true, // normalize the spec values in range [0, 1] + "symmetric_norm": false, // move normalization to range [-1, 1] + "max_norm": 1, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! + "do_trim_silence": true // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + }, + + "distributed":{ + "backend": "nccl", + "url": "tcp:\/\/localhost:54321" + }, + + "reinit_layers": [], + + "model": "TacotronGST", // one of the model in models/ + "grad_clip": 1, // upper limit for gradients for clipping. + "epochs": 10000, // total number of epochs to train. + "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. + "lr_decay": false, // if true, Noam learning rate decaying is applied through training. + "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "windowing": false, // Enables attention windowing. Used only in eval mode. + "memory_size": 5, // ONLY TACOTRON - memory queue size used to queue network predictions to feed autoregressive connection. Useful if r < 5. + "attention_norm": "sigmoid", // softmax or sigmoid. Suggested to use softmax for Tacotron2 and sigmoid for Tacotron. + "prenet_type": "original", // "original" or "bn". + "prenet_dropout": true, // enable/disable dropout at prenet. + "use_forward_attn": true, // if it uses forward attention. In general, it aligns faster. + "forward_attn_mask": false, // Apply forward attention mask af inference to prevent bad modes. Try it if your model does not align well. + "transition_agent": true, // enable/disable transition agent of forward attention. + "location_attn": false, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. + "loss_masking": true, // enable / disable loss masking against the sequence padding. + "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. + "stopnet": true, // Train stopnet predicting the end of synthesis. + "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. + "eval_batch_size":16, + "r": 5, // Number of frames to predict for step. + "wd": 0.000001, // Weight decay weight. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. + "print_step": 10, // Number of steps to log traning on console. + "batch_group_size": 0, //Number of batches to shuffle after bucketing. + + "run_eval": true, + "test_delay_epochs": 5, //Until attention is aligned, testing only wastes computation time. + "test_sentences_file": null, + "data_path": "/media/erogol/data_ssd/Data/Mozilla/", // DATASET-RELATED: can overwritten from command argument + "meta_file_train": "metadata_train.txt", // DATASET-RELATED: metafile for training dataloader. + "meta_file_val": "metadata_val.txt", // DATASET-RELATED: metafile for evaluation dataloader. + "dataset": "mozilla", // DATASET-RELATED: one of TTS.dataset.preprocessors depending on your target dataset. Use "tts_cache" for pre-computed dataset by extract_features.py + "min_seq_len": 0, // DATASET-RELATED: minimum text length to use in training + "max_seq_len": 150, // DATASET-RELATED: maximum text length + "output_path": "../keep/", // DATASET-RELATED: output path for all training outputs. + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "phoneme_cache_path": "mozilla_us_phonemes", // phoneme computation is slow, therefore, it caches results in the given folder. + "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages + "text_cleaner": "phoneme_cleaners" + } + \ No newline at end of file diff --git a/train.py b/train.py index f2490b4f..77857946 100644 --- a/train.py +++ b/train.py @@ -89,7 +89,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler, # setup input data text_input = data[0] text_lengths = data[1] - linear_input = data[2] if c.model == "Tacotron" else None + linear_input = data[2] if c.model in ["Tacotron", "TacotronGST"] else None mel_input = data[3] mel_lengths = data[4] stop_targets = data[5] @@ -116,7 +116,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler, text_lengths = text_lengths.cuda(non_blocking=True) mel_input = mel_input.cuda(non_blocking=True) mel_lengths = mel_lengths.cuda(non_blocking=True) - linear_input = linear_input.cuda(non_blocking=True) if c.model == "Tacotron" else None + linear_input = linear_input.cuda(non_blocking=True) if c.model in ["Tacotron", "TacotronGST"] else None stop_targets = stop_targets.cuda(non_blocking=True) # forward pass model @@ -127,13 +127,13 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler, stop_loss = criterion_st(stop_tokens, stop_targets) if c.stopnet else torch.zeros(1) if c.loss_masking: decoder_loss = criterion(decoder_output, mel_input, mel_lengths) - if c.model == "Tacotron": + if c.model in ["Tacotron", "TacotronGST"]: postnet_loss = criterion(postnet_output, linear_input, mel_lengths) else: postnet_loss = criterion(postnet_output, mel_input, mel_lengths) else: decoder_loss = criterion(decoder_output, mel_input) - if c.model == "Tacotron": + if c.model in ["Tacotron", "TacotronGST"]: postnet_loss = criterion(postnet_output, linear_input) else: postnet_loss = criterion(postnet_output, mel_input) @@ -199,7 +199,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler, # Diagnostic visualizations const_spec = postnet_output[0].data.cpu().numpy() - gt_spec = linear_input[0].data.cpu().numpy() if c.model == "Tacotron" else mel_input[0].data.cpu().numpy() + gt_spec = linear_input[0].data.cpu().numpy() if c.model in ["Tacotron", "TacotronGST"] else mel_input[0].data.cpu().numpy() align_img = alignments[0].data.cpu().numpy() figures = { @@ -210,7 +210,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler, tb_logger.tb_train_figures(current_step, figures) # Sample audio - if c.model == "Tacotron": + if c.model in ["Tacotron", "TacotronGST"]: train_audio = ap.inv_spectrogram(const_spec.T) else: train_audio = ap.inv_mel_spectrogram(const_spec.T) @@ -273,7 +273,7 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch): # setup input data text_input = data[0] text_lengths = data[1] - linear_input = data[2] if c.model == "Tacotron" else None + linear_input = data[2] if c.model in ["Tacotron", "TacotronGST"] else None mel_input = data[3] mel_lengths = data[4] stop_targets = data[5] @@ -289,7 +289,7 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch): text_input = text_input.cuda() mel_input = mel_input.cuda() mel_lengths = mel_lengths.cuda() - linear_input = linear_input.cuda() if c.model == "Tacotron" else None + linear_input = linear_input.cuda() if c.model in ["Tacotron", "TacotronGST"] else None stop_targets = stop_targets.cuda() # forward pass @@ -300,13 +300,13 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch): stop_loss = criterion_st(stop_tokens, stop_targets) if c.stopnet else torch.zeros(1) if c.loss_masking: decoder_loss = criterion(decoder_output, mel_input, mel_lengths) - if c.model == "Tacotron": + if c.model in ["Tacotron", "TacotronGST"]: postnet_loss = criterion(postnet_output, linear_input, mel_lengths) else: postnet_loss = criterion(postnet_output, mel_input, mel_lengths) else: decoder_loss = criterion(decoder_output, mel_input) - if c.model == "Tacotron": + if c.model in ["Tacotron", "TacotronGST"]: postnet_loss = criterion(postnet_output, linear_input) else: postnet_loss = criterion(postnet_output, mel_input) @@ -339,7 +339,7 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch): # Diagnostic visualizations idx = np.random.randint(mel_input.shape[0]) const_spec = postnet_output[idx].data.cpu().numpy() - gt_spec = linear_input[idx].data.cpu().numpy() if c.model == "Tacotron" else mel_input[idx].data.cpu().numpy() + gt_spec = linear_input[idx].data.cpu().numpy() if c.model in ["Tacotron", "TacotronGST"] else mel_input[idx].data.cpu().numpy() align_img = alignments[idx].data.cpu().numpy() eval_figures = { @@ -350,7 +350,7 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch): tb_logger.tb_eval_figures(current_step, eval_figures) # Sample audio - if c.model == "Tacotron": + if c.model in ["Tacotron", "TacotronGST"]: eval_audio = ap.inv_spectrogram(const_spec.T) else: eval_audio = ap.inv_mel_spectrogram(const_spec.T) @@ -410,9 +410,9 @@ def main(args): optimizer_st = None if c.loss_masking: - criterion = L1LossMasked() if c.model == "Tacotron" else MSELossMasked() + criterion = L1LossMasked() if c.model in ["Tacotron", "TacotronGST"] else MSELossMasked() else: - criterion = nn.L1Loss() if c.model == "Tacotron" else nn.MSELoss() + criterion = nn.L1Loss() if c.model in ["Tacotron", "TacotronGST"] else nn.MSELoss() criterion_st = nn.BCEWithLogitsLoss() if c.stopnet else None if args.restore_path: diff --git a/utils/audio.py b/utils/audio.py index 0feb3653..93ecd684 100644 --- a/utils/audio.py +++ b/utils/audio.py @@ -10,7 +10,6 @@ from scipy import signal, io class AudioProcessor(object): def __init__(self, - bits=None, sample_rate=None, num_mels=None, min_level_db=None, @@ -32,7 +31,6 @@ class AudioProcessor(object): print(" > Setting up Audio Processor...") - self.bits = bits self.sample_rate = sample_rate self.num_mels = num_mels self.min_level_db = min_level_db @@ -218,25 +216,29 @@ class AudioProcessor(object): return librosa.effects.trim( wav, top_db=40, frame_length=1024, hop_length=256)[0] - # WaveRNN repo specific functions - # def mulaw_encode(self, wav, qc): - # mu = qc - 1 - # wav_abs = np.minimum(np.abs(wav), 1.0) - # magnitude = np.log(1 + mu * wav_abs) / np.log(1. + mu) - # signal = np.sign(wav) * magnitude - # # Quantize signal to the specified number of levels. - # signal = (signal + 1) / 2 * mu + 0.5 - # return signal.astype(np.int32) + def mulaw_encode(self, wav, qc): + mu = 2 ** qc - 1 + # wav_abs = np.minimum(np.abs(wav), 1.0) + signal = np.sign(wav) * np.log(1 + mu * np.abs(wav)) / np.log(1. + mu) + # Quantize signal to the specified number of levels. + signal = (signal + 1) / 2 * mu + 0.5 + return np.floor(signal,) - # def mulaw_decode(self, wav, qc): - # """Recovers waveform from quantized values.""" - # mu = qc - 1 - # # Map values back to [-1, 1]. - # casted = wav.astype(np.float32) - # signal = 2 * (casted / mu) - 1 - # # Perform inverse of mu-law transformation. - # magnitude = (1 / mu) * ((1 + mu) ** abs(signal) - 1) - # return np.sign(signal) * magnitude + @staticmethod + def mulaw_decode(wav, qc): + """Recovers waveform from quantized values.""" + # from IPython.core.debugger import set_trace + # set_trace() + mu = 2 ** qc - 1 + x = np.sign(wav) / mu * ((1 + mu) ** np.abs(wav) - 1) + return x + # mu = 2 ** qc - 1. + # # Map values back to [-1, 1]. + # # casted = wav.astype(np.float32) + # # signal = 2 * casted / mu - 1 + # # Perform inverse of mu-law transformation. + # magnitude = (1 / mu) * ((1 + mu) ** abs(wav) - 1) + # return np.sign(wav) * magnitude def load_wav(self, filename, encode=False): x, sr = sf.read(filename) @@ -249,8 +251,8 @@ class AudioProcessor(object): def encode_16bits(self, x): return np.clip(x * 2**15, -2**15, 2**15 - 1).astype(np.int16) - def quantize(self, x): - return (x + 1.) * (2**self.bits - 1) / 2 + def quantize(self, x, bits): + return (x + 1.) * (2**bits - 1) / 2 - def dequantize(self, x): - return 2 * x / (2**self.bits - 1) - 1 + def dequantize(self, x, bits): + return 2 * x / (2**bits - 1) - 1 diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 60519871..4fecd2a0 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -247,7 +247,7 @@ def setup_model(num_chars, c): print(" > Using model: {}".format(c.model)) MyModel = importlib.import_module('models.' + c.model.lower()) MyModel = getattr(MyModel, c.model) - if c.model.lower() == "tacotron": + if c.model.lower() in ["tacotron", "tacotrongst"]: model = MyModel( num_chars=num_chars, r=c.r,