From 9eb3c3a342586a629968bbc2cd1dd07682513114 Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 12 Feb 2020 23:54:33 +0100 Subject: [PATCH 1/6] setting stft parameters with constants --- config.json | 12 +++++------- utils/audio.py | 9 ++++++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/config.json b/config.json index 9e4fa906..62cccac8 100644 --- a/config.json +++ b/config.json @@ -9,8 +9,8 @@ "num_mels": 80, // size of the mel spec frame. "num_freq": 1025, // number of stft frequency levels. Size of the linear spectogram frame. "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "frame_length_ms": 50, // stft window length in ms. - "frame_shift_ms": 12.5, // stft window hop-lengh in ms. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. "min_level_db": -100, // normalization range "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. @@ -62,7 +62,7 @@ "prenet_dropout": true, // enable/disable dropout at prenet. // ATTENTION - "attention_type": "graves", // 'original' or 'graves' + "attention_type": "original", // 'original' or 'graves' "attention_heads": 4, // number of attention heads (only for 'graves') "attention_norm": "sigmoid", // softmax or sigmoid. Suggested to use softmax for Tacotron2 and sigmoid for Tacotron. "windowing": false, // Enables attention windowing. Used only in eval mode. @@ -92,8 +92,7 @@ "max_seq_len": 150, // DATASET-RELATED: maximum text length // PATHS - // "output_path": "/data5/rw/pit/keep/", // DATASET-RELATED: output path for all training outputs. - "output_path": "/home/erogol/Models/LJSpeech/", + "output_path": "/data4/rw/home/Trainings/", // PHONEMES "phoneme_cache_path": "mozilla_us_phonemes", // phoneme computation is slow, therefore, it caches results in the given folder. @@ -110,8 +109,7 @@ [ { "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - // "path": "/home/erogol/Data/LJSpeech-1.1", + "path": "/root/LJSpeech-1.1/", "meta_file_train": "metadata.csv", "meta_file_val": null } diff --git a/utils/audio.py b/utils/audio.py index 7b2c4834..771e6a43 100644 --- a/utils/audio.py +++ b/utils/audio.py @@ -12,6 +12,8 @@ class AudioProcessor(object): min_level_db=None, frame_shift_ms=None, frame_length_ms=None, + hop_length=None, + win_length=None, ref_level_db=None, num_freq=None, power=None, @@ -49,7 +51,12 @@ class AudioProcessor(object): self.do_trim_silence = do_trim_silence self.trim_db = trim_db self.sound_norm = sound_norm - self.n_fft, self.hop_length, self.win_length = self._stft_parameters() + if hop_length is None: + self.n_fft, self.hop_length, self.win_length = self._stft_parameters() + else: + self.hop_length = hop_length + self.win_length = win_length + self.n_fft = (self.num_freq - 1) * 2 assert min_level_db != 0.0, " [!] min_level_db is 0" members = vars(self) for key, value in members.items(): From 8feb326a60fce455ba439d4e1fb7bf0e66642bd4 Mon Sep 17 00:00:00 2001 From: Edresson Casanova Date: Sun, 1 Mar 2020 15:47:08 -0300 Subject: [PATCH 2/6] add text parameters in config.json --- config.json | 10 +++++++ datasets/TTSDataset.py | 9 ++++-- notebooks/Benchmark-PWGAN.ipynb | 6 +++- notebooks/Benchmark.ipynb | 6 +++- notebooks/ExtractTTSpectrogram.ipynb | 8 ++++-- notebooks/TestAttention.ipynb | 6 +++- server/synthesizer.py | 9 +++++- synthesize.py | 8 +++++- tests/test_demo_server.py | 5 +++- tests/test_loader.py | 1 + train.py | 8 ++++-- utils/generic_utils.py | 9 ++++++ utils/synthesis.py | 5 ++-- utils/text/__init__.py | 41 +++++++++++++++++++++++----- utils/text/symbols.py | 21 +++++++++----- utils/visual.py | 5 ++-- 16 files changed, 126 insertions(+), 31 deletions(-) diff --git a/config.json b/config.json index c1a8158d..2a7c4551 100644 --- a/config.json +++ b/config.json @@ -27,6 +27,16 @@ "trim_db": 60 // threshold for timming silence. Set this according to your dataset. }, + // VOCABULARY PARAMETERS + "text":{ + "pad": "_", + "eos": "~", + "bos": "^", + "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", + "punctuations":"!'(),-.:;? ", + "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" + }, + // DISTRIBUTED TRAINING "distributed":{ "backend": "nccl", diff --git a/datasets/TTSDataset.py b/datasets/TTSDataset.py index a45d77ff..cccd65a2 100644 --- a/datasets/TTSDataset.py +++ b/datasets/TTSDataset.py @@ -15,6 +15,7 @@ class MyDataset(Dataset): text_cleaner, ap, meta_data, + tp=None, batch_group_size=0, min_seq_len=0, max_seq_len=float("inf"), @@ -49,6 +50,7 @@ class MyDataset(Dataset): self.min_seq_len = min_seq_len self.max_seq_len = max_seq_len self.ap = ap + self.tp = tp self.use_phonemes = use_phonemes self.phoneme_cache_path = phoneme_cache_path self.phoneme_language = phoneme_language @@ -81,7 +83,8 @@ class MyDataset(Dataset): config option.""" phonemes = phoneme_to_sequence(text, [self.cleaners], language=self.phoneme_language, - enable_eos_bos=False) + enable_eos_bos=False, + tp=self.tp) phonemes = np.asarray(phonemes, dtype=np.int32) np.save(cache_path, phonemes) return phonemes @@ -101,7 +104,7 @@ class MyDataset(Dataset): phonemes = self._generate_and_cache_phoneme_sequence(text, cache_path) if self.enable_eos_bos: - phonemes = pad_with_eos_bos(phonemes) + phonemes = pad_with_eos_bos(phonemes, tp=self.tp) phonemes = np.asarray(phonemes, dtype=np.int32) return phonemes @@ -113,7 +116,7 @@ class MyDataset(Dataset): text = self._load_or_generate_phoneme_sequence(wav_file, text) else: text = np.asarray( - text_to_sequence(text, [self.cleaners]), dtype=np.int32) + text_to_sequence(text, [self.cleaners], tp=self.tp), dtype=np.int32) assert text.size > 0, self.items[idx][1] assert wav.size > 0, self.items[idx][1] diff --git a/notebooks/Benchmark-PWGAN.ipynb b/notebooks/Benchmark-PWGAN.ipynb index 430d329f..4a2a21d7 100644 --- a/notebooks/Benchmark-PWGAN.ipynb +++ b/notebooks/Benchmark-PWGAN.ipynb @@ -132,7 +132,7 @@ "outputs": [], "source": [ "# LOAD TTS MODEL\n", - "from TTS.utils.text.symbols import symbols, phonemes\n", + "from TTS.utils.text.symbols import make_symbols, symbols, phonemes\n", "\n", "# multi speaker \n", "if CONFIG.use_speaker_embedding:\n", @@ -142,6 +142,10 @@ " speakers = []\n", " speaker_id = None\n", "\n", + "# if the vocabulary was passed, replace the default\n", + "if 'text' in CONFIG.keys():\n", + " symbols, phonemes = make_symbols(**CONFIG.text)\n", + "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", "model = setup_model(num_chars, len(speakers), CONFIG)\n", diff --git a/notebooks/Benchmark.ipynb b/notebooks/Benchmark.ipynb index 00ac7d16..528d7a3b 100644 --- a/notebooks/Benchmark.ipynb +++ b/notebooks/Benchmark.ipynb @@ -65,7 +65,7 @@ "from TTS.utils.text import text_to_sequence\n", "from TTS.utils.synthesis import synthesis\n", "from TTS.utils.visual import visualize\n", - "from TTS.utils.text.symbols import symbols, phonemes\n", + "from TTS.utils.text.symbols import make_symbols, symbols, phonemes\n", "\n", "import IPython\n", "from IPython.display import Audio\n", @@ -149,6 +149,10 @@ " speakers = []\n", " speaker_id = None\n", "\n", + "# if the vocabulary was passed, replace the default\n", + "if 'text' in CONFIG.keys():\n", + " symbols, phonemes = make_symbols(**CONFIG.text)\n", + "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", "model = setup_model(num_chars, len(speakers), CONFIG)\n", diff --git a/notebooks/ExtractTTSpectrogram.ipynb b/notebooks/ExtractTTSpectrogram.ipynb index 20038f78..2313e47e 100644 --- a/notebooks/ExtractTTSpectrogram.ipynb +++ b/notebooks/ExtractTTSpectrogram.ipynb @@ -37,7 +37,7 @@ "from TTS.utils.audio import AudioProcessor\n", "from TTS.utils.visual import plot_spectrogram\n", "from TTS.utils.generic_utils import load_config, setup_model, sequence_mask\n", - "from TTS.utils.text.symbols import symbols, phonemes\n", + "from TTS.utils.text.symbols import make_symbols, symbols, phonemes\n", "\n", "%matplotlib inline\n", "\n", @@ -94,6 +94,10 @@ "metadata": {}, "outputs": [], "source": [ + "# if the vocabulary was passed, replace the default\n", + "if 'text' in C.keys():\n", + " symbols, phonemes = make_symbols(**C.text)\n", + "\n", "# load the model\n", "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n", "# TODO: multiple speaker\n", @@ -116,7 +120,7 @@ "preprocessor = importlib.import_module('TTS.datasets.preprocess')\n", "preprocessor = getattr(preprocessor, DATASET.lower())\n", "meta_data = preprocessor(DATA_PATH,METADATA_FILE)\n", - "dataset = MyDataset(checkpoint['r'], C.text_cleaner, ap, meta_data, use_phonemes=C.use_phonemes, phoneme_cache_path=C.phoneme_cache_path, enable_eos_bos=C.enable_eos_bos_chars)\n", + "dataset = MyDataset(checkpoint['r'], C.text_cleaner, ap, meta_data,tp=C.text if 'text' in C.keys() else None, use_phonemes=C.use_phonemes, phoneme_cache_path=C.phoneme_cache_path, enable_eos_bos=C.enable_eos_bos_chars)\n", "loader = DataLoader(dataset, batch_size=BATCH_SIZE, num_workers=4, collate_fn=dataset.collate_fn, shuffle=False, drop_last=False)" ] }, diff --git a/notebooks/TestAttention.ipynb b/notebooks/TestAttention.ipynb index a1867d13..5310fb92 100644 --- a/notebooks/TestAttention.ipynb +++ b/notebooks/TestAttention.ipynb @@ -100,7 +100,7 @@ "outputs": [], "source": [ "# LOAD TTS MODEL\n", - "from TTS.utils.text.symbols import symbols, phonemes\n", + "from TTS.utils.text.symbols import make_symbols, symbols, phonemes\n", "\n", "# multi speaker \n", "if CONFIG.use_speaker_embedding:\n", @@ -110,6 +110,10 @@ " speakers = []\n", " speaker_id = None\n", "\n", + "# if the vocabulary was passed, replace the default\n", + "if 'text' in CONFIG.keys():\n", + " symbols, phonemes = make_symbols(**CONFIG.text)\n", + "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", "model = setup_model(num_chars, len(speakers), CONFIG)\n", diff --git a/server/synthesizer.py b/server/synthesizer.py index 347bef21..f001afcd 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -10,7 +10,7 @@ from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import load_config, setup_model from TTS.utils.speakers import load_speaker_mapping from TTS.utils.synthesis import * -from TTS.utils.text import phonemes, symbols +from TTS.utils.text import make_symbols, phonemes, symbols alphabets = r"([A-Za-z])" prefixes = r"(Mr|St|Mrs|Ms|Dr)[.]" @@ -38,12 +38,19 @@ class Synthesizer(object): self.config.pwgan_config, self.config.use_cuda) def load_tts(self, tts_checkpoint, tts_config, use_cuda): + global symbols, phonemes + print(" > Loading TTS model ...") print(" | > model config: ", tts_config) print(" | > checkpoint file: ", tts_checkpoint) + self.tts_config = load_config(tts_config) self.use_phonemes = self.tts_config.use_phonemes self.ap = AudioProcessor(**self.tts_config.audio) + + if 'text' in self.tts_config.keys(): + symbols, phonemes = make_symbols(**self.tts_config.text) + if self.use_phonemes: self.input_size = len(phonemes) else: diff --git a/synthesize.py b/synthesize.py index bf85d7c9..d294701f 100644 --- a/synthesize.py +++ b/synthesize.py @@ -8,7 +8,7 @@ import string from TTS.utils.synthesis import synthesis from TTS.utils.generic_utils import load_config, setup_model -from TTS.utils.text.symbols import symbols, phonemes +from TTS.utils.text.symbols import make_symbols, symbols, phonemes from TTS.utils.audio import AudioProcessor @@ -48,6 +48,8 @@ def tts(model, if __name__ == "__main__": + global symbols, phonemes + parser = argparse.ArgumentParser() parser.add_argument('text', type=str, help='Text to generate speech.') parser.add_argument('config_path', @@ -105,6 +107,10 @@ if __name__ == "__main__": # load the audio processor ap = AudioProcessor(**C.audio) + # if the vocabulary was passed, replace the default + if 'text' in C.keys(): + symbols, phonemes = make_symbols(**C.text) + # load speakers if args.speakers_json != '': speakers = json.load(open(args.speakers_json, 'r')) diff --git a/tests/test_demo_server.py b/tests/test_demo_server.py index c343a6a4..3e360e20 100644 --- a/tests/test_demo_server.py +++ b/tests/test_demo_server.py @@ -5,13 +5,16 @@ import torch as T from TTS.server.synthesizer import Synthesizer from TTS.tests import get_tests_input_path, get_tests_output_path -from TTS.utils.text.symbols import phonemes, symbols +from TTS.utils.text.symbols import make_symbols, phonemes, symbols from TTS.utils.generic_utils import load_config, save_checkpoint, setup_model class DemoServerTest(unittest.TestCase): def _create_random_model(self): config = load_config(os.path.join(get_tests_output_path(), 'dummy_model_config.json')) + if 'text' in config.keys(): + symbols, phonemes = make_symbols(**config.text) + num_chars = len(phonemes) if config.use_phonemes else len(symbols) model = setup_model(num_chars, 0, config) output_path = os.path.join(get_tests_output_path()) diff --git a/tests/test_loader.py b/tests/test_loader.py index d8727895..5141fa85 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -38,6 +38,7 @@ class TestTTSDataset(unittest.TestCase): c.text_cleaner, ap=self.ap, meta_data=items, + tp=c.text if 'text' in c.keys() else None, batch_group_size=bgs, min_seq_len=c.min_seq_len, max_seq_len=float("inf"), diff --git a/train.py b/train.py index 7bfb8751..96c268f0 100644 --- a/train.py +++ b/train.py @@ -25,7 +25,7 @@ from TTS.utils.logger import Logger from TTS.utils.speakers import load_speaker_mapping, save_speaker_mapping, \ get_speakers from TTS.utils.synthesis import synthesis -from TTS.utils.text.symbols import phonemes, symbols +from TTS.utils.text.symbols import make_symbols, phonemes, symbols from TTS.utils.visual import plot_alignment, plot_spectrogram from TTS.datasets.preprocess import load_meta_data from TTS.utils.radam import RAdam @@ -49,6 +49,7 @@ def setup_loader(ap, r, is_val=False, verbose=False): c.text_cleaner, meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, + tp=c.text if 'text' in c.keys() else None, batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, min_seq_len=c.min_seq_len, @@ -515,9 +516,12 @@ def evaluate(model, criterion, criterion_st, ap, global_step, epoch): # FIXME: move args definition/parsing inside of main? def main(args): # pylint: disable=redefined-outer-name - global meta_data_train, meta_data_eval + global meta_data_train, meta_data_eval, symbols, phonemes # Audio processor ap = AudioProcessor(**c.audio) + + if 'text' in c.keys(): + symbols, phonemes = make_symbols(**c.text) # DISTRUBUTED if num_gpus > 1: diff --git a/utils/generic_utils.py b/utils/generic_utils.py index a8de5bbb..6aecdc7d 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -425,6 +425,15 @@ def check_config(c): _check_argument('power', c['audio'], restricted=True, val_type=float, min_val=1, max_val=5) _check_argument('griffin_lim_iters', c['audio'], restricted=True, val_type=int, min_val=10, max_val=1000) + # vocabulary parameters + _check_argument('text', c, restricted=False, val_type=dict) # parameter not mandatory + _check_argument('pad', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) # mandatory if "text parameters" else no mandatory + _check_argument('eos', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) + _check_argument('bos', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) + _check_argument('characters', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) + _check_argument('phonemes', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) + _check_argument('punctuations', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) + # normalization parameters _check_argument('signal_norm', c['audio'], restricted=True, val_type=bool) _check_argument('symmetric_norm', c['audio'], restricted=True, val_type=bool) diff --git a/utils/synthesis.py b/utils/synthesis.py index 79a17c78..c5ff2e70 100644 --- a/utils/synthesis.py +++ b/utils/synthesis.py @@ -9,10 +9,11 @@ def text_to_seqvec(text, CONFIG, use_cuda): if CONFIG.use_phonemes: seq = np.asarray( phoneme_to_sequence(text, text_cleaner, CONFIG.phoneme_language, - CONFIG.enable_eos_bos_chars), + CONFIG.enable_eos_bos_chars, + tp=CONFIG.text if 'text' in CONFIG.keys() else None), dtype=np.int32) else: - seq = np.asarray(text_to_sequence(text, text_cleaner), dtype=np.int32) + seq = np.asarray(text_to_sequence(text, text_cleaner, tp=CONFIG.text if 'text' in CONFIG.keys() else None), dtype=np.int32) # torch tensor chars_var = torch.from_numpy(seq).unsqueeze(0) if use_cuda: diff --git a/utils/text/__init__.py b/utils/text/__init__.py index 0e6684d2..fcb239b2 100644 --- a/utils/text/__init__.py +++ b/utils/text/__init__.py @@ -4,7 +4,7 @@ import re import phonemizer from phonemizer.phonemize import phonemize from TTS.utils.text import cleaners -from TTS.utils.text.symbols import symbols, phonemes, _phoneme_punctuations, _bos, \ +from TTS.utils.text.symbols import make_symbols, symbols, phonemes, _phoneme_punctuations, _bos, \ _eos # Mappings from symbol to numeric ID and vice versa: @@ -56,11 +56,23 @@ def text2phone(text, language): return ph -def pad_with_eos_bos(phoneme_sequence): +def pad_with_eos_bos(phoneme_sequence, tp=None): + global _PHONEMES_TO_ID, _bos, _eos + if tp: + _bos = tp['bos'] + _eos = tp['eos'] + _, phonemes = make_symbols(**tp) + _PHONEMES_TO_ID = {s: i for i, s in enumerate(phonemes)} + return [_PHONEMES_TO_ID[_bos]] + list(phoneme_sequence) + [_PHONEMES_TO_ID[_eos]] -def phoneme_to_sequence(text, cleaner_names, language, enable_eos_bos=False): +def phoneme_to_sequence(text, cleaner_names, language, enable_eos_bos=False, tp=None): + global _PHONEMES_TO_ID + if tp: + _, phonemes = make_symbols(**tp) + _PHONEMES_TO_ID = {s: i for i, s in enumerate(phonemes)} + sequence = [] text = text.replace(":", "") clean_text = _clean_text(text, cleaner_names) @@ -72,13 +84,18 @@ def phoneme_to_sequence(text, cleaner_names, language, enable_eos_bos=False): sequence += _phoneme_to_sequence(phoneme) # Append EOS char if enable_eos_bos: - sequence = pad_with_eos_bos(sequence) + sequence = pad_with_eos_bos(sequence, tp=tp) return sequence -def sequence_to_phoneme(sequence): +def sequence_to_phoneme(sequence, tp=None): '''Converts a sequence of IDs back to a string''' + global _ID_TO_PHONEMES result = '' + if tp: + _, phonemes = make_symbols(**tp) + _ID_TO_PHONEMES = {i: s for i, s in enumerate(phonemes)} + for symbol_id in sequence: if symbol_id in _ID_TO_PHONEMES: s = _ID_TO_PHONEMES[symbol_id] @@ -86,7 +103,7 @@ def sequence_to_phoneme(sequence): return result.replace('}{', ' ') -def text_to_sequence(text, cleaner_names): +def text_to_sequence(text, cleaner_names, tp=None): '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text. The text can optionally have ARPAbet sequences enclosed in curly braces embedded @@ -99,6 +116,11 @@ def text_to_sequence(text, cleaner_names): Returns: List of integers corresponding to the symbols in the text ''' + global _SYMBOL_TO_ID + if tp: + symbols, _ = make_symbols(**tp) + _SYMBOL_TO_ID = {s: i for i, s in enumerate(symbols)} + sequence = [] # Check for curly braces and treat their contents as ARPAbet: while text: @@ -113,8 +135,13 @@ def text_to_sequence(text, cleaner_names): return sequence -def sequence_to_text(sequence): +def sequence_to_text(sequence, tp=None): '''Converts a sequence of IDs back to a string''' + global _ID_TO_SYMBOL + if tp: + symbols, _ = make_symbols(**tp) + _ID_TO_SYMBOL = {i: s for i, s in enumerate(symbols)} + result = '' for symbol_id in sequence: if symbol_id in _ID_TO_SYMBOL: diff --git a/utils/text/symbols.py b/utils/text/symbols.py index ee6fd2cf..e4a4b103 100644 --- a/utils/text/symbols.py +++ b/utils/text/symbols.py @@ -5,6 +5,18 @@ Defines the set of symbols used in text input to the model. The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. ''' +def make_symbols(characters, phonemes, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'): + ''' Function to create symbols and phonemes ''' + _phonemes = sorted(list(phonemes)) + + # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters): + _arpabet = ['@' + s for s in _phonemes] + + # Export all symbols: + symbols = [pad, eos, bos] + list(characters) + _arpabet + phonemes = [pad, eos, bos] + list(_phonemes) + list(punctuations) + + return symbols, phonemes _pad = '_' _eos = '~' @@ -20,14 +32,9 @@ _pulmonic_consonants = 'pbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðsz _suprasegmentals = 'ˈˌːˑ' _other_symbols = 'ʍwɥʜʢʡɕʑɺɧ' _diacrilics = 'ɚ˞ɫ' -_phonemes = sorted(list(_vowels + _non_pulmonic_consonants + _pulmonic_consonants + _suprasegmentals + _other_symbols + _diacrilics)) +_phonemes = _vowels + _non_pulmonic_consonants + _pulmonic_consonants + _suprasegmentals + _other_symbols + _diacrilics -# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters): -_arpabet = ['@' + s for s in _phonemes] - -# Export all symbols: -symbols = [_pad, _eos, _bos] + list(_characters) + _arpabet -phonemes = [_pad, _eos, _bos] + list(_phonemes) + list(_punctuations) +symbols, phonemes = make_symbols( _characters, _phonemes,_punctuations, _pad, _eos, _bos) # Generate ALIEN language # from random import shuffle diff --git a/utils/visual.py b/utils/visual.py index ab513666..2f93d812 100644 --- a/utils/visual.py +++ b/utils/visual.py @@ -54,9 +54,10 @@ def visualize(alignment, spectrogram_postnet, stop_tokens, text, hop_length, CON plt.xlabel("Decoder timestamp", fontsize=label_fontsize) plt.ylabel("Encoder timestamp", fontsize=label_fontsize) if CONFIG.use_phonemes: - seq = phoneme_to_sequence(text, [CONFIG.text_cleaner], CONFIG.phoneme_language, CONFIG.enable_eos_bos_chars) - text = sequence_to_phoneme(seq) + seq = phoneme_to_sequence(text, [CONFIG.text_cleaner], CONFIG.phoneme_language, CONFIG.enable_eos_bos_chars, tp=CONFIG.text if 'text' in CONFIG.keys() else None) + text = sequence_to_phoneme(seq, tp=CONFIG.text if 'text' in CONFIG.keys() else None) print(text) + plt.yticks(range(len(text)), list(text)) plt.colorbar() From 59e2752107162b7b6060c3096d24c16a3cbbd0b3 Mon Sep 17 00:00:00 2001 From: Edresson Casanova Date: Mon, 2 Mar 2020 11:46:00 -0300 Subject: [PATCH 3/6] fix travis unit test errors --- datasets/TTSDataset.py | 3 +-- train.py | 3 +-- utils/generic_utils.py | 14 +++++++------- utils/text/__init__.py | 20 ++++++++++---------- utils/text/symbols.py | 10 +++++----- 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/datasets/TTSDataset.py b/datasets/TTSDataset.py index cccd65a2..d649bf23 100644 --- a/datasets/TTSDataset.py +++ b/datasets/TTSDataset.py @@ -77,13 +77,12 @@ class MyDataset(Dataset): def _generate_and_cache_phoneme_sequence(self, text, cache_path): """generate a phoneme sequence from text. - since the usage is for subsequent caching, we never add bos and eos chars here. Instead we add those dynamically later; based on the config option.""" phonemes = phoneme_to_sequence(text, [self.cleaners], language=self.phoneme_language, - enable_eos_bos=False, + enable_eos_bos=False, tp=self.tp) phonemes = np.asarray(phonemes, dtype=np.int32) np.save(cache_path, phonemes) diff --git a/train.py b/train.py index 96c268f0..616d54ac 100644 --- a/train.py +++ b/train.py @@ -519,9 +519,8 @@ def main(args): # pylint: disable=redefined-outer-name global meta_data_train, meta_data_eval, symbols, phonemes # Audio processor ap = AudioProcessor(**c.audio) - if 'text' in c.keys(): - symbols, phonemes = make_symbols(**c.text) + symbols, phonemes = make_symbols(**c.text) # DISTRUBUTED if num_gpus > 1: diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 6aecdc7d..7c2f033a 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -426,13 +426,13 @@ def check_config(c): _check_argument('griffin_lim_iters', c['audio'], restricted=True, val_type=int, min_val=10, max_val=1000) # vocabulary parameters - _check_argument('text', c, restricted=False, val_type=dict) # parameter not mandatory - _check_argument('pad', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) # mandatory if "text parameters" else no mandatory - _check_argument('eos', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) - _check_argument('bos', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) - _check_argument('characters', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) - _check_argument('phonemes', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) - _check_argument('punctuations', c['text'] if 'text' in c.keys() else {}, restricted=True if 'text' in c.keys() else False, val_type=str) + _check_argument('text', c, restricted=False, val_type=dict) + _check_argument('pad', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) + _check_argument('eos', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) + _check_argument('bos', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) + _check_argument('characters', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) + _check_argument('phonemes', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) + _check_argument('punctuations', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) # normalization parameters _check_argument('signal_norm', c['audio'], restricted=True, val_type=bool) diff --git a/utils/text/__init__.py b/utils/text/__init__.py index fcb239b2..ff21ffe0 100644 --- a/utils/text/__init__.py +++ b/utils/text/__init__.py @@ -61,8 +61,8 @@ def pad_with_eos_bos(phoneme_sequence, tp=None): if tp: _bos = tp['bos'] _eos = tp['eos'] - _, phonemes = make_symbols(**tp) - _PHONEMES_TO_ID = {s: i for i, s in enumerate(phonemes)} + _, _phonemes = make_symbols(**tp) + _PHONEMES_TO_ID = {s: i for i, s in enumerate(_phonemes)} return [_PHONEMES_TO_ID[_bos]] + list(phoneme_sequence) + [_PHONEMES_TO_ID[_eos]] @@ -70,8 +70,8 @@ def pad_with_eos_bos(phoneme_sequence, tp=None): def phoneme_to_sequence(text, cleaner_names, language, enable_eos_bos=False, tp=None): global _PHONEMES_TO_ID if tp: - _, phonemes = make_symbols(**tp) - _PHONEMES_TO_ID = {s: i for i, s in enumerate(phonemes)} + _, _phonemes = make_symbols(**tp) + _PHONEMES_TO_ID = {s: i for i, s in enumerate(_phonemes)} sequence = [] text = text.replace(":", "") @@ -93,8 +93,8 @@ def sequence_to_phoneme(sequence, tp=None): global _ID_TO_PHONEMES result = '' if tp: - _, phonemes = make_symbols(**tp) - _ID_TO_PHONEMES = {i: s for i, s in enumerate(phonemes)} + _, _phonemes = make_symbols(**tp) + _ID_TO_PHONEMES = {i: s for i, s in enumerate(_phonemes)} for symbol_id in sequence: if symbol_id in _ID_TO_PHONEMES: @@ -118,8 +118,8 @@ def text_to_sequence(text, cleaner_names, tp=None): ''' global _SYMBOL_TO_ID if tp: - symbols, _ = make_symbols(**tp) - _SYMBOL_TO_ID = {s: i for i, s in enumerate(symbols)} + _symbols, _ = make_symbols(**tp) + _SYMBOL_TO_ID = {s: i for i, s in enumerate(_symbols)} sequence = [] # Check for curly braces and treat their contents as ARPAbet: @@ -139,8 +139,8 @@ def sequence_to_text(sequence, tp=None): '''Converts a sequence of IDs back to a string''' global _ID_TO_SYMBOL if tp: - symbols, _ = make_symbols(**tp) - _ID_TO_SYMBOL = {i: s for i, s in enumerate(symbols)} + _symbols, _ = make_symbols(**tp) + _ID_TO_SYMBOL = {i: s for i, s in enumerate(_symbols)} result = '' for symbol_id in sequence: diff --git a/utils/text/symbols.py b/utils/text/symbols.py index e4a4b103..db83cb29 100644 --- a/utils/text/symbols.py +++ b/utils/text/symbols.py @@ -5,16 +5,16 @@ Defines the set of symbols used in text input to the model. The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. ''' -def make_symbols(characters, phonemes, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'): +def make_symbols(characters, phnms, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'): ''' Function to create symbols and phonemes ''' - _phonemes = sorted(list(phonemes)) + _phonemes_sorted = sorted(list(phnms)) # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters): - _arpabet = ['@' + s for s in _phonemes] + _arpabet = ['@' + s for s in _phonemes_sorted] # Export all symbols: - symbols = [pad, eos, bos] + list(characters) + _arpabet - phonemes = [pad, eos, bos] + list(_phonemes) + list(punctuations) + _symbols = [pad, eos, bos] + list(characters) + _arpabet + _phonemes = [pad, eos, bos] + list(_phonemes_sorted) + list(punctuations) return symbols, phonemes From 4e53896438b5365269e54dae999b6ddab837b0c4 Mon Sep 17 00:00:00 2001 From: Edresson Casanova Date: Mon, 2 Mar 2020 15:33:13 -0300 Subject: [PATCH 4/6] fix travis lint check --- datasets/TTSDataset.py | 2 +- notebooks/Benchmark-PWGAN.ipynb | 2 +- notebooks/Benchmark.ipynb | 2 +- notebooks/TestAttention.ipynb | 2 +- server/synthesizer.py | 12 ++++--- synthesize.py | 2 +- tests/test_demo_server.py | 5 ++- tests/test_loader.py | 2 +- train.py | 1 + utils/text/__init__.py | 55 ++++++++++++++++++--------------- utils/text/symbols.py | 4 +-- utils/visual.py | 2 +- 12 files changed, 52 insertions(+), 39 deletions(-) diff --git a/datasets/TTSDataset.py b/datasets/TTSDataset.py index d649bf23..d3a6f486 100644 --- a/datasets/TTSDataset.py +++ b/datasets/TTSDataset.py @@ -195,7 +195,7 @@ class MyDataset(Dataset): mel = [self.ap.melspectrogram(w).astype('float32') for w in wav] linear = [self.ap.spectrogram(w).astype('float32') for w in wav] - mel_lengths = [m.shape[1] for m in mel] + mel_lengths = [m.shape[1] for m in mel] # compute 'stop token' targets stop_targets = [ diff --git a/notebooks/Benchmark-PWGAN.ipynb b/notebooks/Benchmark-PWGAN.ipynb index 4a2a21d7..19a1a79c 100644 --- a/notebooks/Benchmark-PWGAN.ipynb +++ b/notebooks/Benchmark-PWGAN.ipynb @@ -144,7 +144,7 @@ "\n", "# if the vocabulary was passed, replace the default\n", "if 'text' in CONFIG.keys():\n", - " symbols, phonemes = make_symbols(**CONFIG.text)\n", + " symbols, phonemes = make_symbols(**CONFIG.text)\n", "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", diff --git a/notebooks/Benchmark.ipynb b/notebooks/Benchmark.ipynb index 528d7a3b..bf6f2774 100644 --- a/notebooks/Benchmark.ipynb +++ b/notebooks/Benchmark.ipynb @@ -151,7 +151,7 @@ "\n", "# if the vocabulary was passed, replace the default\n", "if 'text' in CONFIG.keys():\n", - " symbols, phonemes = make_symbols(**CONFIG.text)\n", + " symbols, phonemes = make_symbols(**CONFIG.text)\n", "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", diff --git a/notebooks/TestAttention.ipynb b/notebooks/TestAttention.ipynb index 5310fb92..b0599d80 100644 --- a/notebooks/TestAttention.ipynb +++ b/notebooks/TestAttention.ipynb @@ -112,7 +112,7 @@ "\n", "# if the vocabulary was passed, replace the default\n", "if 'text' in CONFIG.keys():\n", - " symbols, phonemes = make_symbols(**CONFIG.text)\n", + " symbols, phonemes = make_symbols(**CONFIG.text)\n", "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", diff --git a/server/synthesizer.py b/server/synthesizer.py index f001afcd..f0921513 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -9,7 +9,10 @@ import yaml from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import load_config, setup_model from TTS.utils.speakers import load_speaker_mapping +# pylint: disable=unused-wildcard-import +# pylint: disable=wildcard-import from TTS.utils.synthesis import * + from TTS.utils.text import make_symbols, phonemes, symbols alphabets = r"([A-Za-z])" @@ -38,18 +41,19 @@ class Synthesizer(object): self.config.pwgan_config, self.config.use_cuda) def load_tts(self, tts_checkpoint, tts_config, use_cuda): + # pylint: disable=global-statement global symbols, phonemes print(" > Loading TTS model ...") print(" | > model config: ", tts_config) print(" | > checkpoint file: ", tts_checkpoint) - + self.tts_config = load_config(tts_config) self.use_phonemes = self.tts_config.use_phonemes self.ap = AudioProcessor(**self.tts_config.audio) if 'text' in self.tts_config.keys(): - symbols, phonemes = make_symbols(**self.tts_config.text) + symbols, phonemes = make_symbols(**self.tts_config.text) if self.use_phonemes: self.input_size = len(phonemes) @@ -61,7 +65,7 @@ class Synthesizer(object): num_speakers = len(self.tts_speakers) else: num_speakers = 0 - self.tts_model = setup_model(self.input_size, num_speakers=num_speakers, c=self.tts_config) + self.tts_model = setup_model(self.input_size, num_speakers=num_speakers, c=self.tts_config) # load model state cp = torch.load(tts_checkpoint, map_location=torch.device('cpu')) # load the model @@ -91,7 +95,7 @@ class Synthesizer(object): mulaw=self.wavernn_config.mulaw, pad=self.wavernn_config.pad, use_aux_net=self.wavernn_config.use_aux_net, - use_upsample_net = self.wavernn_config.use_upsample_net, + use_upsample_net=self.wavernn_config.use_upsample_net, upsample_factors=self.wavernn_config.upsample_factors, feat_dims=80, compute_dims=128, diff --git a/synthesize.py b/synthesize.py index d294701f..6f3a235f 100644 --- a/synthesize.py +++ b/synthesize.py @@ -109,7 +109,7 @@ if __name__ == "__main__": # if the vocabulary was passed, replace the default if 'text' in C.keys(): - symbols, phonemes = make_symbols(**C.text) + symbols, phonemes = make_symbols(**C.text) # load speakers if args.speakers_json != '': diff --git a/tests/test_demo_server.py b/tests/test_demo_server.py index 3e360e20..36848942 100644 --- a/tests/test_demo_server.py +++ b/tests/test_demo_server.py @@ -10,10 +10,13 @@ from TTS.utils.generic_utils import load_config, save_checkpoint, setup_model class DemoServerTest(unittest.TestCase): + # pylint: disable=R0201 def _create_random_model(self): + # pylint: disable=global-statement + global symbols, phonemes config = load_config(os.path.join(get_tests_output_path(), 'dummy_model_config.json')) if 'text' in config.keys(): - symbols, phonemes = make_symbols(**config.text) + symbols, phonemes = make_symbols(**config.text) num_chars = len(phonemes) if config.use_phonemes else len(symbols) model = setup_model(num_chars, 0, config) diff --git a/tests/test_loader.py b/tests/test_loader.py index 5141fa85..eb23ed19 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -37,7 +37,7 @@ class TestTTSDataset(unittest.TestCase): r, c.text_cleaner, ap=self.ap, - meta_data=items, + meta_data=items, tp=c.text if 'text' in c.keys() else None, batch_group_size=bgs, min_seq_len=c.min_seq_len, diff --git a/train.py b/train.py index 616d54ac..bf5429e9 100644 --- a/train.py +++ b/train.py @@ -516,6 +516,7 @@ def evaluate(model, criterion, criterion_st, ap, global_step, epoch): # FIXME: move args definition/parsing inside of main? def main(args): # pylint: disable=redefined-outer-name + # pylint: disable=global-variable-undefined global meta_data_train, meta_data_eval, symbols, phonemes # Audio processor ap = AudioProcessor(**c.audio) diff --git a/utils/text/__init__.py b/utils/text/__init__.py index ff21ffe0..4361bc13 100644 --- a/utils/text/__init__.py +++ b/utils/text/__init__.py @@ -8,11 +8,11 @@ from TTS.utils.text.symbols import make_symbols, symbols, phonemes, _phoneme_pun _eos # Mappings from symbol to numeric ID and vice versa: -_SYMBOL_TO_ID = {s: i for i, s in enumerate(symbols)} -_ID_TO_SYMBOL = {i: s for i, s in enumerate(symbols)} +_symbol_to_id = {s: i for i, s in enumerate(symbols)} +_id_to_symbol = {i: s for i, s in enumerate(symbols)} -_PHONEMES_TO_ID = {s: i for i, s in enumerate(phonemes)} -_ID_TO_PHONEMES = {i: s for i, s in enumerate(phonemes)} +_phonemes_to_id = {s: i for i, s in enumerate(phonemes)} +_id_to_phonemes = {i: s for i, s in enumerate(phonemes)} # Regular expression matching text enclosed in curly braces: _CURLY_RE = re.compile(r'(.*?)\{(.+?)\}(.*)') @@ -57,21 +57,23 @@ def text2phone(text, language): def pad_with_eos_bos(phoneme_sequence, tp=None): - global _PHONEMES_TO_ID, _bos, _eos + # pylint: disable=global-statement + global _phonemes_to_id, _bos, _eos if tp: _bos = tp['bos'] _eos = tp['eos'] _, _phonemes = make_symbols(**tp) - _PHONEMES_TO_ID = {s: i for i, s in enumerate(_phonemes)} - - return [_PHONEMES_TO_ID[_bos]] + list(phoneme_sequence) + [_PHONEMES_TO_ID[_eos]] + _phonemes_to_id = {s: i for i, s in enumerate(_phonemes)} + + return [_phonemes_to_id[_bos]] + list(phoneme_sequence) + [_phonemes_to_id[_eos]] def phoneme_to_sequence(text, cleaner_names, language, enable_eos_bos=False, tp=None): - global _PHONEMES_TO_ID + # pylint: disable=global-statement + global _phonemes_to_id if tp: _, _phonemes = make_symbols(**tp) - _PHONEMES_TO_ID = {s: i for i, s in enumerate(_phonemes)} + _phonemes_to_id = {s: i for i, s in enumerate(_phonemes)} sequence = [] text = text.replace(":", "") @@ -89,16 +91,17 @@ def phoneme_to_sequence(text, cleaner_names, language, enable_eos_bos=False, tp= def sequence_to_phoneme(sequence, tp=None): + # pylint: disable=global-statement '''Converts a sequence of IDs back to a string''' - global _ID_TO_PHONEMES + global _id_to_phonemes result = '' if tp: _, _phonemes = make_symbols(**tp) - _ID_TO_PHONEMES = {i: s for i, s in enumerate(_phonemes)} - + _id_to_phonemes = {i: s for i, s in enumerate(_phonemes)} + for symbol_id in sequence: - if symbol_id in _ID_TO_PHONEMES: - s = _ID_TO_PHONEMES[symbol_id] + if symbol_id in _id_to_phonemes: + s = _id_to_phonemes[symbol_id] result += s return result.replace('}{', ' ') @@ -116,10 +119,11 @@ def text_to_sequence(text, cleaner_names, tp=None): Returns: List of integers corresponding to the symbols in the text ''' - global _SYMBOL_TO_ID + # pylint: disable=global-statement + global _symbol_to_id if tp: _symbols, _ = make_symbols(**tp) - _SYMBOL_TO_ID = {s: i for i, s in enumerate(_symbols)} + _symbol_to_id = {s: i for i, s in enumerate(_symbols)} sequence = [] # Check for curly braces and treat their contents as ARPAbet: @@ -137,15 +141,16 @@ def text_to_sequence(text, cleaner_names, tp=None): def sequence_to_text(sequence, tp=None): '''Converts a sequence of IDs back to a string''' - global _ID_TO_SYMBOL + # pylint: disable=global-statement + global _id_to_symbol if tp: _symbols, _ = make_symbols(**tp) - _ID_TO_SYMBOL = {i: s for i, s in enumerate(_symbols)} + _id_to_symbol = {i: s for i, s in enumerate(_symbols)} result = '' for symbol_id in sequence: - if symbol_id in _ID_TO_SYMBOL: - s = _ID_TO_SYMBOL[symbol_id] + if symbol_id in _id_to_symbol: + s = _id_to_symbol[symbol_id] # Enclose ARPAbet back in curly braces: if len(s) > 1 and s[0] == '@': s = '{%s}' % s[1:] @@ -163,11 +168,11 @@ def _clean_text(text, cleaner_names): def _symbols_to_sequence(syms): - return [_SYMBOL_TO_ID[s] for s in syms if _should_keep_symbol(s)] + return [_symbol_to_id[s] for s in syms if _should_keep_symbol(s)] def _phoneme_to_sequence(phons): - return [_PHONEMES_TO_ID[s] for s in list(phons) if _should_keep_phoneme(s)] + return [_phonemes_to_id[s] for s in list(phons) if _should_keep_phoneme(s)] def _arpabet_to_sequence(text): @@ -175,8 +180,8 @@ def _arpabet_to_sequence(text): def _should_keep_symbol(s): - return s in _SYMBOL_TO_ID and s not in ['~', '^', '_'] + return s in _symbol_to_id and s not in ['~', '^', '_'] def _should_keep_phoneme(p): - return p in _PHONEMES_TO_ID and p not in ['~', '^', '_'] + return p in _phonemes_to_id and p not in ['~', '^', '_'] diff --git a/utils/text/symbols.py b/utils/text/symbols.py index db83cb29..15862cbd 100644 --- a/utils/text/symbols.py +++ b/utils/text/symbols.py @@ -16,7 +16,7 @@ def make_symbols(characters, phnms, punctuations='!\'(),-.:;? ', pad='_', eos='~ _symbols = [pad, eos, bos] + list(characters) + _arpabet _phonemes = [pad, eos, bos] + list(_phonemes_sorted) + list(punctuations) - return symbols, phonemes + return _symbols, _phonemes _pad = '_' _eos = '~' @@ -34,7 +34,7 @@ _other_symbols = 'ʍwɥʜʢʡɕʑɺɧ' _diacrilics = 'ɚ˞ɫ' _phonemes = _vowels + _non_pulmonic_consonants + _pulmonic_consonants + _suprasegmentals + _other_symbols + _diacrilics -symbols, phonemes = make_symbols( _characters, _phonemes,_punctuations, _pad, _eos, _bos) +symbols, phonemes = make_symbols(_characters, _phonemes, _punctuations, _pad, _eos, _bos) # Generate ALIEN language # from random import shuffle diff --git a/utils/visual.py b/utils/visual.py index 2f93d812..3b24364c 100644 --- a/utils/visual.py +++ b/utils/visual.py @@ -57,7 +57,7 @@ def visualize(alignment, spectrogram_postnet, stop_tokens, text, hop_length, CON seq = phoneme_to_sequence(text, [CONFIG.text_cleaner], CONFIG.phoneme_language, CONFIG.enable_eos_bos_chars, tp=CONFIG.text if 'text' in CONFIG.keys() else None) text = sequence_to_phoneme(seq, tp=CONFIG.text if 'text' in CONFIG.keys() else None) print(text) - + plt.yticks(range(len(text)), list(text)) plt.colorbar() From 36235c5e3fc0f47c56253a99941fc769d744469d Mon Sep 17 00:00:00 2001 From: Edresson Casanova Date: Tue, 3 Mar 2020 09:17:56 -0300 Subject: [PATCH 5/6] rename text to characters in config.json --- config.json | 2 +- notebooks/Benchmark-PWGAN.ipynb | 4 ++-- notebooks/Benchmark.ipynb | 4 ++-- notebooks/ExtractTTSpectrogram.ipynb | 6 +++--- notebooks/TestAttention.ipynb | 4 ++-- server/synthesizer.py | 4 ++-- synthesize.py | 4 ++-- tests/test_demo_server.py | 4 ++-- tests/test_loader.py | 2 +- train.py | 6 +++--- utils/generic_utils.py | 14 +++++++------- utils/synthesis.py | 4 ++-- utils/visual.py | 4 ++-- 13 files changed, 31 insertions(+), 31 deletions(-) diff --git a/config.json b/config.json index 2a7c4551..3722de9d 100644 --- a/config.json +++ b/config.json @@ -28,7 +28,7 @@ }, // VOCABULARY PARAMETERS - "text":{ + "characters":{ "pad": "_", "eos": "~", "bos": "^", diff --git a/notebooks/Benchmark-PWGAN.ipynb b/notebooks/Benchmark-PWGAN.ipynb index 19a1a79c..840da10e 100644 --- a/notebooks/Benchmark-PWGAN.ipynb +++ b/notebooks/Benchmark-PWGAN.ipynb @@ -143,8 +143,8 @@ " speaker_id = None\n", "\n", "# if the vocabulary was passed, replace the default\n", - "if 'text' in CONFIG.keys():\n", - " symbols, phonemes = make_symbols(**CONFIG.text)\n", + "if 'characters' in CONFIG.keys():\n", + " symbols, phonemes = make_symbols(**CONFIG.characters)\n", "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", diff --git a/notebooks/Benchmark.ipynb b/notebooks/Benchmark.ipynb index bf6f2774..7d3a45cf 100644 --- a/notebooks/Benchmark.ipynb +++ b/notebooks/Benchmark.ipynb @@ -150,8 +150,8 @@ " speaker_id = None\n", "\n", "# if the vocabulary was passed, replace the default\n", - "if 'text' in CONFIG.keys():\n", - " symbols, phonemes = make_symbols(**CONFIG.text)\n", + "if 'characters' in CONFIG.keys():\n", + " symbols, phonemes = make_symbols(**CONFIG.characters)\n", "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", diff --git a/notebooks/ExtractTTSpectrogram.ipynb b/notebooks/ExtractTTSpectrogram.ipynb index 2313e47e..b5a88611 100644 --- a/notebooks/ExtractTTSpectrogram.ipynb +++ b/notebooks/ExtractTTSpectrogram.ipynb @@ -95,8 +95,8 @@ "outputs": [], "source": [ "# if the vocabulary was passed, replace the default\n", - "if 'text' in C.keys():\n", - " symbols, phonemes = make_symbols(**C.text)\n", + "if 'characters' in C.keys():\n", + " symbols, phonemes = make_symbols(**C.characters)\n", "\n", "# load the model\n", "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n", @@ -120,7 +120,7 @@ "preprocessor = importlib.import_module('TTS.datasets.preprocess')\n", "preprocessor = getattr(preprocessor, DATASET.lower())\n", "meta_data = preprocessor(DATA_PATH,METADATA_FILE)\n", - "dataset = MyDataset(checkpoint['r'], C.text_cleaner, ap, meta_data,tp=C.text if 'text' in C.keys() else None, use_phonemes=C.use_phonemes, phoneme_cache_path=C.phoneme_cache_path, enable_eos_bos=C.enable_eos_bos_chars)\n", + "dataset = MyDataset(checkpoint['r'], C.text_cleaner, ap, meta_data,tp=C.characters if 'characters' in C.keys() else None, use_phonemes=C.use_phonemes, phoneme_cache_path=C.phoneme_cache_path, enable_eos_bos=C.enable_eos_bos_chars)\n", "loader = DataLoader(dataset, batch_size=BATCH_SIZE, num_workers=4, collate_fn=dataset.collate_fn, shuffle=False, drop_last=False)" ] }, diff --git a/notebooks/TestAttention.ipynb b/notebooks/TestAttention.ipynb index b0599d80..9d3e5e75 100644 --- a/notebooks/TestAttention.ipynb +++ b/notebooks/TestAttention.ipynb @@ -111,8 +111,8 @@ " speaker_id = None\n", "\n", "# if the vocabulary was passed, replace the default\n", - "if 'text' in CONFIG.keys():\n", - " symbols, phonemes = make_symbols(**CONFIG.text)\n", + "if 'characters' in CONFIG.keys():\n", + " symbols, phonemes = make_symbols(**CONFIG.characters)\n", "\n", "# load the model\n", "num_chars = len(phonemes) if CONFIG.use_phonemes else len(symbols)\n", diff --git a/server/synthesizer.py b/server/synthesizer.py index f0921513..f73b73fc 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -52,8 +52,8 @@ class Synthesizer(object): self.use_phonemes = self.tts_config.use_phonemes self.ap = AudioProcessor(**self.tts_config.audio) - if 'text' in self.tts_config.keys(): - symbols, phonemes = make_symbols(**self.tts_config.text) + if 'characters' in self.tts_config.keys(): + symbols, phonemes = make_symbols(**self.tts_config.characters) if self.use_phonemes: self.input_size = len(phonemes) diff --git a/synthesize.py b/synthesize.py index 6f3a235f..1f1ce36f 100644 --- a/synthesize.py +++ b/synthesize.py @@ -108,8 +108,8 @@ if __name__ == "__main__": ap = AudioProcessor(**C.audio) # if the vocabulary was passed, replace the default - if 'text' in C.keys(): - symbols, phonemes = make_symbols(**C.text) + if 'characters' in C.keys(): + symbols, phonemes = make_symbols(**C.characters) # load speakers if args.speakers_json != '': diff --git a/tests/test_demo_server.py b/tests/test_demo_server.py index 36848942..a0837686 100644 --- a/tests/test_demo_server.py +++ b/tests/test_demo_server.py @@ -15,8 +15,8 @@ class DemoServerTest(unittest.TestCase): # pylint: disable=global-statement global symbols, phonemes config = load_config(os.path.join(get_tests_output_path(), 'dummy_model_config.json')) - if 'text' in config.keys(): - symbols, phonemes = make_symbols(**config.text) + if 'characters' in config.keys(): + symbols, phonemes = make_symbols(**config.characters) num_chars = len(phonemes) if config.use_phonemes else len(symbols) model = setup_model(num_chars, 0, config) diff --git a/tests/test_loader.py b/tests/test_loader.py index eb23ed19..d835c5d3 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -38,7 +38,7 @@ class TestTTSDataset(unittest.TestCase): c.text_cleaner, ap=self.ap, meta_data=items, - tp=c.text if 'text' in c.keys() else None, + tp=c.characters if 'characters' in c.keys() else None, batch_group_size=bgs, min_seq_len=c.min_seq_len, max_seq_len=float("inf"), diff --git a/train.py b/train.py index bf5429e9..4bb22a34 100644 --- a/train.py +++ b/train.py @@ -49,7 +49,7 @@ def setup_loader(ap, r, is_val=False, verbose=False): c.text_cleaner, meta_data=meta_data_eval if is_val else meta_data_train, ap=ap, - tp=c.text if 'text' in c.keys() else None, + tp=c.characters if 'characters' in c.keys() else None, batch_group_size=0 if is_val else c.batch_group_size * c.batch_size, min_seq_len=c.min_seq_len, @@ -520,8 +520,8 @@ def main(args): # pylint: disable=redefined-outer-name global meta_data_train, meta_data_eval, symbols, phonemes # Audio processor ap = AudioProcessor(**c.audio) - if 'text' in c.keys(): - symbols, phonemes = make_symbols(**c.text) + if 'characters' in c.keys(): + symbols, phonemes = make_symbols(**c.characters) # DISTRUBUTED if num_gpus > 1: diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 7c2f033a..cf0a05b4 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -426,13 +426,13 @@ def check_config(c): _check_argument('griffin_lim_iters', c['audio'], restricted=True, val_type=int, min_val=10, max_val=1000) # vocabulary parameters - _check_argument('text', c, restricted=False, val_type=dict) - _check_argument('pad', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) - _check_argument('eos', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) - _check_argument('bos', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) - _check_argument('characters', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) - _check_argument('phonemes', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) - _check_argument('punctuations', c['text'] if 'text' in c.keys() else {}, restricted='text' in c.keys(), val_type=str) + _check_argument('characters', c, restricted=False, val_type=dict) + _check_argument('pad', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) + _check_argument('eos', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) + _check_argument('bos', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) + _check_argument('characters', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) + _check_argument('phonemes', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) + _check_argument('punctuations', c['characters'] if 'characters' in c.keys() else {}, restricted='characters' in c.keys(), val_type=str) # normalization parameters _check_argument('signal_norm', c['audio'], restricted=True, val_type=bool) diff --git a/utils/synthesis.py b/utils/synthesis.py index c5ff2e70..42f0408c 100644 --- a/utils/synthesis.py +++ b/utils/synthesis.py @@ -10,10 +10,10 @@ def text_to_seqvec(text, CONFIG, use_cuda): seq = np.asarray( phoneme_to_sequence(text, text_cleaner, CONFIG.phoneme_language, CONFIG.enable_eos_bos_chars, - tp=CONFIG.text if 'text' in CONFIG.keys() else None), + tp=CONFIG.characters if 'characters' in CONFIG.keys() else None), dtype=np.int32) else: - seq = np.asarray(text_to_sequence(text, text_cleaner, tp=CONFIG.text if 'text' in CONFIG.keys() else None), dtype=np.int32) + seq = np.asarray(text_to_sequence(text, text_cleaner, tp=CONFIG.characters if 'characters' in CONFIG.keys() else None), dtype=np.int32) # torch tensor chars_var = torch.from_numpy(seq).unsqueeze(0) if use_cuda: diff --git a/utils/visual.py b/utils/visual.py index 3b24364c..1cb9ac5d 100644 --- a/utils/visual.py +++ b/utils/visual.py @@ -54,8 +54,8 @@ def visualize(alignment, spectrogram_postnet, stop_tokens, text, hop_length, CON plt.xlabel("Decoder timestamp", fontsize=label_fontsize) plt.ylabel("Encoder timestamp", fontsize=label_fontsize) if CONFIG.use_phonemes: - seq = phoneme_to_sequence(text, [CONFIG.text_cleaner], CONFIG.phoneme_language, CONFIG.enable_eos_bos_chars, tp=CONFIG.text if 'text' in CONFIG.keys() else None) - text = sequence_to_phoneme(seq, tp=CONFIG.text if 'text' in CONFIG.keys() else None) + seq = phoneme_to_sequence(text, [CONFIG.text_cleaner], CONFIG.phoneme_language, CONFIG.enable_eos_bos_chars, tp=CONFIG.characters if 'characters' in CONFIG.keys() else None) + text = sequence_to_phoneme(seq, tp=CONFIG.characters if 'characters' in CONFIG.keys() else None) print(text) plt.yticks(range(len(text)), list(text)) From 7ffc1025424e49e40a2b325892d359a3fe21b68c Mon Sep 17 00:00:00 2001 From: Edresson Casanova Date: Thu, 5 Mar 2020 17:44:47 -0300 Subject: [PATCH 6/6] add unittest for vocabulary parameters --- tests/test_config.json | 10 +++++++++ tests/test_text_processing.py | 42 ++++++++++++++++++++++++++--------- utils/text/symbols.py | 4 ++-- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/tests/test_config.json b/tests/test_config.json index 0cd3d751..6d63e6ab 100644 --- a/tests/test_config.json +++ b/tests/test_config.json @@ -19,6 +19,16 @@ "mel_fmax": 7600, // maximum freq level for mel-spec. Tune for dataset!! "do_trim_silence": false }, + + "characters":{ + "pad": "_", + "eos": "~", + "bos": "^", + "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", + "punctuations":"!'(),-.:;? ", + "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" + }, + "hidden_size": 128, "embedding_size": 256, "text_cleaner": "english_cleaners", diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py index aa17f694..6c0c7058 100644 --- a/tests/test_text_processing.py +++ b/tests/test_text_processing.py @@ -1,7 +1,14 @@ +import os +# pylint: disable=unused-wildcard-import +# pylint: disable=wildcard-import +# pylint: disable=unused-import import unittest -import torch as T - from TTS.utils.text import * +from TTS.tests import get_tests_path +from TTS.utils.generic_utils import load_config + +TESTS_PATH = get_tests_path() +conf = load_config(os.path.join(TESTS_PATH, 'test_config.json')) def test_phoneme_to_sequence(): text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" @@ -9,67 +16,80 @@ def test_phoneme_to_sequence(): lang = "en-us" sequence = phoneme_to_sequence(text, text_cleaner, lang) text_hat = sequence_to_phoneme(sequence) + sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) gt = "ɹiːsənt ɹɪsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪnkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹɪspɑːnsəbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjuːleɪʃən ænd lɜːnɪŋ!" - assert text_hat == gt + assert text_hat == text_hat_with_params == gt # multiple punctuations text = "Be a voice, not an! echo?" sequence = phoneme_to_sequence(text, text_cleaner, lang) text_hat = sequence_to_phoneme(sequence) + sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ?" print(text_hat) print(len(sequence)) - assert text_hat == gt + assert text_hat == text_hat_with_params == gt # not ending with punctuation text = "Be a voice, not an! echo" sequence = phoneme_to_sequence(text, text_cleaner, lang) text_hat = sequence_to_phoneme(sequence) + sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ" print(text_hat) print(len(sequence)) - assert text_hat == gt + assert text_hat == text_hat_with_params == gt # original text = "Be a voice, not an echo!" sequence = phoneme_to_sequence(text, text_cleaner, lang) text_hat = sequence_to_phoneme(sequence) + sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!" print(text_hat) print(len(sequence)) - assert text_hat == gt + assert text_hat == text_hat_with_params == gt # extra space after the sentence text = "Be a voice, not an! echo. " sequence = phoneme_to_sequence(text, text_cleaner, lang) text_hat = sequence_to_phoneme(sequence) + sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ." print(text_hat) print(len(sequence)) - assert text_hat == gt + assert text_hat == text_hat_with_params == gt # extra space after the sentence text = "Be a voice, not an! echo. " sequence = phoneme_to_sequence(text, text_cleaner, lang, True) text_hat = sequence_to_phoneme(sequence) + sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) gt = "^biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ.~" print(text_hat) print(len(sequence)) - assert text_hat == gt + assert text_hat == text_hat_with_params == gt # padding char text = "_Be a _voice, not an! echo_" sequence = phoneme_to_sequence(text, text_cleaner, lang) text_hat = sequence_to_phoneme(sequence) + sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ" print(text_hat) print(len(sequence)) - assert text_hat == gt - + assert text_hat == text_hat_with_params == gt def test_text2phone(): text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" gt = "ɹ|iː|s|ə|n|t| |ɹ|ɪ|s|ɜː|tʃ| |æ|t| |h|ɑːɹ|v|ɚ|d| |h|ɐ|z| |ʃ|oʊ|n| |m|ɛ|d|ᵻ|t|eɪ|ɾ|ɪ|ŋ| |f|ɔː|ɹ| |æ|z| |l|ɪ|ɾ|əl| |æ|z| |eɪ|t| |w|iː|k|s| |k|æ|n| |æ|k|tʃ|uː|əl|i| |ɪ|n|k|ɹ|iː|s|,| |ð|ə| |ɡ|ɹ|eɪ| |m|æ|ɾ|ɚ|ɹ| |ɪ|n|ð|ə| |p|ɑːɹ|t|s| |ʌ|v|ð|ə| |b|ɹ|eɪ|n| |ɹ|ɪ|s|p|ɑː|n|s|ə|b|əl| |f|ɔː|ɹ| |ɪ|m|oʊ|ʃ|ə|n|əl| |ɹ|ɛ|ɡ|j|uː|l|eɪ|ʃ|ə|n| |æ|n|d| |l|ɜː|n|ɪ|ŋ|!" lang = "en-us" ph = text2phone(text, lang) - assert gt == ph, f"\n{phonemes} \n vs \n{gt}" + assert gt == ph, f"\n{phonemes} \n vs \n{gt}" \ No newline at end of file diff --git a/utils/text/symbols.py b/utils/text/symbols.py index 15862cbd..544277c5 100644 --- a/utils/text/symbols.py +++ b/utils/text/symbols.py @@ -5,9 +5,9 @@ Defines the set of symbols used in text input to the model. The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. ''' -def make_symbols(characters, phnms, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'): +def make_symbols(characters, phonemes, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'):# pylint: disable=redefined-outer-name ''' Function to create symbols and phonemes ''' - _phonemes_sorted = sorted(list(phnms)) + _phonemes_sorted = sorted(list(phonemes)) # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters): _arpabet = ['@' + s for s in _phonemes_sorted]