diff --git a/.compute b/.compute index 02588f5b..cda787d2 100644 --- a/.compute +++ b/.compute @@ -1,14 +1,14 @@ #!/bin/bash yes | apt-get install sox yes | apt-get install ffmpeg -yes | apt-get install espeak +yes | apt-get install espeak yes | apt-get install tmux yes | apt-get install zsh sh -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)" pip3 install https://download.pytorch.org/whl/cu100/torch-1.3.0%2Bcu100-cp36-cp36m-linux_x86_64.whl sudo sh install.sh -pip install pytorch==1.3.0+cu100 -python3 setup.py develop +# pip install pytorch==1.7.0+cu100 +# python3 setup.py develop # python3 distribute.py --config_path config.json --data_path /data/ro/shared/data/keithito/LJSpeech-1.1/ # cp -R ${USER_DIR}/Mozilla_22050 ../tmp/ # python3 distribute.py --config_path config_tacotron_gst.json --data_path ../tmp/Mozilla_22050/ diff --git a/.travis/script b/.travis/script index 0c24a221..0860f9cf 100755 --- a/.travis/script +++ b/.travis/script @@ -17,5 +17,6 @@ fi if [[ "$TEST_SUITE" == "testscripts" ]]; then # test model training scripts ./tests/test_tts_train.sh - ./tests/test_vocoder_train.sh + ./tests/test_vocoder_gan_train.sh + ./tests/test_vocoder_wavernn_train.sh fi diff --git a/README.md b/README.md index d2bfbeee..e3c24d3b 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ TTS paper collection: https://github.com/erogol/TTS-papers ## TTS Performance

-"Mozilla*" and "Judy*" are our models. +"Mozilla*" and "Judy*" are our models. [Details...](https://github.com/mozilla/TTS/wiki/Mean-Opinion-Score-Results) ## Provided Models and Methods @@ -47,7 +47,10 @@ Speaker Encoder: Vocoders: - MelGAN: [paper](https://arxiv.org/abs/1710.10467) - MultiBandMelGAN: [paper](https://arxiv.org/abs/2005.05106) +- ParallelWaveGAN: [paper](https://arxiv.org/abs/1910.11480) - GAN-TTS discriminators: [paper](https://arxiv.org/abs/1909.11646) +- WaveRNN: [origin][https://github.com/fatchord/WaveRNN/] +- WaveGrad: [paper][https://arxiv.org/abs/2009.00713] You can also help us implement more models. Some TTS related work can be found [here](https://github.com/erogol/TTS-papers). @@ -70,8 +73,8 @@ You can also help us implement more models. Some TTS related work can be found [ ## Main Requirements and Installation Highly recommended to use [miniconda](https://conda.io/miniconda.html) for easier installation. * python>=3.6 - * pytorch>=1.4.1 - * tensorflow>=2.2 + * pytorch>=1.5.0 + * tensorflow>=2.3 * librosa * tensorboard * tensorboardX @@ -149,23 +152,25 @@ head -n 12000 metadata_shuf.csv > metadata_train.csv tail -n 1100 metadata_shuf.csv > metadata_val.csv ``` -To train a new model, you need to define your own ```config.json``` file (check the example) and call with the command below. You also set the model architecture in ```config.json```. +To train a new model, you need to define your own ```config.json``` to define model details, trainin configuration and more (check the examples). Then call the corressponding train script. -```python TTS/bin/train_tts.py --config_path TTS/tts/configs/config.json``` +For instance, in order to train a tacotron or tacotron2 model on LJSpeech dataset, follow these steps. + +```python TTS/bin/train_tacotron.py --config_path TTS/tts/configs/config.json``` To fine-tune a model, use ```--restore_path```. -```python TTS/bin/train_tts.py --config_path TTS/tts/configs/config.json --restore_path /path/to/your/model.pth.tar``` +```python TTS/bin/train_tacotron.py --config_path TTS/tts/configs/config.json --restore_path /path/to/your/model.pth.tar``` To continue an old training run, use ```--continue_path```. -```python TTS/bin/train_tts.py --continue_path /path/to/your/run_folder/``` +```python TTS/bin/train_tacotron.py --continue_path /path/to/your/run_folder/``` -For multi-GPU training use ```distribute.py```. It enables process based multi-GPU training where each process uses a single GPU. +For multi-GPU training, call ```distribute.py```. It runs any provided train script in multi-GPU setting. -```CUDA_VISIBLE_DEVICES="0,1,4" TTS/bin/distribute.py --config_path TTS/tts/configs/config.json``` +```CUDA_VISIBLE_DEVICES="0,1,4" python TTS/bin/distribute.py --script train_tacotron.py --config_path TTS/tts/configs/config.json``` -Each run creates a new output folder and ```config.json``` is copied under this folder. +Each run creates a new output folder accomodating used ```config.json```, model checkpoints and tensorboard logs. In case of any error or intercepted execution, if there is no checkpoint yet under the output folder, the whole folder is going to be removed. @@ -199,7 +204,7 @@ If you like to use TTS to try a new idea and like to share your experiments with - [x] Train TTS with r=1 successfully. - [x] Enable process based distributed training. Similar to (https://github.com/fastai/imagenet-fast/). - [x] Adapting Neural Vocoder. TTS works with WaveRNN and ParallelWaveGAN (https://github.com/erogol/WaveRNN and https://github.com/erogol/ParallelWaveGAN) -- [ ] Multi-speaker embedding. +- [x] Multi-speaker embedding. - [x] Model optimization (model export, model pruning etc.) 1\u001b[0;31m \u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAP\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspectrogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwav\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Max:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Min:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Mean:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mplot_spectrogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Projects/TTS/tts/utils/audio.py\u001b[0m in \u001b[0;36mspectrogram\u001b[0;34m(self, y)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0mD\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stft\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mS\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_amp_to_db\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_normalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmelspectrogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Projects/TTS/tts/utils/audio.py\u001b[0m in \u001b[0;36m_normalize\u001b[0;34m(self, S)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear_scaler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mS\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' [!] Mean-Var stats does not match the given feature dimensions.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 120\u001b[0m \u001b[0;31m# range normalization\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0mS\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mref_level_db\u001b[0m \u001b[0;31m# discard certain range of DB assuming it is air noise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mRuntimeError\u001b[0m: [!] Mean-Var stats does not match the given feature dimensions." + ] + } + ], + "source": [ + "spec = AP.spectrogram(wav)\n", + "print(\"Max:\", spec.max())\n", + "print(\"Min:\", spec.min())\n", + "print(\"Mean:\", spec.mean())\n", + "plot_spectrogram(spec.T, AP);\n", + "\n", + "wav_gen = AP.inv_spectrogram(spec)\n", + "ipd.Audio(wav_gen, rate=AP.sample_rate)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "Collapsed": "false" + }, + "source": [ + "### Compare values for a certain parameter\n", + "\n", + "Optimize your parameters by comparing different values per parameter at a time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "audio={\n", + " 'audio_processor': 'audio',\n", + " 'num_mels': 80, # In general, you don'tneed to change it \n", + " 'num_freq': 1025, # In general, you don'tneed to change it \n", + " 'sample_rate': 22050, # It depends to the sample rate of the dataset.\n", + " 'frame_length_ms': 50, # In general, you don'tneed to change it \n", + " 'frame_shift_ms': 12.5, # In general, you don'tneed to change it \n", + " 'preemphasis': 0.98, # In general, 0 gives better voice recovery but makes traning harder. If your model does not train, try 0.97 - 0.99.\n", + " 'min_level_db': -100,\n", + " 'ref_level_db': 20, # It is the base DB, higher until you remove the background noise in the spectrogram and then lower until you hear a better speech below.\n", + " 'power': 1.5, # Change this value and listen the synthesized voice. 1.2 - 1.5 are some resonable values.\n", + " 'griffin_lim_iters': 60, # It does not give any imporvement for values > 60\n", + " 'signal_norm': True, # This is more about your model. It does not give any change for the synthsis performance.\n", + " 'symmetric_norm': False, # Same as above\n", + " 'max_norm': 1, # Same as above\n", + " 'clip_norm': True, # Same as above\n", + " 'mel_fmin': 0.0, # You can play with this and check mel-spectrogram based voice synthesis below.\n", + " 'mel_fmax': 8000.0, # You can play with this and check mel-spectrogram based voice synthesis below.\n", + " 'do_trim_silence': True} # If you dataset has some silience at the beginning or end, this trims it. Check the AP.load_wav() below,if it causes any difference for the loaded audio file.\n", + "\n", + "AP = AudioProcessor(**audio);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "from librosa import display\n", + "from matplotlib import pylab as plt\n", + "import IPython\n", + "plt.rcParams['figure.figsize'] = (20.0, 16.0)\n", + "\n", + "def compare_values(attribute, values, file):\n", + " \"\"\"\n", + " attributes (str): the names of the attribute you like to test.\n", + " values (list): list of values to compare.\n", + " file (str): file name to perform the tests.\n", + " \"\"\"\n", + " wavs = []\n", + " for idx, val in enumerate(values):\n", + " set_val_cmd = \"AP.{}={}\".format(attribute, val)\n", + " exec(set_val_cmd)\n", + " wav = AP.load_wav(file)\n", + " spec = AP.spectrogram(wav)\n", + " spec_norm = AP._denormalize(spec.T)\n", + " plt.subplot(len(values), 2, 2*idx + 1)\n", + " plt.imshow(spec_norm.T, aspect=\"auto\", origin=\"lower\")\n", + " # plt.colorbar()\n", + " plt.tight_layout()\n", + " wav_gen = AP.inv_spectrogram(spec)\n", + " wavs.append(wav_gen)\n", + " plt.subplot(len(values), 2, 2*idx + 2)\n", + " display.waveplot(wav, alpha=0.5)\n", + " display.waveplot(wav_gen, alpha=0.25)\n", + " plt.title(\"{}={}\".format(attribute, val))\n", + " plt.tight_layout()\n", + " \n", + " wav = AP.load_wav(file)\n", + " print(\" > Ground-truth\")\n", + " IPython.display.display(IPython.display.Audio(wav, rate=AP.sample_rate))\n", + " \n", + " for idx, wav_gen in enumerate(wavs):\n", + " val = values[idx]\n", + " print(\" > {} = {}\".format(attribute, val))\n", + " IPython.display.display(IPython.display.Audio(wav_gen, rate=AP.sample_rate))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "compare_values(\"preemphasis\", [0, 0.5, 0.97, 0.98, 0.99], file_paths[10])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "compare_values(\"ref_level_db\", [10, 15, 20, 25, 30, 35, 40], file_paths[10])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/requirements.txt b/requirements.txt index 7887ac5a..31b49916 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,4 @@ pylint==2.5.3 gdown umap-learn cython +pyyaml diff --git a/run_tests.sh b/run_tests.sh index 27f54b24..9abe7803 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,3 +1,4 @@ +set -e TF_CPP_MIN_LOG_LEVEL=3 # tests @@ -6,7 +7,10 @@ nosetests tests -x &&\ # runtime tests ./tests/test_server_package.sh && \ ./tests/test_tts_train.sh && \ -./tests/test_vocoder_train.sh && \ +./tests/test_glow-tts_train.sh && \ +./tests/test_vocoder_gan_train.sh && \ +./tests/test_vocoder_wavernn_train.sh && \ +./tests/test_vocoder_wavegrad_train.sh && \ # linter check cardboardlinter --refspec master \ No newline at end of file diff --git a/setup.py b/setup.py index 0127e84d..2893036a 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ args, unknown_args = parser.parse_known_args() # Remove our arguments from argv so that setuptools doesn't see them sys.argv = [sys.argv[0]] + unknown_args -version = '0.0.5' +version = '0.0.6' # Adapted from https://github.com/pytorch/pytorch cwd = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/inputs/test_glow_tts.json b/tests/inputs/test_glow_tts.json new file mode 100644 index 00000000..c1bc33fd --- /dev/null +++ b/tests/inputs/test_glow_tts.json @@ -0,0 +1,134 @@ +{ + "model": "glow_tts", + "run_name": "glow-tts-gatedconv", + "run_description": "glow-tts model training with gated conv.", + + // AUDIO PARAMETERS + "audio":{ + "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. + + // Griffin-Lim + "power": 1.1, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! + "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 1.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // VOCABULARY PARAMETERS + // if custom character set is not defined, + // default set in symbols.py is used + // "characters":{ + // "pad": "_", + // "eos": "~", + // "bos": "^", + // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", + // "punctuations":"!'(),-.:;? ", + // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" + // }, + + "add_blank": false, // if true add a new token after each token of the sentence. This increases the size of the input sequence, but has considerably improved the prosody of the GlowTTS model. + + // DISTRIBUTED TRAINING + "mixed_precision": false, + "distributed":{ + "backend": "nccl", + "url": "tcp:\/\/localhost:54323" + }, + + "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. + + // MODEL PARAMETERS + "use_mas": false, // use Monotonic Alignment Search if true. Otherwise use pre-computed attention alignments. + + // TRAINING + "batch_size": 2, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. + "eval_batch_size":1, + "r": 1, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. + "loss_masking": true, // enable / disable loss masking against the sequence padding. + + // VALIDATION + "run_eval": true, + "test_delay_epochs": 0, //Until attention is aligned, testing only wastes computation time. + "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. + + // OPTIMIZER + "noam_schedule": true, // use noam warmup and lr schedule. + "grad_clip": 5.0, // upper limit for gradients for clipping. + "epochs": 1, // total number of epochs to train. + "lr": 1e-3, // Initial learning rate. If Noam decay is active, maximum learning rate. + "wd": 0.000001, // Weight decay weight. + "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. + + "encoder_type": "gatedconv", + + // TENSORBOARD and LOGGING + "print_step": 25, // Number of steps to log training on console. + "tb_plot_step": 100, // Number of steps to plot TB training figures. + "print_eval": false, // If True, it prints intermediate loss values in evalulation. + "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + "apex_amp_level": null, + + // DATA LOADING + "text_cleaner": "phoneme_cleaners", + "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "batch_group_size": 0, //Number of batches to shuffle after bucketing. + "min_seq_len": 3, // DATASET-RELATED: minimum text length to use in training + "max_seq_len": 500, // DATASET-RELATED: maximum text length + "compute_f0": false, // compute f0 values in data-loader + + // PATHS + "output_path": "tests/train_outputs/", + + // PHONEMES + "phoneme_cache_path": "tests/outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. + "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages + + // MULTI-SPEAKER and GST + "use_external_speaker_embedding_file": false, + "external_speaker_embedding_file": null, + "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. + + // DATASETS + "datasets": // List of datasets. They all merged and they get different speaker_ids. + [ + { + "name": "ljspeech", + "path": "tests/data/ljspeech/", + "meta_file_train": "metadata.csv", + "meta_file_val": "metadata.csv" + } + ] +} + + diff --git a/tests/inputs/test_train_config.json b/tests/inputs/test_train_config.json index ddb71384..5b2dff2d 100644 --- a/tests/inputs/test_train_config.json +++ b/tests/inputs/test_train_config.json @@ -67,13 +67,24 @@ "gradual_training": [[0, 7, 4]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. "loss_masking": true, // enable / disable loss masking against the sequence padding. "ga_alpha": 10.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "apex_amp_level": null, + "mixed_precision": false, // VALIDATION "run_eval": true, "test_delay_epochs": 0, //Until attention is aligned, testing only wastes computation time. "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. + // LOSS SETTINGS + "loss_masking": true, // enable / disable loss masking against the sequence padding. + "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled + "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled + "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled + "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled + "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled + "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled + "ga_alpha": 5.0, // weight for guided attention loss. If > 0, guided attention is enabled. + "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. + // OPTIMIZER "noam_schedule": false, // use noam warmup and lr schedule. "grad_clip": 1.0, // upper limit for gradients for clipping. diff --git a/tests/inputs/test_vocoder_wavegrad.json b/tests/inputs/test_vocoder_wavegrad.json new file mode 100644 index 00000000..8fa0bbe1 --- /dev/null +++ b/tests/inputs/test_vocoder_wavegrad.json @@ -0,0 +1,114 @@ +{ + "run_name": "wavegrad-ljspeech", + "run_description": "wavegrad ljspeech", + + "audio":{ + "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! + "spec_gain": 1.0, // scaler value appplied after log transform of spectrogram. + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // DISTRIBUTED TRAINING + "mixed_precision": false, + "distributed":{ + "backend": "nccl", + "url": "tcp:\/\/localhost:54322" + }, + + "target_loss": "avg_wavegrad_loss", // loss value to pick the best model to save after each epoch + + // MODEL PARAMETERS + "generator_model": "wavegrad", + "model_params":{ + "y_conv_channels":32, + "x_conv_channels":768, + "ublock_out_channels": [512, 512, 256, 128, 128], + "dblock_out_channels": [128, 128, 256, 512], + "upsample_factors": [4, 4, 4, 2, 2], + "upsample_dilations": [ + [1, 2, 1, 2], + [1, 2, 1, 2], + [1, 2, 4, 8], + [1, 2, 4, 8], + [1, 2, 4, 8]], + "use_weight_norm": true + }, + + // DATASET + "data_path": "tests/data/ljspeech/wavs/", // root data path. It finds all wav files recursively from there. + "feature_path": null, // if you use precomputed features + "seq_len": 6144, // 24 * hop_length + "pad_short": 0, // additional padding for short wavs + "conv_pad": 0, // additional padding against convolutions applied to spectrograms + "use_noise_augment": false, // add noise to the audio signal for augmentation + "use_cache": true, // use in memory cache to keep the computed features. This might cause OOM. + + "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. + + // TRAINING + "batch_size": 1, // Batch size for training. + "train_noise_schedule":{ + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 1000 + }, + "test_noise_schedule":{ + "min_val": 1e-6, + "max_val": 1e-2, + "num_steps": 2 + }, + + // VALIDATION + "run_eval": true, // enable/disable evaluation run + + // OPTIMIZER + "epochs": 1, // total number of epochs to train. + "clip_grad": 1.0, // Generator gradient clipping threshold. Apply gradient clipping if > 0 + "lr_scheduler": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate + "lr_scheduler_params": { + "gamma": 0.5, + "milestones": [100000, 200000, 300000, 400000, 500000, 600000] + }, + "lr": 1e-4, // Initial learning rate. If Noam decay is active, maximum learning rate. + + // TENSORBOARD and LOGGING + "print_step": 250, // Number of steps to log traning on console. + "print_eval": false, // If True, it prints loss values for each step in eval run. + "save_step": 10000, // Number of training steps expected to plot training stats on TB and save model checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + // DATA LOADING + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "eval_split_size": 4, + + // PATHS + "output_path": "tests/train_outputs/" +} + diff --git a/tests/inputs/test_vocoder_wavernn_config.json b/tests/inputs/test_vocoder_wavernn_config.json new file mode 100644 index 00000000..9df32fef --- /dev/null +++ b/tests/inputs/test_vocoder_wavernn_config.json @@ -0,0 +1,107 @@ +{ + "run_name": "wavernn_test", + "run_description": "wavernn_test training", + + // AUDIO PARAMETERS + "audio":{ + "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 0, // reference level db, theoretically 20db is the sound of air. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! + "spec_gain": 20.0, // scaler value appplied after log transform of spectrogram. + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // Generating / Synthesizing + "batched": true, + "target_samples": 11000, // target number of samples to be generated in each batch entry + "overlap_samples": 550, // number of samples for crossfading between batches + + // DISTRIBUTED TRAINING + // "distributed":{ + // "backend": "nccl", + // "url": "tcp:\/\/localhost:54321" + // }, + + // MODEL PARAMETERS + "use_aux_net": true, + "use_upsample_net": true, + "upsample_factors": [4, 8, 8], // this needs to correctly factorise hop_length + "seq_len": 1280, // has to be devideable by hop_length + "mode": "mold", // mold [string], gauss [string], bits [int] + "mulaw": false, // apply mulaw if mode is bits + "padding": 2, // pad the input for resnet to see wider input length + + // DATASET + //"use_gta": true, // use computed gta features from the tts model + "data_path": "tests/data/ljspeech/wavs/", // path containing training wav files + "feature_path": null, // path containing computed features from wav files if null compute them + + // MODEL PARAMETERS + "wavernn_model_params": { + "rnn_dims": 512, + "fc_dims": 512, + "compute_dims": 128, + "res_out_dims": 128, + "num_res_blocks": 10, + "use_aux_net": true, + "use_upsample_net": true, + "upsample_factors": [4, 8, 8] // this needs to correctly factorise hop_length + }, + "mixed_precision": false, + + // TRAINING + "batch_size": 4, // Batch size for training. Lower values than 32 might cause hard to learn attention. + "epochs": 1, // total number of epochs to train. + + // VALIDATION + "run_eval": true, + "test_every_epochs": 10, // Test after set number of epochs (Test every 20 epochs for example) + + // OPTIMIZER + "grad_clip": 4, // apply gradient clipping if > 0 + "lr_scheduler": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate + "lr_scheduler_params": { + "gamma": 0.5, + "milestones": [200000, 400000, 600000] + }, + "lr": 1e-4, // initial learning rate + + // TENSORBOARD and LOGGING + "print_step": 25, // Number of steps to log traning on console. + "print_eval": false, // If True, it prints loss values for each step in eval run. + "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + // DATA LOADING + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "eval_split_size": 10, // number of samples for testing + + // PATHS + "output_path": "tests/train_outputs/" +} + diff --git a/tests/test_encoder.py b/tests/test_encoder.py index a646eaa6..4d4dbba1 100644 --- a/tests/test_encoder.py +++ b/tests/test_encoder.py @@ -62,7 +62,7 @@ class GE2ELossTests(unittest.TestCase): assert output.item() >= 0.0 # check speaker loss with orthogonal d-vectors dummy_input = T.empty(3, 64) - dummy_input = T.nn.init.orthogonal(dummy_input) + dummy_input = T.nn.init.orthogonal_(dummy_input) dummy_input = T.cat( [ dummy_input[0].repeat(5, 1, 1).transpose(0, 1), @@ -91,7 +91,7 @@ class AngleProtoLossTests(unittest.TestCase): # check speaker loss with orthogonal d-vectors dummy_input = T.empty(3, 64) - dummy_input = T.nn.init.orthogonal(dummy_input) + dummy_input = T.nn.init.orthogonal_(dummy_input) dummy_input = T.cat( [ dummy_input[0].repeat(5, 1, 1).transpose(0, 1), diff --git a/tests/test_glow-tts_train.sh b/tests/test_glow-tts_train.sh new file mode 100755 index 00000000..add7292d --- /dev/null +++ b/tests/test_glow-tts_train.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -xe +BASEDIR=$(dirname "$0") +echo "$BASEDIR" +# run training +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_glow_tts.py --config_path $BASEDIR/inputs/test_glow_tts.json +# find the training folder +LATEST_FOLDER=$(ls $BASEDIR/train_outputs/| sort | tail -1) +echo $LATEST_FOLDER +# continue the previous training +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_glow_tts.py --continue_path $BASEDIR/train_outputs/$LATEST_FOLDER +# remove all the outputs +rm -rf $BASEDIR/train_outputs/ diff --git a/tests/test_layers.py b/tests/test_layers.py index 57be51e5..5426e195 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -2,7 +2,7 @@ import unittest import torch as T from TTS.tts.layers.tacotron import Prenet, CBHG, Decoder, Encoder -from TTS.tts.layers.losses import L1LossMasked +from TTS.tts.layers.losses import L1LossMasked, SSIMLoss from TTS.tts.utils.generic_utils import sequence_mask # pylint: disable=unused-variable @@ -149,3 +149,72 @@ class L1LossMaskedTests(unittest.TestCase): (sequence_mask(dummy_length).float() - 1.0) * 100.0).unsqueeze(2) output = layer(dummy_input + mask, dummy_target, dummy_length) assert output.item() == 0, "0 vs {}".format(output.item()) + + +class SSIMLossTests(unittest.TestCase): + def test_in_out(self): #pylint: disable=no-self-use + # test input == target + layer = SSIMLoss() + dummy_input = T.ones(4, 8, 128).float() + dummy_target = T.ones(4, 8, 128).float() + dummy_length = (T.ones(4) * 8).long() + output = layer(dummy_input, dummy_target, dummy_length) + assert output.item() == 0.0 + + # test input != target + dummy_input = T.ones(4, 8, 128).float() + dummy_target = T.zeros(4, 8, 128).float() + dummy_length = (T.ones(4) * 8).long() + output = layer(dummy_input, dummy_target, dummy_length) + assert abs(output.item() - 1.0) < 1e-4 , "1.0 vs {}".format(output.item()) + + # test if padded values of input makes any difference + dummy_input = T.ones(4, 8, 128).float() + dummy_target = T.zeros(4, 8, 128).float() + dummy_length = (T.arange(5, 9)).long() + mask = ( + (sequence_mask(dummy_length).float() - 1.0) * 100.0).unsqueeze(2) + output = layer(dummy_input + mask, dummy_target, dummy_length) + assert abs(output.item() - 1.0) < 1e-4, "1.0 vs {}".format(output.item()) + + dummy_input = T.rand(4, 8, 128).float() + dummy_target = dummy_input.detach() + dummy_length = (T.arange(5, 9)).long() + mask = ( + (sequence_mask(dummy_length).float() - 1.0) * 100.0).unsqueeze(2) + output = layer(dummy_input + mask, dummy_target, dummy_length) + assert output.item() == 0, "0 vs {}".format(output.item()) + + # seq_len_norm = True + # test input == target + layer = L1LossMasked(seq_len_norm=True) + dummy_input = T.ones(4, 8, 128).float() + dummy_target = T.ones(4, 8, 128).float() + dummy_length = (T.ones(4) * 8).long() + output = layer(dummy_input, dummy_target, dummy_length) + assert output.item() == 0.0 + + # test input != target + dummy_input = T.ones(4, 8, 128).float() + dummy_target = T.zeros(4, 8, 128).float() + dummy_length = (T.ones(4) * 8).long() + output = layer(dummy_input, dummy_target, dummy_length) + assert output.item() == 1.0, "1.0 vs {}".format(output.item()) + + # test if padded values of input makes any difference + dummy_input = T.ones(4, 8, 128).float() + dummy_target = T.zeros(4, 8, 128).float() + dummy_length = (T.arange(5, 9)).long() + mask = ( + (sequence_mask(dummy_length).float() - 1.0) * 100.0).unsqueeze(2) + output = layer(dummy_input + mask, dummy_target, dummy_length) + assert abs(output.item() - 1.0) < 1e-5, "1.0 vs {}".format(output.item()) + + dummy_input = T.rand(4, 8, 128).float() + dummy_target = dummy_input.detach() + dummy_length = (T.arange(5, 9)).long() + mask = ( + (sequence_mask(dummy_length).float() - 1.0) * 100.0).unsqueeze(2) + output = layer(dummy_input + mask, dummy_target, dummy_length) + assert output.item() == 0, "0 vs {}".format(output.item()) + diff --git a/tests/test_server_package.sh b/tests/test_server_package.sh index 83ffc6f0..7e75415a 100755 --- a/tests/test_server_package.sh +++ b/tests/test_server_package.sh @@ -6,12 +6,12 @@ if [[ ! -f tests/outputs/checkpoint_10.pth.tar ]]; then exit 1 fi +rm -f dist/*.whl +python setup.py --quiet bdist_wheel --checkpoint tests/outputs/checkpoint_10.pth.tar --model_config tests/outputs/dummy_model_config.json + python -m venv /tmp/venv source /tmp/venv/bin/activate pip install --quiet --upgrade pip setuptools wheel - -rm -f dist/*.whl -python setup.py --quiet bdist_wheel --checkpoint tests/outputs/checkpoint_10.pth.tar --model_config tests/outputs/dummy_model_config.json pip install --quiet dist/TTS*.whl # this is related to https://github.com/librosa/librosa/issues/1160 diff --git a/tests/test_tacotron_model.py b/tests/test_tacotron_model.py index 8309aa58..c56a6565 100644 --- a/tests/test_tacotron_model.py +++ b/tests/test_tacotron_model.py @@ -294,6 +294,7 @@ class SCGSTMultiSpeakeTacotronTrainTest(unittest.TestCase): mel_spec = torch.rand(8, 30, c.audio['num_mels']).to(device) linear_spec = torch.rand(8, 30, c.audio['fft_size']).to(device) mel_lengths = torch.randint(20, 30, (8, )).long().to(device) + mel_lengths[-1] = mel_spec.size(1) stop_targets = torch.zeros(8, 30, 1).float().to(device) speaker_embeddings = torch.rand(8, 55).to(device) diff --git a/tests/test_tacotron_train.sh b/tests/test_tacotron_train.sh new file mode 100755 index 00000000..8138fb75 --- /dev/null +++ b/tests/test_tacotron_train.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -xe +BASEDIR=$(dirname "$0") +echo "$BASEDIR" +# run training +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tts.py --config_path $BASEDIR/inputs/test_train_config.json +# find the training folder +LATEST_FOLDER=$(ls $BASEDIR/train_outputs/| sort | tail -1) +echo $LATEST_FOLDER +# continue the previous training +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tts.py --continue_path $BASEDIR/train_outputs/$LATEST_FOLDER +# remove all the outputs +rm -rf $BASEDIR/train_outputs/ diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py index 1eb9f9a8..ae3250a8 100644 --- a/tests/test_text_processing.py +++ b/tests/test_text_processing.py @@ -11,6 +11,7 @@ from TTS.utils.io import load_config conf = load_config(os.path.join(get_tests_input_path(), 'test_config.json')) def test_phoneme_to_sequence(): + text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" text_cleaner = ["phoneme_cleaners"] lang = "en-us" @@ -20,7 +21,7 @@ def test_phoneme_to_sequence(): text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters) gt = "ɹiːsənt ɹɪsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪnkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹɪspɑːnsəbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjuːleɪʃən ænd lɜːnɪŋ!" assert text_hat == text_hat_with_params == gt - + # multiple punctuations text = "Be a voice, not an! echo?" sequence = phoneme_to_sequence(text, text_cleaner, lang) @@ -87,6 +88,84 @@ def test_phoneme_to_sequence(): print(len(sequence)) assert text_hat == text_hat_with_params == gt +def test_phoneme_to_sequence_with_blank_token(): + + text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" + text_cleaner = ["phoneme_cleaners"] + lang = "en-us" + sequence = phoneme_to_sequence(text, text_cleaner, lang) + text_hat = sequence_to_phoneme(sequence) + _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) + gt = "ɹiːsənt ɹɪsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪnkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹɪspɑːnsəbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjuːleɪʃən ænd lɜːnɪŋ!" + assert text_hat == text_hat_with_params == gt + + # multiple punctuations + text = "Be a voice, not an! echo?" + sequence = phoneme_to_sequence(text, text_cleaner, lang) + text_hat = sequence_to_phoneme(sequence) + _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) + gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ?" + print(text_hat) + print(len(sequence)) + assert text_hat == text_hat_with_params == gt + + # not ending with punctuation + text = "Be a voice, not an! echo" + sequence = phoneme_to_sequence(text, text_cleaner, lang) + text_hat = sequence_to_phoneme(sequence) + _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) + gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ" + print(text_hat) + print(len(sequence)) + assert text_hat == text_hat_with_params == gt + + # original + text = "Be a voice, not an echo!" + sequence = phoneme_to_sequence(text, text_cleaner, lang) + text_hat = sequence_to_phoneme(sequence) + _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) + gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!" + print(text_hat) + print(len(sequence)) + assert text_hat == text_hat_with_params == gt + + # extra space after the sentence + text = "Be a voice, not an! echo. " + sequence = phoneme_to_sequence(text, text_cleaner, lang) + text_hat = sequence_to_phoneme(sequence) + _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) + gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ." + print(text_hat) + print(len(sequence)) + assert text_hat == text_hat_with_params == gt + + # extra space after the sentence + text = "Be a voice, not an! echo. " + sequence = phoneme_to_sequence(text, text_cleaner, lang, True) + text_hat = sequence_to_phoneme(sequence) + _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) + gt = "^biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ.~" + print(text_hat) + print(len(sequence)) + assert text_hat == text_hat_with_params == gt + + # padding char + text = "_Be a _voice, not an! echo_" + sequence = phoneme_to_sequence(text, text_cleaner, lang) + text_hat = sequence_to_phoneme(sequence) + _ = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters, add_blank=True) + text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters, add_blank=True) + gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ" + print(text_hat) + print(len(sequence)) + assert text_hat == text_hat_with_params == gt + def test_text2phone(): text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" gt = "ɹ|iː|s|ə|n|t| |ɹ|ɪ|s|ɜː|tʃ| |æ|t| |h|ɑːɹ|v|ɚ|d| |h|ɐ|z| |ʃ|oʊ|n| |m|ɛ|d|ᵻ|t|eɪ|ɾ|ɪ|ŋ| |f|ɔː|ɹ| |æ|z| |l|ɪ|ɾ|əl| |æ|z| |eɪ|t| |w|iː|k|s| |k|æ|n| |æ|k|tʃ|uː|əl|i| |ɪ|n|k|ɹ|iː|s|,| |ð|ə| |ɡ|ɹ|eɪ| |m|æ|ɾ|ɚ|ɹ| |ɪ|n|ð|ə| |p|ɑːɹ|t|s| |ʌ|v|ð|ə| |b|ɹ|eɪ|n| |ɹ|ɪ|s|p|ɑː|n|s|ə|b|əl| |f|ɔː|ɹ| |ɪ|m|oʊ|ʃ|ə|n|əl| |ɹ|ɛ|ɡ|j|uː|l|eɪ|ʃ|ə|n| |æ|n|d| |l|ɜː|n|ɪ|ŋ|!" diff --git a/tests/test_tts_train.sh b/tests/test_tts_train.sh index 55379a1e..9268ea96 100755 --- a/tests/test_tts_train.sh +++ b/tests/test_tts_train.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash - +set -xe BASEDIR=$(dirname "$0") echo "$BASEDIR" # run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tts.py --config_path $BASEDIR/inputs/test_train_config.json +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --config_path $BASEDIR/inputs/test_train_config.json # find the training folder LATEST_FOLDER=$(ls $BASEDIR/train_outputs/| sort | tail -1) echo $LATEST_FOLDER # continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tts.py --continue_path $BASEDIR/train_outputs/$LATEST_FOLDER +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_tacotron.py --continue_path $BASEDIR/train_outputs/$LATEST_FOLDER # remove all the outputs rm -rf $BASEDIR/train_outputs/ diff --git a/tests/test_vocoder_datasets.py b/tests/test_vocoder_gan_datasets.py similarity index 100% rename from tests/test_vocoder_datasets.py rename to tests/test_vocoder_gan_datasets.py diff --git a/tests/test_vocoder_train.sh b/tests/test_vocoder_gan_train.sh similarity index 56% rename from tests/test_vocoder_train.sh rename to tests/test_vocoder_gan_train.sh index fa99b4bd..0ed2b599 100755 --- a/tests/test_vocoder_train.sh +++ b/tests/test_vocoder_gan_train.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash - +set -xe BASEDIR=$(dirname "$0") echo "$BASEDIR" # create run dir mkdir $BASEDIR/train_outputs # run training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder.py --config_path $BASEDIR/inputs/test_vocoder_multiband_melgan_config.json +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_gan.py --config_path $BASEDIR/inputs/test_vocoder_multiband_melgan_config.json # find the training folder LATEST_FOLDER=$(ls $BASEDIR/train_outputs/| sort | tail -1) echo $LATEST_FOLDER # continue the previous training -CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder.py --continue_path $BASEDIR/train_outputs/$LATEST_FOLDER +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_gan.py --continue_path $BASEDIR/train_outputs/$LATEST_FOLDER # remove all the outputs rm -rf $BASEDIR/train_outputs/$LATEST_FOLDER diff --git a/tests/test_vocoder_wavegrad_train.sh b/tests/test_vocoder_wavegrad_train.sh new file mode 100755 index 00000000..33ffe865 --- /dev/null +++ b/tests/test_vocoder_wavegrad_train.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -xe +BASEDIR=$(dirname "$0") +echo "$BASEDIR" +# create run dir +mkdir -p $BASEDIR/train_outputs +# run training +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavegrad.py --config_path $BASEDIR/inputs/test_vocoder_wavegrad.json +# find the training folder +LATEST_FOLDER=$(ls $BASEDIR/train_outputs/| sort | tail -1) +echo $LATEST_FOLDER +# continue the previous training +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavegrad.py --continue_path $BASEDIR/train_outputs/$LATEST_FOLDER +# remove all the outputs +rm -rf $BASEDIR/train_outputs/$LATEST_FOLDER \ No newline at end of file diff --git a/tests/test_vocoder_wavernn.py b/tests/test_vocoder_wavernn.py new file mode 100644 index 00000000..2464cfa3 --- /dev/null +++ b/tests/test_vocoder_wavernn.py @@ -0,0 +1,31 @@ +import numpy as np +import torch +import random +from TTS.vocoder.models.wavernn import WaveRNN + + +def test_wavernn(): + model = WaveRNN( + rnn_dims=512, + fc_dims=512, + mode=10, + mulaw=False, + pad=2, + use_aux_net=True, + use_upsample_net=True, + upsample_factors=[4, 8, 8], + feat_dims=80, + compute_dims=128, + res_out_dims=128, + num_res_blocks=10, + hop_length=256, + sample_rate=22050, + ) + dummy_x = torch.rand((2, 1280)) + dummy_m = torch.rand((2, 80, 9)) + y_size = random.randrange(20, 60) + dummy_y = torch.rand((80, y_size)) + output = model(dummy_x, dummy_m) + assert np.all(output.shape == (2, 1280, 4 * 256)), output.shape + output = model.inference(dummy_y, True, 5500, 550) + assert np.all(output.shape == (256 * (y_size - 1),)) diff --git a/tests/test_vocoder_wavernn_datasets.py b/tests/test_vocoder_wavernn_datasets.py new file mode 100644 index 00000000..a95e247a --- /dev/null +++ b/tests/test_vocoder_wavernn_datasets.py @@ -0,0 +1,92 @@ +import os +import shutil + +import numpy as np +from tests import get_tests_path, get_tests_input_path, get_tests_output_path +from torch.utils.data import DataLoader + +from TTS.utils.audio import AudioProcessor +from TTS.utils.io import load_config +from TTS.vocoder.datasets.wavernn_dataset import WaveRNNDataset +from TTS.vocoder.datasets.preprocess import load_wav_feat_data, preprocess_wav_files + +file_path = os.path.dirname(os.path.realpath(__file__)) +OUTPATH = os.path.join(get_tests_output_path(), "loader_tests/") +os.makedirs(OUTPATH, exist_ok=True) + +C = load_config(os.path.join(get_tests_input_path(), + "test_vocoder_wavernn_config.json")) + +test_data_path = os.path.join(get_tests_path(), "data/ljspeech/") +test_mel_feat_path = os.path.join(test_data_path, "mel") +test_quant_feat_path = os.path.join(test_data_path, "quant") +ok_ljspeech = os.path.exists(test_data_path) + + +def wavernn_dataset_case(batch_size, seq_len, hop_len, pad, mode, mulaw, num_workers): + """ run dataloader with given parameters and check conditions """ + ap = AudioProcessor(**C.audio) + + C.batch_size = batch_size + C.mode = mode + C.seq_len = seq_len + C.data_path = test_data_path + + preprocess_wav_files(test_data_path, C, ap) + _, train_items = load_wav_feat_data( + test_data_path, test_mel_feat_path, 5) + + dataset = WaveRNNDataset(ap=ap, + items=train_items, + seq_len=seq_len, + hop_len=hop_len, + pad=pad, + mode=mode, + mulaw=mulaw + ) + # sampler = DistributedSampler(dataset) if num_gpus > 1 else None + loader = DataLoader(dataset, + shuffle=True, + collate_fn=dataset.collate, + batch_size=batch_size, + num_workers=num_workers, + pin_memory=True, + ) + + max_iter = 10 + count_iter = 0 + + try: + for data in loader: + x_input, mels, _ = data + expected_feat_shape = (ap.num_mels, + (x_input.shape[-1] // hop_len) + (pad * 2)) + assert np.all( + mels.shape[1:] == expected_feat_shape), f" [!] {mels.shape} vs {expected_feat_shape}" + + assert (mels.shape[2] - pad * 2) * hop_len == x_input.shape[1] + count_iter += 1 + if count_iter == max_iter: + break + # except AssertionError: + # shutil.rmtree(test_mel_feat_path) + # shutil.rmtree(test_quant_feat_path) + finally: + shutil.rmtree(test_mel_feat_path) + shutil.rmtree(test_quant_feat_path) + + +def test_parametrized_wavernn_dataset(): + ''' test dataloader with different parameters ''' + params = [ + [16, C.audio['hop_length'] * 10, C.audio['hop_length'], 2, 10, True, 0], + [16, C.audio['hop_length'] * 10, C.audio['hop_length'], 2, "mold", False, 4], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 2, 9, False, 0], + [1, C.audio['hop_length'], C.audio['hop_length'], 2, 10, True, 0], + [1, C.audio['hop_length'], C.audio['hop_length'], 2, "mold", False, 0], + [1, C.audio['hop_length'] * 5, C.audio['hop_length'], 4, 10, False, 2], + [1, C.audio['hop_length'] * 5, C.audio['hop_length'], 2, "mold", False, 0], + ] + for param in params: + print(param) + wavernn_dataset_case(*param) diff --git a/tests/test_vocoder_wavernn_train.sh b/tests/test_vocoder_wavernn_train.sh new file mode 100755 index 00000000..40e86012 --- /dev/null +++ b/tests/test_vocoder_wavernn_train.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -xe +BASEDIR=$(dirname "$0") +echo "$BASEDIR" +# create run dir +mkdir -p $BASEDIR/train_outputs +# run training +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavernn.py --config_path $BASEDIR/inputs/test_vocoder_wavernn_config.json +# find the training folder +LATEST_FOLDER=$(ls $BASEDIR/train_outputs/| sort | tail -1) +echo $LATEST_FOLDER +# continue the previous training +CUDA_VISIBLE_DEVICES="" python TTS/bin/train_vocoder_wavernn.py --continue_path $BASEDIR/train_outputs/$LATEST_FOLDER +# remove all the outputs +rm -rf $BASEDIR/train_outputs/$LATEST_FOLDER \ No newline at end of file diff --git a/tests/test_wavegrad_layers.py b/tests/test_wavegrad_layers.py new file mode 100644 index 00000000..d81ae47d --- /dev/null +++ b/tests/test_wavegrad_layers.py @@ -0,0 +1,92 @@ +import torch + +from TTS.vocoder.layers.wavegrad import PositionalEncoding, FiLM, UBlock, DBlock +from TTS.vocoder.models.wavegrad import Wavegrad + + +def test_positional_encoding(): + layer = PositionalEncoding(50) + inp = torch.rand(32, 50, 100) + nl = torch.rand(32) + o = layer(inp, nl) + + assert o.shape[0] == 32 + assert o.shape[1] == 50 + assert o.shape[2] == 100 + assert isinstance(o, torch.FloatTensor) + + +def test_film(): + layer = FiLM(50, 76) + inp = torch.rand(32, 50, 100) + nl = torch.rand(32) + shift, scale = layer(inp, nl) + + assert shift.shape[0] == 32 + assert shift.shape[1] == 76 + assert shift.shape[2] == 100 + assert isinstance(shift, torch.FloatTensor) + + assert scale.shape[0] == 32 + assert scale.shape[1] == 76 + assert scale.shape[2] == 100 + assert isinstance(scale, torch.FloatTensor) + + layer.apply_weight_norm() + layer.remove_weight_norm() + + +def test_ublock(): + inp1 = torch.rand(32, 50, 100) + inp2 = torch.rand(32, 50, 50) + nl = torch.rand(32) + + layer_film = FiLM(50, 100) + layer = UBlock(50, 100, 2, [1, 2, 4, 8]) + + scale, shift = layer_film(inp1, nl) + o = layer(inp2, shift, scale) + + assert o.shape[0] == 32 + assert o.shape[1] == 100 + assert o.shape[2] == 100 + assert isinstance(o, torch.FloatTensor) + + layer.apply_weight_norm() + layer.remove_weight_norm() + + +def test_dblock(): + inp = torch.rand(32, 50, 130) + layer = DBlock(50, 100, 2) + o = layer(inp) + + assert o.shape[0] == 32 + assert o.shape[1] == 100 + assert o.shape[2] == 65 + assert isinstance(o, torch.FloatTensor) + + layer.apply_weight_norm() + layer.remove_weight_norm() + + +def test_wavegrad_forward(): + x = torch.rand(32, 1, 20 * 300) + c = torch.rand(32, 80, 20) + noise_scale = torch.rand(32) + + model = Wavegrad(in_channels=80, + out_channels=1, + upsample_factors=[5, 5, 3, 2, 2], + upsample_dilations=[[1, 2, 1, 2], [1, 2, 1, 2], + [1, 2, 4, 8], [1, 2, 4, 8], + [1, 2, 4, 8]]) + o = model.forward(x, c, noise_scale) + + assert o.shape[0] == 32 + assert o.shape[1] == 1 + assert o.shape[2] == 20 * 300 + assert isinstance(o, torch.FloatTensor) + + model.apply_weight_norm() + model.remove_weight_norm() diff --git a/tests/test_wavegrad_train.py b/tests/test_wavegrad_train.py new file mode 100644 index 00000000..700e94d1 --- /dev/null +++ b/tests/test_wavegrad_train.py @@ -0,0 +1,62 @@ +import unittest + +import numpy as np +import torch +from torch import optim +from TTS.vocoder.models.wavegrad import Wavegrad + +#pylint: disable=unused-variable + +torch.manual_seed(1) +use_cuda = torch.cuda.is_available() +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + +class WavegradTrainTest(unittest.TestCase): + def test_train_step(self): # pylint: disable=no-self-use + """Test if all layers are updated in a basic training cycle""" + input_dummy = torch.rand(8, 1, 20 * 300).to(device) + mel_spec = torch.rand(8, 80, 20).to(device) + + criterion = torch.nn.L1Loss().to(device) + model = Wavegrad(in_channels=80, + out_channels=1, + upsample_factors=[5, 5, 3, 2, 2], + upsample_dilations=[[1, 2, 1, 2], [1, 2, 1, 2], + [1, 2, 4, 8], [1, 2, 4, 8], + [1, 2, 4, 8]]) + + model_ref = Wavegrad(in_channels=80, + out_channels=1, + upsample_factors=[5, 5, 3, 2, 2], + upsample_dilations=[[1, 2, 1, 2], [1, 2, 1, 2], + [1, 2, 4, 8], [1, 2, 4, 8], + [1, 2, 4, 8]]) + model.train() + model.to(device) + betas = np.linspace(1e-6, 1e-2, 1000) + model.compute_noise_level(betas) + model_ref.load_state_dict(model.state_dict()) + model_ref.to(device) + count = 0 + for param, param_ref in zip(model.parameters(), + model_ref.parameters()): + assert (param - param_ref).sum() == 0, param + count += 1 + optimizer = optim.Adam(model.parameters(), lr=0.001) + for i in range(5): + y_hat = model.forward(input_dummy, mel_spec, torch.rand(8).to(device)) + optimizer.zero_grad() + loss = criterion(y_hat, input_dummy) + loss.backward() + optimizer.step() + # check parameter changes + count = 0 + for param, param_ref in zip(model.parameters(), + model_ref.parameters()): + # ignore pre-higway layer since it works conditional + # if count not in [145, 59]: + assert (param != param_ref).any( + ), "param {} with shape {} not updated!! \n{}\n{}".format( + count, param.shape, param, param_ref) + count += 1