From 07c961382fcc1882a80e4ef65e3c9a357928d49d Mon Sep 17 00:00:00 2001 From: Edresson Date: Wed, 5 Aug 2020 08:43:27 -0300 Subject: [PATCH] add support for CorentinJ Speaker encoder and add notebook for extract embeddings --- mozilla_voice_tts/speaker_encoder/config.json | 27 +- mozilla_voice_tts/speaker_encoder/model.py | 38 +- ...- ExtractSpeakerEmbeddings-by-sample.ipynb | 2 +- ...J-ExtractSpeakerEmbeddings-by-sample.ipynb | 25495 ++++++++++++++++ 4 files changed, 25541 insertions(+), 21 deletions(-) create mode 100644 notebooks/GE2E-CorentinJ-ExtractSpeakerEmbeddings-by-sample.ipynb diff --git a/mozilla_voice_tts/speaker_encoder/config.json b/mozilla_voice_tts/speaker_encoder/config.json index 37d976ce..11da0cf6 100644 --- a/mozilla_voice_tts/speaker_encoder/config.json +++ b/mozilla_voice_tts/speaker_encoder/config.json @@ -1,14 +1,14 @@ { - "run_name": "libritts_100+360-angleproto", - "run_description": "train speaker encoder for libritts 100 and 360", + "run_name": "Model compatible to CorentinJ/Real-Time-Voice-Cloning", + "run_description": "train speaker encoder with voxceleb1, voxceleb2 and libriSpeech ", "audio":{ // Audio processing parameters - "num_mels": 80, // size of the mel spec frame. - "num_freq": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. + "num_mels": 40, // size of the mel spec frame. + "fft_size": 400, // number of stft frequency levels. Size of the linear spectogram frame. + "sample_rate": 16000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "win_length": 400, // stft window length in ms. + "hop_length": 160, // stft window hop-lengh in ms. "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. @@ -27,7 +27,7 @@ "trim_db": 60 // threshold for timming silence. Set this according to your dataset. }, "reinit_layers": [], - "loss": "angleproto", // "ge2e" to use Generalized End-to-End loss and "angleproto" to use Angular Prototypical loss (new SOTA) + "loss": "ge2e", // "ge2e" to use Generalized End-to-End loss and "angleproto" to use Angular Prototypical loss (new SOTA) "grad_clip": 3.0, // upper limit for gradients for clipping. "epochs": 1000, // total number of epochs to train. "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. @@ -41,12 +41,13 @@ "checkpoint": true, // If true, it saves checkpoints per "save_step" "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. "print_step": 1, // Number of steps to log traning on console. - "output_path": "../../checkpoints/libri_tts/speaker_encoder/", // DATASET-RELATED: output path for all training outputs. + "output_path": "../../checkpoints/voxceleb_librispeech/speaker_encoder/", // DATASET-RELATED: output path for all training outputs. "model": { - "input_dim": 80, // input_dim == num_mels - "proj_dim": 128, - "lstm_dim": 384, - "num_lstm_layers": 3 + "input_dim": 40, + "proj_dim": 256, + "lstm_dim": 256, + "num_lstm_layers": 3, + "use_lstm_with_projection": false }, "datasets": [ diff --git a/mozilla_voice_tts/speaker_encoder/model.py b/mozilla_voice_tts/speaker_encoder/model.py index ca2abe31..df0527bc 100644 --- a/mozilla_voice_tts/speaker_encoder/model.py +++ b/mozilla_voice_tts/speaker_encoder/model.py @@ -16,15 +16,33 @@ class LSTMWithProjection(nn.Module): o, (_, _) = self.lstm(x) return self.linear(o) +class LSTMWithoutProjection(nn.Module): + def __init__(self, input_dim, lstm_dim, proj_dim, num_lstm_layers): + super().__init__() + self.lstm = nn.LSTM(input_size=input_dim, + hidden_size=lstm_dim, + num_layers=num_lstm_layers, + batch_first=True) + self.linear = nn.Linear(lstm_dim, proj_dim, bias=True) + self.relu = nn.ReLU() + def forward(self, x): + _, (hidden, _) = self.lstm(x) + return self.relu(self.linear(hidden[-1])) class SpeakerEncoder(nn.Module): - def __init__(self, input_dim, proj_dim=256, lstm_dim=768, num_lstm_layers=3): + def __init__(self, input_dim, proj_dim=256, lstm_dim=768, num_lstm_layers=3, use_lstm_with_projection=True): super().__init__() + self.use_lstm_with_projection = use_lstm_with_projection layers = [] - layers.append(LSTMWithProjection(input_dim, lstm_dim, proj_dim)) - for _ in range(num_lstm_layers - 1): - layers.append(LSTMWithProjection(proj_dim, lstm_dim, proj_dim)) - self.layers = nn.Sequential(*layers) + # choise LSTM layer + if use_lstm_with_projection: + layers.append(LSTMWithProjection(input_dim, lstm_dim, proj_dim)) + for _ in range(num_lstm_layers - 1): + layers.append(LSTMWithProjection(proj_dim, lstm_dim, proj_dim)) + self.layers = nn.Sequential(*layers) + else: + self.layers = LSTMWithoutProjection(input_dim, lstm_dim, proj_dim, num_lstm_layers) + self._init_layers() def _init_layers(self): @@ -37,12 +55,18 @@ class SpeakerEncoder(nn.Module): def forward(self, x): # TODO: implement state passing for lstms d = self.layers(x) - d = torch.nn.functional.normalize(d[:, -1], p=2, dim=1) + if self.use_lstm_with_projection: + d = torch.nn.functional.normalize(d[:, -1], p=2, dim=1) + else: + d = torch.nn.functional.normalize(d, p=2, dim=1) return d def inference(self, x): d = self.layers.forward(x) - d = torch.nn.functional.normalize(d[:, -1], p=2, dim=1) + if self.use_lstm_with_projection: + d = torch.nn.functional.normalize(d[:, -1], p=2, dim=1) + else: + d = torch.nn.functional.normalize(d, p=2, dim=1) return d def compute_embedding(self, x, num_frames=160, overlap=0.5): diff --git a/notebooks/AngleProto-Speaker_Encoder- ExtractSpeakerEmbeddings-by-sample.ipynb b/notebooks/AngleProto-Speaker_Encoder- ExtractSpeakerEmbeddings-by-sample.ipynb index d660a7f5..15206130 100644 --- a/notebooks/AngleProto-Speaker_Encoder- ExtractSpeakerEmbeddings-by-sample.ipynb +++ b/notebooks/AngleProto-Speaker_Encoder- ExtractSpeakerEmbeddings-by-sample.ipynb @@ -79,7 +79,7 @@ "#Preprocess dataset\n", "meta_data = []\n", "for i in range(len(DATASETS_NAME)):\n", - " preprocessor = importlib.import_module('TTS.datasets.preprocess')\n", + " preprocessor = importlib.import_module('TTS.tts.datasets.preprocess')\n", " preprocessor = getattr(preprocessor, DATASETS_NAME[i].lower())\n", " meta_data += preprocessor(DATASETS_PATH[i],DATASETS_METAFILE[i])\n", " \n", diff --git a/notebooks/GE2E-CorentinJ-ExtractSpeakerEmbeddings-by-sample.ipynb b/notebooks/GE2E-CorentinJ-ExtractSpeakerEmbeddings-by-sample.ipynb new file mode 100644 index 00000000..576a95fe --- /dev/null +++ b/notebooks/GE2E-CorentinJ-ExtractSpeakerEmbeddings-by-sample.ipynb @@ -0,0 +1,25495 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is a noteboook used to generate the speaker embeddings with the CorentinJ GE2E model trained with Angular Prototypical loss for multi-speaker training.\n", + "\n", + "Before running this script please DON'T FORGET:\n", + "- to set the right paths in the cell below.\n", + "\n", + "Repositories:\n", + "- TTS: https://github.com/mozilla/TTS\n", + "- CorentinJ GE2E: https://github.com/Edresson/GE2E-Speaker-Encoder" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import os\n", + "import importlib\n", + "import random\n", + "import librosa\n", + "import torch\n", + "\n", + "import numpy as np\n", + "from TTS.utils.io import load_config\n", + "from tqdm import tqdm\n", + "from TTS.tts.utils.speakers import save_speaker_mapping, load_speaker_mapping\n", + "\n", + "# you may need to change this depending on your system\n", + "os.environ['CUDA_VISIBLE_DEVICES']='0'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cloning into 'Real-Time-Voice-Cloning'...\n", + "remote: Enumerating objects: 5, done.\u001b[K\n", + "remote: Counting objects: 100% (5/5), done.\u001b[K\n", + "remote: Compressing objects: 100% (5/5), done.\u001b[K\n", + "remote: Total 2508 (delta 0), reused 3 (delta 0), pack-reused 2503\u001b[K\n", + "Receiving objects: 100% (2508/2508), 360.78 MiB | 17.84 MiB/s, done.\n", + "Resolving deltas: 100% (1387/1387), done.\n", + "Checking connectivity... done.\n" + ] + } + ], + "source": [ + "# Clone encoder \n", + "!git clone https://github.com/CorentinJ/Real-Time-Voice-Cloning.git\n", + "os.chdir('Real-Time-Voice-Cloning/')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "#Install voxceleb_trainer Requeriments\n", + "!python -m pip install umap-learn visdom webrtcvad librosa>=0.5.1 matplotlib>=2.0.2 numpy>=1.14.0 scipy>=1.0.0 tqdm sounddevice Unidecode inflect multiprocess numba" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2020-08-05 06:51:05-- https://github.com/Edresson/Real-Time-Voice-Cloning/releases/download/checkpoints/pretrained.zip\n", + "Resolving github.com (github.com)... 18.231.5.6\n", + "Connecting to github.com (github.com)|18.231.5.6|:443... connected.\n", + "HTTP request sent, awaiting response... 301 Moved Permanently\n", + "Location: https://github.com/Edresson/GE2E-Speaker-Encoder/releases/download/checkpoints/pretrained.zip [following]\n", + "--2020-08-05 06:51:05-- https://github.com/Edresson/GE2E-Speaker-Encoder/releases/download/checkpoints/pretrained.zip\n", + "Reusing existing connection to github.com:443.\n", + "HTTP request sent, awaiting response... 302 Found\n", + "Location: https://github-production-release-asset-2e65be.s3.amazonaws.com/263893598/f7f31d80-96df-11ea-8345-261fc35f9849?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20200805%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200805T101614Z&X-Amz-Expires=300&X-Amz-Signature=df7724c28668ebd5dfbcc6a9b51f6afb78193c30119f3a1c3eef678188aabd1e&X-Amz-SignedHeaders=host&actor_id=0&repo_id=263893598&response-content-disposition=attachment%3B%20filename%3Dpretrained.zip&response-content-type=application%2Foctet-stream [following]\n", + "--2020-08-05 06:51:05-- https://github-production-release-asset-2e65be.s3.amazonaws.com/263893598/f7f31d80-96df-11ea-8345-261fc35f9849?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20200805%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200805T101614Z&X-Amz-Expires=300&X-Amz-Signature=df7724c28668ebd5dfbcc6a9b51f6afb78193c30119f3a1c3eef678188aabd1e&X-Amz-SignedHeaders=host&actor_id=0&repo_id=263893598&response-content-disposition=attachment%3B%20filename%3Dpretrained.zip&response-content-type=application%2Foctet-stream\n", + "Resolving github-production-release-asset-2e65be.s3.amazonaws.com (github-production-release-asset-2e65be.s3.amazonaws.com)... 52.216.18.24\n", + "Connecting to github-production-release-asset-2e65be.s3.amazonaws.com (github-production-release-asset-2e65be.s3.amazonaws.com)|52.216.18.24|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 383640573 (366M) [application/octet-stream]\n", + "Saving to: ‘pretrained.zip’\n", + "\n", + "pretrained.zip 100%[===================>] 365,87M 6,62MB/s in 56s \n", + "\n", + "2020-08-05 06:52:03 (6,48 MB/s) - ‘pretrained.zip’ saved [383640573/383640573]\n", + "\n", + "Archive: pretrained.zip\n", + " creating: encoder/saved_models/\n", + " inflating: encoder/saved_models/pretrained.pt \n", + " creating: synthesizer/saved_models/\n", + " creating: synthesizer/saved_models/logs-pretrained/\n", + " creating: synthesizer/saved_models/logs-pretrained/taco_pretrained/\n", + " extracting: synthesizer/saved_models/logs-pretrained/taco_pretrained/checkpoint \n", + " inflating: synthesizer/saved_models/logs-pretrained/taco_pretrained/tacotron_model.ckpt-278000.data-00000-of-00001 \n", + " inflating: synthesizer/saved_models/logs-pretrained/taco_pretrained/tacotron_model.ckpt-278000.index \n", + " inflating: synthesizer/saved_models/logs-pretrained/taco_pretrained/tacotron_model.ckpt-278000.meta \n", + " creating: vocoder/saved_models/\n", + " creating: vocoder/saved_models/pretrained/\n", + " inflating: vocoder/saved_models/pretrained/pretrained.pt \n" + ] + } + ], + "source": [ + "#Download encoder Checkpoint\n", + "!wget https://github.com/Edresson/Real-Time-Voice-Cloning/releases/download/checkpoints/pretrained.zip\n", + "!unzip pretrained.zip" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from encoder import inference as encoder\n", + "from encoder.params_model import model_embedding_size as speaker_embedding_size\n", + "from pathlib import Path" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Preparing the encoder, the synthesizer and the vocoder...\n", + "Loaded encoder \"pretrained.pt\" trained to step 1564501\n", + "Testing your configuration with small inputs.\n", + "\tTesting the encoder...\n", + "(256,)\n" + ] + } + ], + "source": [ + "print(\"Preparing the encoder, the synthesizer and the vocoder...\")\n", + "encoder.load_model(Path('encoder/saved_models/pretrained.pt'))\n", + "print(\"Testing your configuration with small inputs.\")\n", + "# Forward an audio waveform of zeroes that lasts 1 second. Notice how we can get the encoder's\n", + "# sampling rate, which may differ.\n", + "# If you're unfamiliar with digital audio, know that it is encoded as an array of floats \n", + "# (or sometimes integers, but mostly floats in this projects) ranging from -1 to 1.\n", + "# The sampling rate is the number of values (samples) recorded per second, it is set to\n", + "# 16000 for the encoder. Creating an array of length will always correspond \n", + "# to an audio of 1 second.\n", + "print(\"\\tTesting the encoder...\")\n", + "\n", + "wav = np.zeros(encoder.sampling_rate) \n", + "embed = encoder.embed_utterance(wav)\n", + "print(embed.shape)\n", + "\n", + "# Embeddings are L2-normalized (this isn't important here, but if you want to make your own \n", + "# embeddings it will be).\n", + "#embed /= np.linalg.norm(embed) # for random embedding\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "SAVE_PATH = '../'" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# Set constants\n", + "DATASETS_NAME = ['vctk'] # list the datasets\n", + "DATASETS_PATH = ['../../../../../datasets/VCTK-Corpus-removed-silence/']\n", + "DATASETS_METAFILE = ['']\n", + "USE_CUDA = True" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + " 0%| | 0/44063 [00:00