From bf0d1a7b3c2f82ae1e4a601c78c84c735fc60a3a Mon Sep 17 00:00:00 2001 From: Eren Date: Tue, 5 Jun 2018 14:15:48 +0200 Subject: [PATCH 1/5] Server component added --- server/README.md | 6 +++ server/conf.json | 7 +++ server/server.py | 34 ++++++++++++ server/synthesizer.py | 63 ++++++++++++++++++++++ server/templates/index.html | 104 ++++++++++++++++++++++++++++++++++++ 5 files changed, 214 insertions(+) create mode 100644 server/README.md create mode 100644 server/conf.json create mode 100644 server/server.py create mode 100644 server/synthesizer.py create mode 100644 server/templates/index.html diff --git a/server/README.md b/server/README.md new file mode 100644 index 00000000..4326b183 --- /dev/null +++ b/server/README.md @@ -0,0 +1,6 @@ +## TTS example web-server +Steps to run: +1. Download one of the models given on the main page. +2. Set paths and other options in server/conf.json. +3. Run the server ```python server/server.py -c conf.json``` +4. Go to ```localhost:[given_port]``` and enjoy. \ No newline at end of file diff --git a/server/conf.json b/server/conf.json new file mode 100644 index 00000000..031ac6ec --- /dev/null +++ b/server/conf.json @@ -0,0 +1,7 @@ +{ + "model_path":"/home/egolge/projects/models/May-22-2018_03_24PM-e6112f7", + "model_name":"checkpoint_272976.pth.tar", + "model_config":"config.json", + "port": 5000, + "use_cuda": true +} \ No newline at end of file diff --git a/server/server.py b/server/server.py new file mode 100644 index 00000000..459161ff --- /dev/null +++ b/server/server.py @@ -0,0 +1,34 @@ +#!flask/bin/python +import argparse +from synthesizer import Synthesizer +from TTS.utils.generic_utils import load_config +from flask import (Flask, Response, request, + render_template, send_file) + +parser = argparse.ArgumentParser() +parser.add_argument('-c', '--config_path', type=str, + help='path to config file for training') +args = parser.parse_args() + +config = load_config(args.config_path) +app = Flask(__name__) +synthesizer = Synthesizer() +synthesizer.load_model(config.model_path, config.model_name, + config.model_config, config.use_cuda) + +@app.route('/') +def index(): + return render_template('index.html') + +@app.route('/api/tts', methods=['GET']) +def tts(): + text = request.args.get('text') + print(" > Model input: {}".format(text)) + data = synthesizer.tts(text) + return send_file(data, + attachment_filename="testing.wav", + as_attachment=True, + mimetype='audio/wav') + +if __name__ == '__main__': + app.run(debug=True, host='0.0.0.0', port=config.port) \ No newline at end of file diff --git a/server/synthesizer.py b/server/synthesizer.py new file mode 100644 index 00000000..f477c23a --- /dev/null +++ b/server/synthesizer.py @@ -0,0 +1,63 @@ +import io +import os +import librosa +import torch +import numpy as np +from TTS.utils.text import text_to_sequence +from TTS.utils.generic_utils import load_config +from TTS.utils.audio import AudioProcessor +from TTS.models.tacotron import Tacotron +from matplotlib import pylab as plt + + +class Synthesizer(object): + + def load_model(self, model_path, model_name, model_config, use_cuda): + model_config = os.path.join(model_path, model_config) + self.model_file = os.path.join(model_path, model_name) + print(" > Loading model ...") + print(" | > model config: ", model_config) + print(" | > model file: ", self.model_file) + config = load_config(model_config) + self.config = config + self.use_cuda = use_cuda + self.model = Tacotron(config.embedding_size, config.num_freq, config.num_mels, config.r) + self.ap = AudioProcessor(config.sample_rate, config.num_mels, config.min_level_db, + config.frame_shift_ms, config.frame_length_ms, config.preemphasis, + config.ref_level_db, config.num_freq, config.power, griffin_lim_iters=30) + # load model state + if use_cuda: + cp = torch.load(self.model_file) + else: + cp = torch.load(self.model_file, map_location=lambda storage, loc: storage) + # load the model + self.model.load_state_dict(cp['model']) + if use_cuda: + self.model.cuda() + self.model.eval() + + def save_wav(self, wav, path): + wav *= 32767 / max(0.01, np.max(np.abs(wav))) + librosa.output.write_wav(path, wav.astype(np.float), self.config.sample_rate, norm=True) + + def tts(self, text): + text_cleaner = [self.config.text_cleaner] + wavs = [] + for sen in text.split('.'): + if len(sen) < 3: + continue + sen +='.' + sen = sen.strip() + seq = np.array(text_to_sequence(text, text_cleaner)) + chars_var = torch.from_numpy(seq).unsqueeze(0) + if self.use_cuda: + chars_var = chars_var.cuda() + mel_out, linear_out, alignments, stop_tokens = self.model.forward(chars_var) + linear_out = linear_out[0].data.cpu().numpy() + wav = self.ap.inv_spectrogram(linear_out.T) + wav = wav[:self.ap.find_endpoint(wav)] + out = io.BytesIO() + wavs.append(wav) + wavs.append(np.zeros(10000)) + self.save_wav(wav, out) + return out \ No newline at end of file diff --git a/server/templates/index.html b/server/templates/index.html new file mode 100644 index 00000000..40a53ff8 --- /dev/null +++ b/server/templates/index.html @@ -0,0 +1,104 @@ + + + + + + + + + + + Bare - Start Bootstrap Template + + + + + + + + + + + + + + +
+
+
+

Mozilla TTS server example.

+

It is "work-in-progress" with an "far-to-be-alpha" release.

+
    +
+ +

+ +

+
+
+
+ + + + + + + + + From 25969c60a4c0e218afa2586db1a6cf427fdb475c Mon Sep 17 00:00:00 2001 From: Eren Date: Tue, 5 Jun 2018 16:15:08 +0200 Subject: [PATCH 2/5] Update requirements --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 084c8bf8..f2997902 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,6 @@ tensorboard tensorboardX torch matplotlib -Pillow \ No newline at end of file +Pillow +flask +scipy \ No newline at end of file From 249e2b9a278d214f8ccd940ca9832ac89bc2a991 Mon Sep 17 00:00:00 2001 From: Eren Date: Tue, 5 Jun 2018 16:15:23 +0200 Subject: [PATCH 3/5] Update server readme --- server/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/README.md b/server/README.md index 4326b183..974b695b 100644 --- a/server/README.md +++ b/server/README.md @@ -1,6 +1,9 @@ ## TTS example web-server Steps to run: 1. Download one of the models given on the main page. -2. Set paths and other options in server/conf.json. -3. Run the server ```python server/server.py -c conf.json``` -4. Go to ```localhost:[given_port]``` and enjoy. \ No newline at end of file +2. Checkout the corresponding commit history. +2. Set paths and other options in the file ```server/conf.json```. +3. Run the server ```python server/server.py -c conf.json```. (Requires Flask) +4. Go to ```localhost:[given_port]``` and enjoy. + +Note that the audio quality on browser is slightly worse due to the encoder quantization. \ No newline at end of file From 6e4145ee4b2a6b3d89c010d1e5ab436f15f48e3c Mon Sep 17 00:00:00 2001 From: Eren Date: Tue, 5 Jun 2018 16:15:57 +0200 Subject: [PATCH 4/5] Remove the noise in the code --- server/server.py | 4 +--- server/synthesizer.py | 16 ++++++++++++---- server/templates/index.html | 18 +++++++++--------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/server/server.py b/server/server.py index 459161ff..01267447 100644 --- a/server/server.py +++ b/server/server.py @@ -25,9 +25,7 @@ def tts(): text = request.args.get('text') print(" > Model input: {}".format(text)) data = synthesizer.tts(text) - return send_file(data, - attachment_filename="testing.wav", - as_attachment=True, + return send_file(data, mimetype='audio/wav') if __name__ == '__main__': diff --git a/server/synthesizer.py b/server/synthesizer.py index f477c23a..8444a702 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -2,7 +2,9 @@ import io import os import librosa import torch +import scipy import numpy as np +import soundfile as sf from TTS.utils.text import text_to_sequence from TTS.utils.generic_utils import load_config from TTS.utils.audio import AudioProcessor @@ -24,7 +26,7 @@ class Synthesizer(object): self.model = Tacotron(config.embedding_size, config.num_freq, config.num_mels, config.r) self.ap = AudioProcessor(config.sample_rate, config.num_mels, config.min_level_db, config.frame_shift_ms, config.frame_length_ms, config.preemphasis, - config.ref_level_db, config.num_freq, config.power, griffin_lim_iters=30) + config.ref_level_db, config.num_freq, config.power, griffin_lim_iters=60) # load model state if use_cuda: cp = torch.load(self.model_file) @@ -37,8 +39,13 @@ class Synthesizer(object): self.model.eval() def save_wav(self, wav, path): - wav *= 32767 / max(0.01, np.max(np.abs(wav))) - librosa.output.write_wav(path, wav.astype(np.float), self.config.sample_rate, norm=True) + wav *= 32767 / max(1e-8, np.max(np.abs(wav))) + # sf.write(path, wav.astype(np.int32), self.config.sample_rate, format='wav') + # wav = librosa.util.normalize(wav.astype(np.float), norm=np.inf, axis=None) + # wav = wav / wav.max() + # sf.write(path, wav.astype('float'), self.config.sample_rate, format='ogg') + scipy.io.wavfile.write(path, self.config.sample_rate, wav.astype(np.int16)) + # librosa.output.write_wav(path, wav.astype(np.int16), self.config.sample_rate, norm=True) def tts(self, text): text_cleaner = [self.config.text_cleaner] @@ -47,6 +54,7 @@ class Synthesizer(object): if len(sen) < 3: continue sen +='.' + print(sen) sen = sen.strip() seq = np.array(text_to_sequence(text, text_cleaner)) chars_var = torch.from_numpy(seq).unsqueeze(0) @@ -55,7 +63,7 @@ class Synthesizer(object): mel_out, linear_out, alignments, stop_tokens = self.model.forward(chars_var) linear_out = linear_out[0].data.cpu().numpy() wav = self.ap.inv_spectrogram(linear_out.T) - wav = wav[:self.ap.find_endpoint(wav)] + # wav = wav[:self.ap.find_endpoint(wav)] out = io.BytesIO() wavs.append(wav) wavs.append(np.zeros(10000)) diff --git a/server/templates/index.html b/server/templates/index.html index 40a53ff8..f5c2bdf3 100644 --- a/server/templates/index.html +++ b/server/templates/index.html @@ -72,15 +72,15 @@ function q(selector) {return document.querySelector(selector)} q('#text').focus() q('#speak-button').addEventListener('click', function(e) { - text = q('#text').value.trim() - if (text) { - q('#message').textContent = 'Synthesizing...' - q('#speak-button').disabled = true - q('#audio').hidden = true - synthesize(text) - } - e.preventDefault() - return false + text = q('#text').value + if (text) { + q('#message').textContent = 'Synthesizing...' + q('#speak-button').disabled = true + q('#audio').hidden = true + synthesize(text) + } + e.preventDefault() + return false }) function synthesize(text) { fetch('/api/tts?text=' + encodeURIComponent(text), {cache: 'no-cache'}) From 12d5f051d423dca25c7f337fc6a05c22cee59298 Mon Sep 17 00:00:00 2001 From: Eren Date: Tue, 5 Jun 2018 16:18:25 +0200 Subject: [PATCH 5/5] Small edit --- server/templates/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/templates/index.html b/server/templates/index.html index f5c2bdf3..b120d83a 100644 --- a/server/templates/index.html +++ b/server/templates/index.html @@ -8,7 +8,7 @@ - Bare - Start Bootstrap Template + Mozillia - Text2Speech engine - +