From 3c5aeb5e22e7938aef41702eab8a1ee683209f75 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 29 Aug 2019 11:49:53 +0200 Subject: [PATCH 1/2] Fix installation by using an explicit symlink --- __init__.py | 0 datasets/TTSDataset.py | 4 ++-- distribute.py | 2 +- layers/losses.py | 2 +- models/tacotron.py | 4 ++-- models/tacotron2.py | 4 ++-- models/tacotrongst.py | 6 +++--- notebooks/Benchmark.ipynb | 2 +- notebooks/ExtractTTSpectrogram.ipynb | 8 ++++---- server/server.py | 2 +- server/synthesizer.py | 10 +++++----- setup.py | 4 ++-- synthesize.py | 8 ++++---- tests/generic_utils_text.py | 4 ++-- tests/symbols_tests.py | 2 +- tests/test_audio.py | 6 +++--- tests/test_demo_server.py | 8 ++++---- tests/test_layers.py | 6 +++--- tests/test_loader.py | 8 ++++---- tests/test_preprocessors.py | 4 ++-- tests/test_tacotron2_model.py | 6 +++--- tests/test_tacotron_model.py | 6 +++--- tests/test_text_processing.py | 2 +- train.py | 30 ++++++++++++++-------------- tts_namespace/README.md | 29 +++++++++++++++++++++++++++ tts_namespace/TTS | 1 + utils/speakers.py | 2 +- utils/text/__init__.py | 4 ++-- utils/visual.py | 2 +- 29 files changed, 103 insertions(+), 73 deletions(-) create mode 100644 __init__.py create mode 100644 tts_namespace/README.md create mode 120000 tts_namespace/TTS diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/datasets/TTSDataset.py b/datasets/TTSDataset.py index dea6cb8c..cbb4bf97 100644 --- a/datasets/TTSDataset.py +++ b/datasets/TTSDataset.py @@ -5,8 +5,8 @@ import torch import random from torch.utils.data import Dataset -from utils.text import text_to_sequence, phoneme_to_sequence, pad_with_eos_bos -from utils.data import prepare_data, prepare_tensor, prepare_stop_target +from TTS.utils.text import text_to_sequence, phoneme_to_sequence, pad_with_eos_bos +from TTS.utils.data import prepare_data, prepare_tensor, prepare_stop_target class MyDataset(Dataset): diff --git a/distribute.py b/distribute.py index 22c27b1c..f65fbe71 100644 --- a/distribute.py +++ b/distribute.py @@ -9,7 +9,7 @@ import torch.distributed as dist from torch.utils.data.sampler import Sampler from torch.autograd import Variable from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors -from utils.generic_utils import load_config, create_experiment_folder +from TTS.utils.generic_utils import load_config, create_experiment_folder class DistributedSampler(Sampler): diff --git a/layers/losses.py b/layers/losses.py index 5a95c0fe..a6bf95d3 100644 --- a/layers/losses.py +++ b/layers/losses.py @@ -1,6 +1,6 @@ from torch import nn from torch.nn import functional -from utils.generic_utils import sequence_mask +from TTS.utils.generic_utils import sequence_mask class L1LossMasked(nn.Module): diff --git a/models/tacotron.py b/models/tacotron.py index bf312db4..69a6fa03 100644 --- a/models/tacotron.py +++ b/models/tacotron.py @@ -1,7 +1,7 @@ # coding: utf-8 from torch import nn -from layers.tacotron import Encoder, Decoder, PostCBHG -from utils.generic_utils import sequence_mask +from TTS.layers.tacotron import Encoder, Decoder, PostCBHG +from TTS.utils.generic_utils import sequence_mask class Tacotron(nn.Module): diff --git a/models/tacotron2.py b/models/tacotron2.py index 05b4c0fd..a91d6e2e 100644 --- a/models/tacotron2.py +++ b/models/tacotron2.py @@ -1,7 +1,7 @@ from math import sqrt from torch import nn -from layers.tacotron2 import Encoder, Decoder, Postnet -from utils.generic_utils import sequence_mask +from TTS.layers.tacotron2 import Encoder, Decoder, Postnet +from TTS.utils.generic_utils import sequence_mask # TODO: match function arguments with tacotron diff --git a/models/tacotrongst.py b/models/tacotrongst.py index c18a5e98..5ea389d9 100644 --- a/models/tacotrongst.py +++ b/models/tacotrongst.py @@ -1,8 +1,8 @@ # coding: utf-8 from torch import nn -from layers.tacotron import Encoder, Decoder, PostCBHG -from layers.gst_layers import GST -from utils.generic_utils import sequence_mask +from TTS.layers.tacotron import Encoder, Decoder, PostCBHG +from TTS.layers.gst_layers import GST +from TTS.utils.generic_utils import sequence_mask class TacotronGST(nn.Module): diff --git a/notebooks/Benchmark.ipynb b/notebooks/Benchmark.ipynb index 13e31ed9..81f81641 100644 --- a/notebooks/Benchmark.ipynb +++ b/notebooks/Benchmark.ipynb @@ -138,7 +138,7 @@ "outputs": [], "source": [ "# LOAD TTS MODEL\n", - "from utils.text.symbols import symbols, phonemes\n", + "from TTS.utils.text.symbols import symbols, phonemes\n", "\n", "# multi speaker \n", "if CONFIG.use_speaker_embedding:\n", diff --git a/notebooks/ExtractTTSpectrogram.ipynb b/notebooks/ExtractTTSpectrogram.ipynb index f044300d..a0d0be60 100644 --- a/notebooks/ExtractTTSpectrogram.ipynb +++ b/notebooks/ExtractTTSpectrogram.ipynb @@ -105,10 +105,10 @@ "metadata": {}, "outputs": [], "source": [ - "from utils.text.symbols import symbols, phonemes\n", - "from utils.generic_utils import sequence_mask\n", - "from layers.losses import L1LossMasked\n", - "from utils.text.symbols import symbols, phonemes\n", + "from TTS.utils.text.symbols import symbols, phonemes\n", + "from TTS.utils.generic_utils import sequence_mask\n", + "from TTS.layers.losses import L1LossMasked\n", + "from TTS.utils.text.symbols import symbols, phonemes\n", "\n", "# load the model\n", "num_chars = len(phonemes) if C.use_phonemes else len(symbols)\n", diff --git a/server/server.py b/server/server.py index 95fa1caf..0244d612 100644 --- a/server/server.py +++ b/server/server.py @@ -1,7 +1,7 @@ #!flask/bin/python import argparse from synthesizer import Synthesizer -from utils.generic_utils import load_config +from TTS.utils.generic_utils import load_config from flask import Flask, request, render_template, send_file parser = argparse.ArgumentParser() diff --git a/server/synthesizer.py b/server/synthesizer.py index 3848968f..00311914 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -5,11 +5,11 @@ import numpy as np import torch import sys -from utils.audio import AudioProcessor -from utils.generic_utils import load_config, setup_model -from utils.text import phonemes, symbols -from utils.speakers import load_speaker_mapping -from utils.synthesis import * +from TTS.utils.audio import AudioProcessor +from TTS.utils.generic_utils import load_config, setup_model +from TTS.utils.text import phonemes, symbols +from TTS.utils.speakers import load_speaker_mapping +from TTS.utils.synthesis import * import re alphabets = r"([A-Za-z])" diff --git a/setup.py b/setup.py index b824b106..aae8a015 100644 --- a/setup.py +++ b/setup.py @@ -62,8 +62,8 @@ setup( version=version, url='https://github.com/mozilla/TTS', description='Text to Speech with Deep Learning', - package_dir={'TTS': '.'}, - packages=['TTS'] + ['TTS.' + pkg for pkg in find_packages()], + package_dir={'': 'tts_namespace'}, + packages=find_packages('tts_namespace'), cmdclass={ 'build_py': build_py, 'develop': develop, diff --git a/synthesize.py b/synthesize.py index 33a31c69..23c67c73 100644 --- a/synthesize.py +++ b/synthesize.py @@ -4,10 +4,10 @@ import argparse import torch import string -from utils.synthesis import synthesis -from utils.generic_utils import load_config, setup_model -from utils.text.symbols import symbols, phonemes -from utils.audio import AudioProcessor +from TTS.utils.synthesis import synthesis +from TTS.utils.generic_utils import load_config, setup_model +from TTS.utils.text.symbols import symbols, phonemes +from TTS.utils.audio import AudioProcessor def tts(model, diff --git a/tests/generic_utils_text.py b/tests/generic_utils_text.py index 2ef39c09..228df2df 100644 --- a/tests/generic_utils_text.py +++ b/tests/generic_utils_text.py @@ -1,8 +1,8 @@ import unittest import torch as T -from utils.generic_utils import save_checkpoint, save_best_model -from layers.tacotron import Prenet +from TTS.utils.generic_utils import save_checkpoint, save_best_model +from TTS.layers.tacotron import Prenet OUT_PATH = '/tmp/test.pth.tar' diff --git a/tests/symbols_tests.py b/tests/symbols_tests.py index d35e887e..4c32c7d6 100644 --- a/tests/symbols_tests.py +++ b/tests/symbols_tests.py @@ -1,6 +1,6 @@ import unittest -from utils.text import phonemes +from TTS.utils.text import phonemes class SymbolsTest(unittest.TestCase): def test_uniqueness(self): #pylint: disable=no-self-use diff --git a/tests/test_audio.py b/tests/test_audio.py index b2c4a135..fc5deb48 100644 --- a/tests/test_audio.py +++ b/tests/test_audio.py @@ -1,9 +1,9 @@ import os import unittest -from tests import get_tests_path, get_tests_input_path, get_tests_output_path -from utils.audio import AudioProcessor -from utils.generic_utils import load_config +from TTS.tests import get_tests_path, get_tests_input_path, get_tests_output_path +from TTS.utils.audio import AudioProcessor +from TTS.utils.generic_utils import load_config TESTS_PATH = get_tests_path() OUT_PATH = os.path.join(get_tests_output_path(), "audio_tests") diff --git a/tests/test_demo_server.py b/tests/test_demo_server.py index 0d0a3ac6..80c774e8 100644 --- a/tests/test_demo_server.py +++ b/tests/test_demo_server.py @@ -3,10 +3,10 @@ import unittest import torch as T -from server.synthesizer import Synthesizer -from tests import get_tests_input_path, get_tests_output_path, get_tests_path -from utils.text.symbols import phonemes, symbols -from utils.generic_utils import load_config, save_checkpoint, setup_model +from TTS.server.synthesizer import Synthesizer +from TTS.tests import get_tests_input_path, get_tests_output_path +from TTS.utils.text.symbols import phonemes, symbols +from TTS.utils.generic_utils import load_config, save_checkpoint, setup_model class DemoServerTest(unittest.TestCase): diff --git a/tests/test_layers.py b/tests/test_layers.py index 7d9e0650..cf27e30c 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -1,9 +1,9 @@ import unittest import torch as T -from layers.tacotron import Prenet, CBHG, Decoder, Encoder -from layers.losses import L1LossMasked -from utils.generic_utils import sequence_mask +from TTS.layers.tacotron import Prenet, CBHG, Decoder, Encoder +from TTS.layers.losses import L1LossMasked +from TTS.utils.generic_utils import sequence_mask #pylint: disable=unused-variable diff --git a/tests/test_loader.py b/tests/test_loader.py index 9d151820..fe1cefef 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -5,10 +5,10 @@ import torch import numpy as np from torch.utils.data import DataLoader -from utils.generic_utils import load_config -from utils.audio import AudioProcessor -from datasets import TTSDataset -from datasets.preprocess import ljspeech +from TTS.utils.generic_utils import load_config +from TTS.utils.audio import AudioProcessor +from TTS.datasets import TTSDataset +from TTS.datasets.preprocess import ljspeech #pylint: disable=unused-variable diff --git a/tests/test_preprocessors.py b/tests/test_preprocessors.py index 6f4b6df1..993ee495 100644 --- a/tests/test_preprocessors.py +++ b/tests/test_preprocessors.py @@ -1,8 +1,8 @@ import unittest import os -from tests import get_tests_input_path +from TTS.tests import get_tests_input_path -from datasets.preprocess import common_voice +from TTS.datasets.preprocess import common_voice class TestPreprocessors(unittest.TestCase): diff --git a/tests/test_tacotron2_model.py b/tests/test_tacotron2_model.py index 9ec2d4dc..a26f1ddf 100644 --- a/tests/test_tacotron2_model.py +++ b/tests/test_tacotron2_model.py @@ -6,9 +6,9 @@ import numpy as np from torch import optim from torch import nn -from utils.generic_utils import load_config -from layers.losses import MSELossMasked -from models.tacotron2 import Tacotron2 +from TTS.utils.generic_utils import load_config +from TTS.layers.losses import MSELossMasked +from TTS.models.tacotron2 import Tacotron2 #pylint: disable=unused-variable diff --git a/tests/test_tacotron_model.py b/tests/test_tacotron_model.py index b44cb58f..acd7af41 100644 --- a/tests/test_tacotron_model.py +++ b/tests/test_tacotron_model.py @@ -5,9 +5,9 @@ import unittest from torch import optim from torch import nn -from utils.generic_utils import load_config -from layers.losses import L1LossMasked -from models.tacotron import Tacotron +from TTS.utils.generic_utils import load_config +from TTS.layers.losses import L1LossMasked +from TTS.models.tacotron import Tacotron #pylint: disable=unused-variable diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py index 62440e47..8f8e6fab 100644 --- a/tests/test_text_processing.py +++ b/tests/test_text_processing.py @@ -1,7 +1,7 @@ import unittest import torch as T -from utils.text import * +from TTS.utils.text import * def test_phoneme_to_sequence(): text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!" diff --git a/train.py b/train.py index 1f808e49..5b5a4b13 100644 --- a/train.py +++ b/train.py @@ -10,24 +10,24 @@ import torch.nn as nn from torch import optim from torch.utils.data import DataLoader -from datasets.TTSDataset import MyDataset +from TTS.datasets.TTSDataset import MyDataset from distribute import (DistributedSampler, apply_gradient_allreduce, init_distributed, reduce_tensor) -from layers.losses import L1LossMasked, MSELossMasked -from utils.audio import AudioProcessor -from utils.generic_utils import (NoamLR, check_update, count_parameters, - create_experiment_folder, get_git_branch, - load_config, remove_experiment_folder, - save_best_model, save_checkpoint, weight_decay, - set_init_dict, copy_config_file, setup_model, - split_dataset, gradual_training_scheduler) -from utils.logger import Logger -from utils.speakers import load_speaker_mapping, save_speaker_mapping, \ +from TTS.layers.losses import L1LossMasked, MSELossMasked +from TTS.utils.audio import AudioProcessor +from TTS.utils.generic_utils import (NoamLR, check_update, count_parameters, + create_experiment_folder, get_git_branch, + load_config, remove_experiment_folder, + save_best_model, save_checkpoint, weight_decay, + set_init_dict, copy_config_file, setup_model, + split_dataset, gradual_training_scheduler) +from TTS.utils.logger import Logger +from TTS.utils.speakers import load_speaker_mapping, save_speaker_mapping, \ get_speakers -from utils.synthesis import synthesis -from utils.text.symbols import phonemes, symbols -from utils.visual import plot_alignment, plot_spectrogram -from datasets.preprocess import get_preprocessor_by_name +from TTS.utils.synthesis import synthesis +from TTS.utils.text.symbols import phonemes, symbols +from TTS.utils.visual import plot_alignment, plot_spectrogram +from TTS.datasets.preprocess import get_preprocessor_by_name torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = False diff --git a/tts_namespace/README.md b/tts_namespace/README.md new file mode 100644 index 00000000..c5b2ddbf --- /dev/null +++ b/tts_namespace/README.md @@ -0,0 +1,29 @@ +This folder contains a symlink called TTS to the parent folder: + + lrwxr-xr-x TTS -> .. + +This is used to appease the distribute/setuptools gods. When the project was +initially set up, the repository folder itself was considered a namespace, and +development was done with `sys.path` hacks. This means if you tried to install +TTS, `setup.py` would see the packages `models`, `utils`, `layers`... instead of + `TTS.models`, `TTS.utils`... + +Installing TTS would then pollute the package namespace with generic names like +those above. In order to make things installable in both install and development +modes (`pip install /path/to/TTS` and `pip install -e /path/to/TTS`), we needed +to add an additional 'TTS' namespace to avoid this pollution. A virtual redirect +using `packages_dir` in `setup.py` is not enough because it breaks the editable +installation, which can only handle the simplest of `package_dir` redirects. + +Our solution is to use a symlink in order to add the extra `TTS` namespace. In +`setup.py`, we only look for packages inside `tts_namespace` (this folder), +which contains a symlink called TTS pointing to the repository root. The final +result is that `setuptools.find_packages` will find `TTS.models`, `TTS.utils`... + +With this hack, `pip install -e` will then add a symlink to the `tts_namespace` +in your `site-packages` folder, which works properly. It's important not to add +anything else in this folder because it will pollute the package namespace when +installing the project. + +This does not work if you check out your project on a filesystem that does not +support symlinks. \ No newline at end of file diff --git a/tts_namespace/TTS b/tts_namespace/TTS new file mode 120000 index 00000000..a96aa0ea --- /dev/null +++ b/tts_namespace/TTS @@ -0,0 +1 @@ +.. \ No newline at end of file diff --git a/utils/speakers.py b/utils/speakers.py index a1c273cf..4b11531b 100644 --- a/utils/speakers.py +++ b/utils/speakers.py @@ -1,7 +1,7 @@ import os import json -from datasets.preprocess import get_preprocessor_by_name +from TTS.datasets.preprocess import get_preprocessor_by_name def make_speakers_json_path(out_path): diff --git a/utils/text/__init__.py b/utils/text/__init__.py index 226e2e8d..1c5b98c3 100644 --- a/utils/text/__init__.py +++ b/utils/text/__init__.py @@ -3,8 +3,8 @@ import re import phonemizer from phonemizer.phonemize import phonemize -from utils.text import cleaners -from utils.text.symbols import symbols, phonemes, _phoneme_punctuations, _bos, \ +from TTS.utils.text import cleaners +from TTS.utils.text.symbols import symbols, phonemes, _phoneme_punctuations, _bos, \ _eos # Mappings from symbol to numeric ID and vice versa: diff --git a/utils/visual.py b/utils/visual.py index 982fa53a..1ee87cfb 100644 --- a/utils/visual.py +++ b/utils/visual.py @@ -2,7 +2,7 @@ import librosa import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt -from utils.text import phoneme_to_sequence, sequence_to_phoneme +from TTS.utils.text import phoneme_to_sequence, sequence_to_phoneme def plot_alignment(alignment, info=None): From 28644a717e8bd8fd9fce644b6874a216b6e2d20f Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 29 Aug 2019 12:11:31 +0200 Subject: [PATCH 2/2] Fix tests --- .travis/script | 2 ++ tests/inputs/server_config.json | 2 +- utils/generic_utils.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis/script b/.travis/script index 4aa275be..41a17a4c 100755 --- a/.travis/script +++ b/.travis/script @@ -11,5 +11,7 @@ fi if [[ "$TEST_SUITE" == "unittest" ]]; then # Run tests on all pushes + pushd tts_namespace python -m unittest + popd fi diff --git a/tests/inputs/server_config.json b/tests/inputs/server_config.json index d3220d7d..8ac266bd 100644 --- a/tests/inputs/server_config.json +++ b/tests/inputs/server_config.json @@ -1,5 +1,5 @@ { - "tts_path":"tests/outputs/", // tts model root folder + "tts_path":"TTS/tests/outputs/", // tts model root folder "tts_file":"checkpoint_10.pth.tar", // tts checkpoint file "tts_config":"dummy_model_config.json", // tts config.json file "tts_speakers": null, // json file listing speaker ids. null if no speaker embedding. diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 51155254..1c16834a 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -250,7 +250,7 @@ def set_init_dict(model_dict, checkpoint, c): def setup_model(num_chars, num_speakers, c): print(" > Using model: {}".format(c.model)) - MyModel = importlib.import_module('models.' + c.model.lower()) + MyModel = importlib.import_module('TTS.models.' + c.model.lower()) MyModel = getattr(MyModel, c.model) if c.model.lower() in ["tacotron", "tacotrongst"]: model = MyModel(