From 529348d6dc6fa6ffb49585abf01fbae3b8640d58 Mon Sep 17 00:00:00 2001 From: Eren Golge Date: Fri, 30 Aug 2019 10:29:22 +0200 Subject: [PATCH] lint fixes --- setup.py | 10 ++--- utils/radam.py | 108 +++++++++++++------------------------------------ 2 files changed, 32 insertions(+), 86 deletions(-) diff --git a/setup.py b/setup.py index 51ee87ae..a8e52739 100644 --- a/setup.py +++ b/setup.py @@ -66,11 +66,11 @@ setup( package_dir={'': 'tts_namespace'}, packages=find_packages('tts_namespace'), project_urls={ - 'Documentation': 'https://github.com/mozilla/TTS/wiki', - 'Tracker': 'https://github.com/mozilla/TTS/issues', - 'Repository': 'https://github.com/mozilla/TTS', - 'Discussions': 'https://discourse.mozilla.org/c/tts', - }, + 'Documentation': 'https://github.com/mozilla/TTS/wiki', + 'Tracker': 'https://github.com/mozilla/TTS/issues', + 'Repository': 'https://github.com/mozilla/TTS', + 'Discussions': 'https://discourse.mozilla.org/c/tts', + }, cmdclass={ 'build_py': build_py, 'develop': develop, diff --git a/utils/radam.py b/utils/radam.py index d3a65dc5..57323541 100644 --- a/utils/radam.py +++ b/utils/radam.py @@ -1,6 +1,7 @@ import math import torch -from torch.optim.optimizer import Optimizer, required +from torch.optim.optimizer import Optimizer + class RAdam(Optimizer): @@ -9,7 +10,7 @@ class RAdam(Optimizer): self.buffer = [[None, None, None] for ind in range(10)] super(RAdam, self).__init__(params, defaults) - def __setstate__(self, state): + def __setstate__(self, state): # pylint: disable= useless-super-delegation super(RAdam, self).__setstate__(state) def step(self, closure=None): @@ -25,19 +26,21 @@ class RAdam(Optimizer): continue grad = p.grad.data.float() if grad.is_sparse: - raise RuntimeError('RAdam does not support sparse gradients') + raise RuntimeError( + 'RAdam does not support sparse gradients') p_data_fp32 = p.data.float() state = self.state[p] - if len(state) == 0: + if not state: state['step'] = 0 state['exp_avg'] = torch.zeros_like(p_data_fp32) state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) else: state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) - state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) + state['exp_avg_sq'] = state['exp_avg_sq'].type_as( + p_data_fp32) exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] beta1, beta2 = group['betas'] @@ -53,21 +56,24 @@ class RAdam(Optimizer): buffered[0] = state['step'] beta2_t = beta2 ** state['step'] N_sma_max = 2 / (1 - beta2) - 1 - N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) + N_sma = N_sma_max - 2 * \ + state['step'] * beta2_t / (1 - beta2_t) buffered[1] = N_sma # more conservative since it's an approximated value if N_sma >= 5: - step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) + step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * ( + N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) else: step_size = group['lr'] / (1 - beta1 ** state['step']) buffered[2] = step_size if group['weight_decay'] != 0: - p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) + p_data_fp32.add_(-group['weight_decay'] + * group['lr'], p_data_fp32) # more conservative since it's an approximated value - if N_sma >= 5: + if N_sma >= 5: denom = exp_avg_sq.sqrt().add_(group['eps']) p_data_fp32.addcdiv_(-step_size, exp_avg, denom) else: @@ -77,6 +83,7 @@ class RAdam(Optimizer): return loss + class PlainRAdam(Optimizer): def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): @@ -84,7 +91,7 @@ class PlainRAdam(Optimizer): super(PlainRAdam, self).__init__(params, defaults) - def __setstate__(self, state): + def __setstate__(self, state): # pylint: disable= useless-super-delegation super(PlainRAdam, self).__setstate__(state) def step(self, closure=None): @@ -100,19 +107,21 @@ class PlainRAdam(Optimizer): continue grad = p.grad.data.float() if grad.is_sparse: - raise RuntimeError('RAdam does not support sparse gradients') + raise RuntimeError( + 'RAdam does not support sparse gradients') p_data_fp32 = p.data.float() state = self.state[p] - if len(state) == 0: + if not state: state['step'] = 0 state['exp_avg'] = torch.zeros_like(p_data_fp32) state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) else: state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) - state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) + state['exp_avg_sq'] = state['exp_avg_sq'].type_as( + p_data_fp32) exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] beta1, beta2 = group['betas'] @@ -126,11 +135,13 @@ class PlainRAdam(Optimizer): N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) if group['weight_decay'] != 0: - p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) + p_data_fp32.add_(-group['weight_decay'] + * group['lr'], p_data_fp32) # more conservative since it's an approximated value - if N_sma >= 5: - step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) + if N_sma >= 5: + step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * ( + N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) denom = exp_avg_sq.sqrt().add_(group['eps']) p_data_fp32.addcdiv_(-step_size, exp_avg, denom) else: @@ -140,68 +151,3 @@ class PlainRAdam(Optimizer): p.data.copy_(p_data_fp32) return loss - - -class AdamW(Optimizer): - - def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, warmup = 0): - defaults = dict(lr=lr, betas=betas, eps=eps, - weight_decay=weight_decay, warmup = warmup) - super(AdamW, self).__init__(params, defaults) - - def __setstate__(self, state): - super(AdamW, self).__setstate__(state) - - def step(self, closure=None): - loss = None - if closure is not None: - loss = closure() - - for group in self.param_groups: - - for p in group['params']: - if p.grad is None: - continue - grad = p.grad.data.float() - if grad.is_sparse: - raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead') - - p_data_fp32 = p.data.float() - - state = self.state[p] - - if len(state) == 0: - state['step'] = 0 - state['exp_avg'] = torch.zeros_like(p_data_fp32) - state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) - else: - state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) - state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) - - exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] - beta1, beta2 = group['betas'] - - state['step'] += 1 - - exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) - exp_avg.mul_(beta1).add_(1 - beta1, grad) - - denom = exp_avg_sq.sqrt().add_(group['eps']) - bias_correction1 = 1 - beta1 ** state['step'] - bias_correction2 = 1 - beta2 ** state['step'] - - if group['warmup'] > state['step']: - scheduled_lr = 1e-8 + state['step'] * group['lr'] / group['warmup'] - else: - scheduled_lr = group['lr'] - - step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 - - if group['weight_decay'] != 0: - p_data_fp32.add_(-group['weight_decay'] * scheduled_lr, p_data_fp32) - - p_data_fp32.addcdiv_(-step_size, exp_avg, denom) - - p.data.copy_(p_data_fp32) - - return loss