From d670dbaab55cdbbca40579a77371092d4b3ac9d2 Mon Sep 17 00:00:00 2001 From: Zohten Date: Sun, 18 Jul 2021 17:58:06 +0200 Subject: [PATCH 1/3] replace len(list(x.size())) by x.dim() --- ranger21/ranger21.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ranger21/ranger21.py b/ranger21/ranger21.py index a30b38c..bdf7b82 100644 --- a/ranger21/ranger21.py +++ b/ranger21/ranger21.py @@ -74,7 +74,7 @@ def get_chebs(num_epochs): def normalize_gradient(x, use_channels=False, epsilon=1e-8): """ use stdev to normalize gradients """ - size = len(list(x.size())) + size = x.dim() # print(f"size = {size}") if (size > 1) and use_channels: @@ -91,7 +91,7 @@ def normalize_gradient(x, use_channels=False, epsilon=1e-8): def centralize_gradient(x, gc_conv_only=False): """credit - https://github.com/Yonghongwei/Gradient-Centralization """ - size = len(list(x.size())) + size = x.dim() # print(f"size = {size}") if gc_conv_only: From cd022ff593738c8d9c0b411e294008690b22e1ba Mon Sep 17 00:00:00 2001 From: Zohten Date: Sun, 18 Jul 2021 18:00:21 +0200 Subject: [PATCH 2/3] fix stable weight decay for madgrad --- ranger21/ranger21.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ranger21/ranger21.py b/ranger21/ranger21.py index bdf7b82..fed39da 100644 --- a/ranger21/ranger21.py +++ b/ranger21/ranger21.py @@ -702,7 +702,10 @@ class Ranger21(TO.Optimizer): raise ValueError("failed to set param size") # stable weight decay - variance_normalized = math.sqrt(variance_ma_sum / param_size) + if self.use_madgrad: + variance_normalized = torch.pow(variance_ma_sum / param_size, 1/3) + else: + variance_normalized = math.sqrt(variance_ma_sum / param_size) # variance_mean = variance_ma_sum / param_size if math.isnan(variance_normalized): raise RuntimeError("hit nan for variance_normalized") From 72825ba6ca2c9c57a9b7b349ae4120357cc5bbde Mon Sep 17 00:00:00 2001 From: Zohten Date: Sun, 18 Jul 2021 18:02:09 +0200 Subject: [PATCH 3/3] Softplus for madgrad denominator --- ranger21/ranger21.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ranger21/ranger21.py b/ranger21/ranger21.py index fed39da..bdcb8df 100644 --- a/ranger21/ranger21.py +++ b/ranger21/ranger21.py @@ -808,7 +808,7 @@ class Ranger21(TO.Optimizer): s = state["s"] if momentum == 0: # Compute x_0 from other known quantities - rms = grad_sum_sq.pow(1 / 3).add_(eps) + rms = F.softplus(grad_sum_sq.pow(1 / 3), beta=50) x0 = p.data.addcdiv(s, rms, value=1) else: x0 = state["x0"] @@ -820,7 +820,7 @@ class Ranger21(TO.Optimizer): # print(f"gsumsq = {grad_sum_sq}") grad_sum_sq.addcmul_(inner_grad, inner_grad, value=lamb) - rms = grad_sum_sq.pow(1 / 3).add_(eps) + rms = F.softplus(grad_sum_sq.pow(1 / 3), beta=50) # Update s s.data.add_(inner_grad, alpha=lamb)