wassname/Ranger21
1
0
Fork 0
mirror of https://github.com/wassname/Ranger21.git synced 2026-09-09 11:15:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23 from TheZothen/main

Browse Source 
Activate/deactivate softplus for MADGRAD & choosing beta softplus
This commit is contained in:
Less Wright
2021-07-19 17:06:58 -07:00
committed by GitHub
parent 60c0725d01 f5422d1af0
commit 57add4c944
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ranger21/ranger21.py
+9 -5
View File
@@ -116,6 +116,7 @@ class Ranger21(TO.Optimizer):
use_madgrad=False,
use_adabelief=False,
softplus=True,
beta_softplus=50,
using_gc=True,
using_normgc=True,
gc_conv_only=False,
@@ -172,6 +173,7 @@ class Ranger21(TO.Optimizer):
# softplus for denom
self.softplus = softplus
self.beta_softplus = beta_softplus
# norm loss
self.normloss_active = normloss_active
@@ -809,7 +811,9 @@ class Ranger21(TO.Optimizer):
s = state["s"]
if momentum == 0:
# Compute x_0 from other known quantities
rms = F.softplus(grad_sum_sq.pow(1 / 3), beta=50)
rms = grad_sum_sq.pow(1 / 3)
if self.softplus:
rms = F.softplus(rms, beta=self.beta_softplus)
x0 = p.data.addcdiv(s, rms, value=1)
else:
x0 = state["x0"]
@@ -821,7 +825,9 @@ class Ranger21(TO.Optimizer):
# print(f"gsumsq = {grad_sum_sq}")
grad_sum_sq.addcmul_(inner_grad, inner_grad, value=lamb)
rms = F.softplus(grad_sum_sq.pow(1 / 3), beta=50)
rms = grad_sum_sq.pow(1 / 3)
if self.softplus:
rms = F.softplus(rms, beta=self.beta_softplus)
# Update s
s.data.add_(inner_grad, alpha=lamb)
@@ -891,9 +897,7 @@ class Ranger21(TO.Optimizer):
# softplus the denom
if self.softplus:
smooth = 50
sp = torch.nn.Softplus(smooth)
denom = sp(denom)
denom = F.softplus(denom, beta=self.beta_softplus)
pnmomentum = (
grad_ma.mul(1 + self.momentum_pnm)