wassname/Ranger21
1
0
Fork 0
mirror of https://github.com/wassname/Ranger21.git synced 2026-09-10 11:40:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Softplus for madgrad denominator

Browse Source
This commit is contained in:
Zohten
2021-07-18 18:02:09 +02:00
parent cd022ff593
commit 72825ba6ca
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ranger21/ranger21.py
+2 -2
View File
@@ -808,7 +808,7 @@ class Ranger21(TO.Optimizer):
s = state["s"]
if momentum == 0:
# Compute x_0 from other known quantities
rms = grad_sum_sq.pow(1 / 3).add_(eps)
rms = F.softplus(grad_sum_sq.pow(1 / 3), beta=50)
x0 = p.data.addcdiv(s, rms, value=1)
else:
x0 = state["x0"]
@@ -820,7 +820,7 @@ class Ranger21(TO.Optimizer):
# print(f"gsumsq = {grad_sum_sq}")
grad_sum_sq.addcmul_(inner_grad, inner_grad, value=lamb)
rms = grad_sum_sq.pow(1 / 3).add_(eps)
rms = F.softplus(grad_sum_sq.pow(1 / 3), beta=50)
# Update s
s.data.add_(inner_grad, alpha=lamb)