diff --git a/src/Ranger21.py b/src/Ranger21.py index 6bd2551..ad0502c 100644 --- a/src/Ranger21.py +++ b/src/Ranger21.py @@ -1 +1,21 @@ # Ranger21 - @lessw2020 + +# core components based on: + +# MADGRAD: https://arxiv.org/abs/2101.11075 + +# warmup: https://arxiv.org/abs/1910.04209v3 + +# stable weight decay: https://arxiv.org/abs/2011.11152v3 + +# Gradient Centralization: https://arxiv.org/abs/2004.01461v2 + + +import torch +import torch.optim + +import math +import collections + +import copy +from torch import linalg as LA