From d5db2f572faf3012cb3fce4c77448e79a5913967 Mon Sep 17 00:00:00 2001
From: Less Wright <lessw@etrillium.com>
Date: Sun, 4 Apr 2021 21:41:29 -0700
Subject: [PATCH] basic framework

---
 src/Ranger21.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/src/Ranger21.py b/src/Ranger21.py
index ad0502c..3e70166 100644
--- a/src/Ranger21.py
+++ b/src/Ranger21.py
@@ -19,3 +19,62 @@ import collections
 
 import copy
 from torch import linalg as LA
+
+class Ranger21(torch.optim.Optimizer):
+    def __init__(self,
+                params,
+                lr,
+                num_batches_per_epoch = None,
+                num_epochs = None,
+                num_warmup_iterations = 1000,
+                weight_decay=0,
+                decay_type = "stable",
+
+                warmup_type = 'linear',
+                use_GC=True):
+
+                # todo - checks on incoming params
+
+                self.num_batches = num_batches_per_epoch
+                self.num_epochs = num_epochs
+                self.num_warmup_iters = num_warmup_iterations
+                self.warmup_type=warmup_type
+                self.use_GC = use_GC
+                self.starting_lr = lr
+
+                #decay
+                self.decay = weight_decay
+                self.decay_type = decay_type
+
+    def warmup_dampening(self, step):
+        # not usable yet
+        style = self.warmup_type
+        step +=1
+        warmup = self.num_warmup_iters
+
+        if style is None:
+            return 1.0
+
+        if style=='linear':
+            return min(1.0, (step+1/warmup) )
+
+        elif style=='exponential':
+            return 1.0 - math.exp(-step/warmup)
+        else:
+            raise ValueError(f"warmup type {style} not implemented.")
+
+
+
+
+    def step(self,
+            closure = None,
+            passed_loss = None):
+
+            # let's build in a loss pass through for HyperExplorer
+            loss = None
+            if closure is not None and isinstance(closure, collections.Callable):
+                loss = closure()
+            
+
+
+