From 5bb150c0da36ab2842cb8701f11540a50d24b3a9 Mon Sep 17 00:00:00 2001 From: Less Wright Date: Sun, 25 Apr 2021 13:33:28 -0700 Subject: [PATCH] linear warmdown --- src/Ranger21.py | 49 ++++++++++++++++++------ src/__pycache__/Ranger21.cpython-37.pyc | Bin 12744 -> 13001 bytes 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/Ranger21.py b/src/Ranger21.py index b68ea21..951da3e 100644 --- a/src/Ranger21.py +++ b/src/Ranger21.py @@ -109,7 +109,7 @@ class Ranger21(TO.Optimizer): num_warmup_iterations=None, use_warmdown=True, warmdown_start_pct=0.65, - min_lr=1e-6, + warmdown_min_lr=3e-5, weight_decay=1e-4, decay_type="stable", warmup_type="linear", @@ -174,8 +174,15 @@ class Ranger21(TO.Optimizer): "missing total iterations, which is calced from num epochs and num iters per epoch param" ) + # lr + self.starting_lr = lr + self.current_lr = lr + self.tracking_lr = [] + + # warm down - self.min_lr = min_lr + self.min_lr = warmdown_min_lr + self.warmdown_lr_delta = self.starting_lr - self.min_lr self.use_warm_down = use_warmdown if self.use_warm_down: @@ -183,7 +190,8 @@ class Ranger21(TO.Optimizer): self.start_warm_down = int( self.warm_down_start_pct * num_epochs * num_batches_per_epoch ) - self.warmdown_displayed = False # print when warmdown begins... + self.warmdown_total_iterations = self.total_iterations - self.start_warm_down + self.warmdown_displayed = False # print when warmdown begins... @@ -268,7 +276,7 @@ class Ranger21(TO.Optimizer): if self.use_warm_down: print( - f"\nWarm-down: Cosine warmdown, starting at {self.warm_down_start_pct*100}%, iteration {self.start_warm_down} of {self.total_iterations}" + f"\nWarm-down: Linear warmdown, starting at {self.warm_down_start_pct*100}%, iteration {self.start_warm_down} of {self.total_iterations}" ) print(f"warm down will decay until {self.min_lr} lr") @@ -373,7 +381,7 @@ class Ranger21(TO.Optimizer): raise ValueError(f"warmup type {style} not implemented.") def get_warm_down(self, lr, iteration): - """ cosine style warmdown """ + """ linear style warmdown """ if iteration < self.start_warm_down: return lr @@ -383,15 +391,34 @@ class Ranger21(TO.Optimizer): print(f"--> Warmdown starting now....") self.warmdown_displayed = True - new_lr = ( - self.min_lr - + self.starting_lr - * (1 + math.cos(math.pi * iteration / self.total_iterations)) - / 2 - ) + warmdown_iteration = iteration - self.start_warm_down + # linear start 3672 5650 total iterations 1972 iterations + + warmdown_pct = 1 - (warmdown_iteration / self.warmdown_total_iterations) + # .5 + lr_buffer = self.warmdown_lr_delta + reduction = lr_buffer * (1- warmdown_pct) + #print(f"lr reduction = {reduction} for {warmdown_pct} with iter {warmdown_iteration} and total iter {iteration}") + new_lr = self.starting_lr - reduction + # 3 - 1.5 = 1.5 + #lr_buffer_pct = lr_buffer * warmdown_pct + # .75 + #new_lr = self.starting_lr * warmdown_pct + #new_lr = max(new_lr, self.min_lr) + self.current_lr = new_lr return new_lr + + #new_lr = ( + # self.min_lr + # + self.starting_lr + # * (1 + math.cos(math.pi * warmdown_iteration / self.warmdown_total_iterations)) + # / 2 + #) + #self.current_lr = new_lr + #return new_lr + # def new_epoch_handler(self, iteration): # self.epoch_count +=1 diff --git a/src/__pycache__/Ranger21.cpython-37.pyc b/src/__pycache__/Ranger21.cpython-37.pyc index 631e95e017451ea8aad4a414311b107ab1244943..4e161f76350f161e39228c57263da6d8367da101 100644 GIT binary patch delta 2169 zcmZ{lU2GIp6oBW>?#|9`+l98u-fMT_XkQ56_twI&H)17I%?f!XZ z%ECI!3Pgj!5aJ~vQbGJT{vmFHzUYI#Ktd!wnBYXCz8K>ZB)&*u^qf0MtBG!Mzq#kk zopaAU_nfoWlGpXXwwfBh0Kc)D&rKfBT?w>H-c)pabmhWdY_rrseq*ZieOa`GMA$+$ z%?QAV)eM9&7lzRsLS58q0=9tMa)!_?Rs^J&sel}_8iRt>5EM=j(6xdl1f!LIwOaFn zu1#|T+d0=*F_^Ele83LN59}=St^ef!UmCY+HF+N0>ayy9-BuLXV}TQid-#fV7HpO1 z<(u?bVNIBq!MQKZ{!0UzPxD(VHGfzL2@&BnbZvcBvfx08_56?;UKWz?l8u#m-vfmG|_rCz4wt0V0Vq zk%&m@!~kKZYo2e*!b00)n5wDA38na14I+Rr0KW_DPApj26)3qH#!$H(F@u;y%(EYA zqoW#5Rm8)HQN(VN=?o$Th@&jn5GU){ zP{YT=Xoxj6lQg*+mfc6YI4@G=XHE5u`j|SWW3^Q@{;aJ+-)8)i+BO?7$OC*UHaiX zeFSFUYnBa0c90Z19vKP9pz^_Z5Qgy*vm-6#(~yP4=rcu4JVA({xgb9Y*25g+ZONq( z3p8`C2ue^ybMBJI@_^PI5`scWg74gxv?ypE%MFw)7tm{oi^2sN?ZFowOM-8`78asp zdB+4z9uwle;wmMXGuUYJrgCvbnkrP+7Lu5`%6&@FqfF=}b8bUmUcmlyQ5F-+oo$zgfx$Rr3kBPdaa_C)FRxc^@Npr#nr8c!N zJ*`vQx8xpuncZq`Z7UN9VKdkiL_cwh0Z}4QFrwduvudIczn2B0uWU-d3c3w3g%}6e zlA6!!SDh%0HXxmsi0wf!?Ns_%rgTNO z@WSKHW6{_v@>Dl;dPq;eeP>jY@@OoTlG8$x=!(h;UH#>0obsE-?521qes=7XY>^pDWN|6dyvIMC{-%wA3Lwh~jA!J;^ld z=pJf*6@+*fkG1VhrgO#u6s#Mb2AyWdxt6b|LVk7TQ*qSg? z;CK6exarI6$8`;(he2HKcJy-;M-ZzK|I@DU74Njm?e+%fI@xgV2-(42?6t-XT!THB zmJ!Dhn7>Z=&f<&*93xCIVoucGz}dS9-tD~ETw;c*niG~A>_A^E$p2XaxFin1zX4;C ro#`8DC<|h>M{bvEWm)#gjdBo1H~YCSLXNOM`?gEi`RG-)v;WxN3tbkD delta 1972 zcmZ{k`)?Fg6vyYz?##|E-IaEC%eLDmEyd{r+QkR2REk(|1qv3TNM$>_GnC!#i#yvc z&?)5+F$N8UD^VWCRy4j4YbKg#)I@(!i2uRF-_+kQLSkaV59&F0qR~V*na`Yi@7Z(D zocq1!mT@x^+8hi91o-`7KR4bn`BA7-_PzA;$Me5^v&h!TE6L|flfR#9vV>HF)d++q z*La>#*^0v`UV)Wd+^UZVRx~1L@o~&z{LrswanyH zIaSuERxnjBoyJfUaTM33OsRY{1@rht^cZ3WVX=k!2)i7Ok}kFwT^zz-^f+P)pvwP< zQ^naM2#iSQaQ#2ysbrP?UHt(|Nu6>?3+669ffHdZ9$47=UX(T<@Usd~tuh9m1n zMavv=|k?lI*Z&8hxb4ts`!E5b-?X0Ad2*c#I=ux(h7GHP1eX4-<|38c&A2 zFy({iQFzQlYS3;q3;XH?LoQP__C&@?zomqPrVYQR0er(r;22re)#2+Weo2noTD7}Ni& zc5A_Q{Mm{@wMw>Bq_4n4^hymgiL!wMTr5>-ROJw1e zt81Tu6}lNQjz|F5z!Dld_m7n(+xO$5rl#eeo7J1lgwkB-z8x_FH?g7gKDObQW;}fl_yidI2qd3|M!Hb#%{S z9<8?!ye0XHrPKt~bhlWy*w{d8q>QV)->YEZ^)(N$a|45`kJd;^r&6mZieIT$BJjkN k8hAX4B9odqVNTdZ@h>VFR-?4aejnH(V@asZwk1yf30$q%7XSbN