From c6e38d27cb3c7b32fb1d3455641c6db5aed22929 Mon Sep 17 00:00:00 2001 From: Less Wright Date: Mon, 12 Apr 2021 18:39:11 -0700 Subject: [PATCH] pos neg momo --- src/Ranger21.py | 183 ++++++++++++++++++------ src/__pycache__/Ranger21.cpython-37.pyc | Bin 5424 -> 6423 bytes src/chebyshev_lr_functions.py | 21 +++ 3 files changed, 160 insertions(+), 44 deletions(-) create mode 100644 src/chebyshev_lr_functions.py diff --git a/src/Ranger21.py b/src/Ranger21.py index ee7e610..3232bce 100644 --- a/src/Ranger21.py +++ b/src/Ranger21.py @@ -10,6 +10,8 @@ # Gradient Centralization: https://arxiv.org/abs/2004.01461v2 +# positive negative momentum: https://arxiv.org/abs/2103.17182 + import torch import torch.optim as TO @@ -21,6 +23,25 @@ import collections import copy from torch import linalg as LA +import numpy as np + +def cheb_steps(m, M, T): + C, R = (M + m) / 2.0, (M - m) / 2.0 + thetas = (np.arange(T) + 0.5) / T * np.pi + return 1.0 / (C - R * np.cos(thetas)) + + +def cheb_perm(T): + perm = np.array([0]) + while len(perm) < T: + perm = np.vstack([perm, 2 * len(perm) - 1 - perm]).T.flatten() + return perm + + +# steps = cheb_steps(0.1,1,8) +# perm = cheb_perm(8) +# schedule = steps[perm] + def centralize_gradient(x, gc_conv_only=False): """credit - https://github.com/Yonghongwei/Gradient-Centralization """ @@ -46,6 +67,8 @@ class Ranger21(TO.Optimizer): using_gc=True, gc_conv_only=False, betas=(0.9, 0.999), # temp for checking tuned warmups + momentum_type = 'pnm', + pnm_momentum_factor = 1.0, momentum=0.9, eps=1e-8, num_batches_per_epoch=None, @@ -73,6 +96,11 @@ class Ranger21(TO.Optimizer): self.gc_conv_only = gc_conv_only self.starting_lr = lr + # momentum + self.momentum_pnm = (momentum_type=='pnm') + + self.pnm_momentum = pnm_momentum_factor + # decay self.decay = weight_decay self.decay_type = decay_type @@ -157,8 +185,8 @@ class Ranger21(TO.Optimizer): if p.grad is None: continue - if not self.param_size: - param_size += p.numel() + # if not self.param_size: + param_size += p.numel() grad = p.grad @@ -180,6 +208,16 @@ class Ranger21(TO.Optimizer): state["variance_ma"] = torch.zeros_like( p, memory_format=torch.preserve_format ) + if self.momentum_pnm: + state['neg_grad_ma'] = torch.zeros_like(p, memory_format=torch.preserve_format) + + # Maintains max of all exp. moving avg. of sq. grad. values + state['max_variance_ma'] = torch.zeros_like(p, memory_format=torch.preserve_format) + + # Cumulative products of beta1 + #state["beta1_prod"] = torch.ones_like( + # p.data, memory_format=torch.preserve_format + #) # centralize gradients if self.use_gc: @@ -190,31 +228,41 @@ class Ranger21(TO.Optimizer): # else: # grad = uncentralized_grad + # phase 1, variance computations + + state["step"] += 1 + step = state["step"] + lr = group["lr"] + + + beta1, beta2 = group["betas"] grad_ma = state["grad_ma"] - variance_ma = state["variance_ma"] - step = state["step"] - lr = group["lr"] - - # if self.use_warmup: - # lr = self.warmup_dampening(lr, step) - bias_correction2 = 1 - beta2 ** state["step"] + #print(f"bias2 = {bias_correction2}") + + variance_ma = state["variance_ma"] + + + # print(f"variance_ma, upper loop = {variance_ma}") + # update the exp averages # if not self.use_madgrad: - grad_ma.mul_(beta1).add_(grad, alpha=1 - beta1) - + # grad_ma.mul_(beta1).add_(grad, alpha=1 - beta1) + # print(f"upper loop grad = {grad.shape}") variance_ma.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + # print(f"variance_ma, grad adjusted") variance_ma_debiased = variance_ma / bias_correction2 variance_ma_sum += variance_ma_debiased.sum() + #print(f"variance_ma_sum = {variance_ma_sum}") # else: #madgrad - # should we dupe variance_ma since stable is assuming adam style variance? + # should we dupe variance_ma since stable is assuming adam style] variance? # stable wd # variance_ma_sum += grad_sum_sq.sum() @@ -223,29 +271,36 @@ class Ranger21(TO.Optimizer): # Calculate the sqrt of the mean of all elements in exp_avg_sq_hat # we will run this first epoch only and then memoize - if not self.param_size: - self.param_size = param_size - print(f"params size saved") - print(f"total param groups = {i+1}") - print(f"total params in groups = {j+1}") + if not self.param_size: + self.param_size = param_size + print(f"params size saved") + print(f"total param groups = {i+1}") + print(f"total params in groups = {j+1}") - if not self.param_size: - raise ValueError("failed to set param size") + if not self.param_size: + raise ValueError("failed to set param size") - # debugging - self.variance_sum_tracking.append(variance_ma_sum.item()) + # debugging + self.variance_sum_tracking.append(variance_ma_sum.item()) - # stable weight decay - # if not self.use_madgrad: - variance_normalized = math.sqrt(variance_ma_sum / self.param_size) - # else: - # variance_normalized = math.pow((variance_ma / self.param_size), .3333) + # stable weight decay + # if not self.use_madgrad: + variance_normalized = math.sqrt(variance_ma_sum / param_size) - # print(f"variance mean sqrt = {variance_normalized}") + #variance_mean = variance_ma_sum / param_size + if math.isnan(variance_normalized): + raise RuntimeError("hit nan for variance_normalized") + #print(f"variance_mean = {variance_mean}") + #print(f"variance_normalized = {variance_normalized}") + # else: + # variance_normalized = math.pow((variance_ma / self.param_size), .3333) + + # print(f"variance mean sqrt = {variance_normalized}") # phase 2 - apply weight decay and step + # =========================================== for group in self.param_groups: - + #print(f"In second phase loop") step = state["step"] # Perform stable weight decay @@ -255,11 +310,10 @@ class Ranger21(TO.Optimizer): momentum = group["momentum"] beta1, beta2 = group["betas"] - grad_exp_avg = state["grad_ma"] - variance_ma = state["variance_ma"] if self.use_warmup: lr = self.warmup_dampening(lr, step) + #print(f"lr = {lr}") # madgrad outer ck = 1 - momentum @@ -277,10 +331,11 @@ class Ranger21(TO.Optimizer): continue state = self.state[p] - grad = p.grad - + inner_grad = p.grad + if self.use_madgrad: + # ================== madgrad ============================ if "grad_sum_sq" not in state: state["grad_sum_sq"] = torch.zeros_like(p.data).detach() state["s"] = torch.zeros_like(p.data).detach() @@ -294,8 +349,8 @@ class Ranger21(TO.Optimizer): # centralize gradients if self.use_gc: - grad = centralize_gradient( - grad, + inner_grad = centralize_gradient( + inner_grad, gc_conv_only=self.gc_conv_only, ) @@ -314,11 +369,11 @@ class Ranger21(TO.Optimizer): # print(f"lamb = {lamb}") # print(f"gsumsq = {grad_sum_sq}") - grad_sum_sq.addcmul_(grad, grad, value=lamb) + grad_sum_sq.addcmul_(inner_grad, grad, value=lamb) rms = grad_sum_sq.pow(1 / 3).add_(eps) # Update s - s.data.add_(grad, alpha=lamb) + s.data.add_(inner_grad, alpha=lamb) # Step if momentum == 0: @@ -329,24 +384,64 @@ class Ranger21(TO.Optimizer): # p is a moving average of z p.data.mul_(1 - ck).add_(z, alpha=ck) - else: # adam core + else: # adam with pnm core + # ============= adamW with pnm option ======================== + + + grad = p.grad beta1, beta2 = group["betas"] - grad_exp_avg = state["grad_ma"] + + grad_ma = state["grad_ma"] variance_ma = state["variance_ma"] + + if self.momentum_pnm: + + max_variance_ma = state["max_variance_ma"] + + if state['step'] % 2 == 1: + grad_ma, neg_grad_ma = state['grad_ma'], state['neg_grad_ma'] + else: + grad_ma, neg_grad_ma = state['neg_grad_ma'], state['grad_ma'] + - # grad centralization, if used, was already used in the phase 1 pass as part of grad_exp_avg and variance_ma computations ...so no need to do it again here bias_correction1 = 1 - beta1 ** step bias_correction2 = 1 - beta2 ** step - variance_biased_ma = variance_ma / bias_correction2 + - denom = variance_biased_ma.sqrt().add(eps) + if self.momentum_pnm: + # Maintains the maximum of all 2nd moment running avg. till now + torch.max(max_variance_ma, variance_ma, out=variance_ma) + # Use the max. for normalizing running avg. of gradient + denom = (variance_ma.sqrt() / math.sqrt(bias_correction2)).add_(group['eps']) - step_size = lr / bias_correction1 + + + + # centralize gradients + if self.use_gc: + inner_grad = centralize_gradient( + inner_grad, + gc_conv_only=self.gc_conv_only, + ) + + grad_ma.mul_(beta1**2).add_(grad, alpha=1 - beta1**2) + + noise_norm = math.sqrt((1+beta2) ** 2 + beta2 ** 2) + + step_size = lr / bias_correction1 + + pnmomentum = grad_ma.mul(1+self.momentum_pnm).add(neg_grad_ma,alpha=-self.momentum_pnm).mul(1/noise_norm) + + p.addcdiv_(pnmomentum, denom, value=-step_size) + + # denom = variance_biased_ma.sqrt().add(eps) + + # step_size = lr / bias_correction1 # update weights # p.data.add_(weight_mod, alpha=-step_size) - p.addcdiv_(grad_exp_avg, denom, value=-step_size) - + # p.addcdiv_(grad_ma, denom, value=-step_size) + #print(f"\n End optimizer step\n") return loss diff --git a/src/__pycache__/Ranger21.cpython-37.pyc b/src/__pycache__/Ranger21.cpython-37.pyc index d9e095461d865aded18b3b0efcc1e2ad4f03aa69..747f841b8d4c67ec6c401592db129f894d4654d6 100644 GIT binary patch literal 6423 zcma)AO^h7Jb?)l!>FMeD-PzgQ*&mWkQj|IRSyF^8B?_gbM2U6+dxcQ6XiucHd#8G5 zXLqJ&SlvtR4tfAVOF$RX-a`-sFof*nW}kh`F~}isd~yyc_+Z!vg-MVIxdiYb$@i*f zR-?5Nq<81l->a%uuimTrUcFf?IvT#W|1JLBhku}HzoEkPGm*K4U-S_O*Es8H%`{~_ zy=kyCZ}u{}#^fBZ5l9uSJTCDBrHEopPMq(t?Y9yI4t7&y@Rl}U-W*fuN zcWaXQ)y5?A*+#>ZdN`1lFZ{3_NPW-}kAeQ+q9)=seHMR72Q*3c!Q6>4mS)tM+G*J4!0r{hO$ zYUOv3q4jarW@9#D2Vaf#gdIT_dz!eV)`oSbbv2C}b!exqwV{n&^CUCYM(jc9KI+uE z1x@jk#(zq=h9yYbGUE&3A4;PegwlE##eQo~ zB-*YSXJU)%iGHLXF*2_wdR`kdriE9*ssDdC6Wa95@K#F%yc4@Gx;t?^h+e&Rt=)-t z_P5qs{ob|j_QUoLevg9AwXX}GcY-i}@ivI?yPcsQclx3Gl;jt4Aom)&G zK&Fuu^Qf0*w-d$Ej5@*OAYkUD|Hn2}}V~sEKMZWYHd(?D3)=aILhelVV zJ+@#p{f?;Fw>guPFd6xwGSpHAC9ef9g-fBl2Qw_o4< z^u^n2|MKgXS2m;tquYXl=nx2e(W<7sv>H;f7}9)n6*e2g!INBKB~DW2>`7)+joA?_ z2!}&;tTPP`uX#q22Q`xdXeOBhwGtCFn`A(3jG%eBBn#`IoRip~`J@b5NGhO392}Zg zN{XQ6q{Ov7Tl`sEIho@o&mXGimKjoU}C zRiwFO{zuv;uwhklz?YkrkuQt?h-*l#AK`#RT18s;#3XK%3dc-17z6U`7G`>7QNM=z z67hvLa%IYKviHhnBO~>$kan-%gB|VnWM+#FENS5Ukp&ofdnfjI(DDz(vrw-(8$^Y2 zWa83%gZsT^_FFzDe;D4F?zP(=#GM`tT(~0e`Jwv=UgU1~yWReyPS|#%AdZnnuR6oI z+kFwt@W18`=e~*U4T(?qG1|7>Vaa{ui{Ad=1-JhYgAVgngtMVS|c-R zCDKtzI*>EKozwHw+2PQj=`1)w{CG#2t)SDD6_R2yJ83DB3lDwK@xxZ&MQFlt)!M`K zGBXgJFmBXv76sjHX{oUz@gi;S%Ot72ACZ_7!NoPID^A5z0yXDnCHA)cR@@hY4j^%z zxEE&B<3Ww7cEl^h@cJpkjIimJikFDtCXwF*kv0#u{rzqnNh=83aDQNdoaeQ=eiV7$ z=l6{G8n|iP=oW~!sI!V*GAxEaT~THkj&7wjhJ|C%WQzvMM#->o8&$wn$AMWvi=z+c zruMu(-Tc#Qimod&sB(h| zf-N&E_(K%Gk6)xNL^fQQk>T@EP7zQBtfx3&8wI#O@D3;&nTer}r8=P~0AMM3*d`;A`}zq&qL zKGp2mUZ*IF01*9d(=eoF(k@nyy=CG}l#$SGo|U7}4q|WWbbm>me`hymIZk(zt}#O2 zYQC$;(4H)7@9TtM3Cb)3u_85)T7+A1Sy|gABr-RZd*UPql<@qAZCRMkPbDQF07z3RQBn%oeC1H`w95ME7*!fzE4H?`o0v^~*Zq5Txb+033*=ULuUr=cr^ zFr!-DPhq-TBRmHq&50YhwOPz|f#Hyc8IH9dB4_}N1GXC1pky>fb}XMJk%KyoHwR$WH8CQ@O*zgDt4eKFN-?SIAXRKguP! zU0eJ`oa4ApKf;~8iyI$GcG8=Cl#la?18m9mblj(KsKnvcO)WNd3yBR{g+G6 z%iSciDN(=Q;dj1-Kd<=9;9rj`z%knH94|c4AtUC@Xv-QezJhZHTqr}v0%R;9Eg~%| zV=0ZwBn#>)zyp*A-#qZhXb$-t>Z?ihq?(iw0L){C9C0C7sysn(msFFvRc$m6Tzrwe27sj?6&bi4DoaQ+fA zUOd*Lzk!upI;nzpAt~0hq_zrestBV#(^8*pIo1#U4s*i}A+i3!Uw>|a827)9^#}ii zv8Pr>Hn^QV zSFInCm4oX;=yYwe3uTo59Ck*tr*?+f`7$)Of^-Gxsv2Df-!GDt6PKTZG*^;E?9&>} zwEJwbnym36SaTSD+!yOc=M^XI$`b7cMmwJ@rK5#ELmT<^m-qxEOnrd7(iDG-avl<2Kng$M z?jk%)N%B+Fz>7egWklSg%E%3?X(??<>o9*QDJcKChUiDWkmzC#Tf!3Z+t|XU|XV9k31G3cnu;2$T z!~b<$HJ3J37|_N+u0HWLlHr92@G%Np08!C7P&Yn+^Pn$q-@AU?OF`i~U#7DfqNZPM$h=w$A@JydWRcVlt`EO9MbqxZ*WBxFCsK;TW5WPI z7{P8DI89Vb{prmiYJXdO|FA*p^}B-||LK3@Q-K+(jQ)PyctJWHpm*Ha!2L4cLhKc^ zRJ3+T%4lLP%b*!+N}ift#T*v+)Lg2!qgPl>x9tj~sTdCWR*W^v zQX|@!-9{hu!;BbTujp&mw7091bXWEB&aikIQIN74<$L$;h%3-T`gZaa$`Pm+4Uo;v zdz<1pDm_nxVkinL3iQwr`aML7((&*X!1ue9M-2I3-|wa^D$ErWDJqi}?^5o&L^>cc z8}`$kr`?dnNu1mYy4`84Oz3_ZdJ?`D&w!BIMI@1(LD2KR{*`VP47&oI+ZI#(8thJ^ zmd>14=gGQy{gD}!H1&J$H0RU$FkIi>57jB;ckzTCN@u3b?kK+&bYkKu1CA+l%Ie>I zLmfdV8z`%HaP?@c&Jme=XZqHpq?xfu6~n0T#lD#;@^y;TqeT$dNoq%AF|@yyU2w|( E1NRDM%m4rY literal 5424 zcma)AUyK_^8QN0iO08DQo@dXoW3Qdr zb&@k{sq`XM0#fdY2TIZ1ViLcuT{m~?aa=6 z-^_gT&2PT>jh`!(Tn*2!F2&#e?rBZ?3l+wXiH|Shjax{###vjdWjX8UHG^f}X2;Ss zrgiLEZdBvc9HpJFlA85=s4Wu)@@cRy`XWM=Wc5}7aFap9iukQ*~?lr ze;4Cv8&xLlb~6gvjf-`iYVa6%b9gV{Rc0kx!cuKd-`9574jbwz6VIeNXV*-zg?b~^ zT4rJ;HrG>qN8e%G;N~5@pzSlJMGrw!|9>=74Q5$`FV;oKn~8VIyP71u`0VM^jb?JS zzqwxTc22+AjT%?+-VB?kUl0LrhEa0r0um9ln}Z-}b|ddD*?*TIxl+}o!J8eKyB3B$ zd{lGNXorzB+s!zUX51Wv(hNBFrP&FCNLoq1*AAs6f~XNz4asiFVx#WYyU`858@0DZ z9=*P&i88YP9A95Qo&8@Xb#dnL)3K<(HZH99w&h%XY+C3yM&_=fZ9IiUGs=d^9Cku? zn8(B{YU)%yeoCW{_zFHkt;V=k)A{0C+7{z3FWk=84E0^)rQ1%;5Q$ta`8RA@w?yB^U?XnBX2GK>G%Ko z$N3M(H6Nb+!DAo1z5V<%8y}v!u=e*4o?JOE?KlZG;Vr^X7>GJKg&vv+!H1S1Uc@>i zTB@hH)J~nW#MvEdSWegu;FPkoystAY1%P}b%_B8a7pav_Behc#X)d*pI_N?3^JxjG zn@%Au0Bo9HObbX$X_0H!9C16D+M6cur8YO0wTiY12tr2^aMx(uy2>P0<%8s84THEvdJKbUNJ*aMJHcYcouOSQ=q3mPKH^aW(OISP!-Z z!TBz2P(cAhn$Pk8u>M-WX%`I6jF-{t_L61?uoj*O1HSFu#NP3?y6txNW;1GdahN2? zsMo0$PD(A?H;g6O&WDEO-?HSu!5%V!|9f?R=O(7l-*t+ zf{w4Ef=dmvo)8;?WOBQRA&)o z!uFQ5Ro}5#p(VVZta17=nKCj?tdW3XgE&Icg^B6;a6>jb@gOOzA1F*1k}ZiiN(zrs z@(^h`JlqQU?If0V7&Wj@5%2T9UvCF-?E5N?i5b$q56L5V<8w%~1)Y`kDZ^&?(^bms ztYp}TRwi3CP&TFvhfP7#W)*0bA#wG=%ovO7#r=QspWWUqETXdNR6?H&#a&8!SxNaIcp-jZ2S(*T#DR zZ@h&ByH*?0F~){wYN%LJp;Hpy`ZZmgNm+v94l&4F)pqoG>|RI=NG_+0v&7glQyh0B z-DeezE)|R%$QimIi2rH}^jC1H=;woQRL z8zA!=xnC{4ERRrxh|&vb_kRbXn4-8wYftNqcNT9&A1ohu_He5@g(!L`r&oEG+)UoZ z?6I~?oIx2Gt>$4nij6Sw$D8TBqC|jCcW6 z;zdf7PbVBp=4J@KLCX`r4W-yYqOl?lerM91hjkA&%}J;e?NHPEB__%>VqgGiUYx%uo+xUc;X+-0|Zli)aQ**ok@+Ae$j7jTj_ zv{C;Y>TPn-Nb(pYxuKIfEmLiWTh}+yE*F0f4s`aMeeEgQFpv)OX}*;gKTTYYqwq}} zx2;0zz|}7Lvyc~(V(NkuGHjQRu!&r5pVJbfRZ1PCQ#^M^N1gL2b@_7yooO7QdQv8O z(3;`y9UUX04@+Cd@%t2F3AAipR101lmZ&d`I)yxse42ETm!VTaKEn$j5vpzu1YtOv zPVLR6#eLSAOY@KwD;l4=L!@LH^(EBLV`P$)Aep`cLJo2=yQ&T60HnFS1?36z`sOQS zD~|j4eMM(-$s#W+BC*J4Qi~Usv@tD!WH!xB){I(ykSy`Jkq;|-%c;4yvadLFvbwiM zWT=&+05hDw{^Bdzu72xhsIjq*`s;fD_Pu$|W1b6`?*j70@w}0%)j)pOS4#)J(ppD$ zbrFHnA5CY7oGE`5;HOGD11pZDWxlwN061Ju7ohoTSm5pI@y~&T<9oC4(n2~juch;= z@U3G1l?lo6qy(7zD{KMc6cYb<4{PZ6Z@c>S_tEz~Bz>06-mWP&Ic9M!w`+(k>;qt{ zb$p-go!Hkfw1w*h3xuoTk;p--Wwbhyu4Jf7mq&;@ z+;$ahfmg^gtkO($c;QLq?fcUsqn_8Oy;_HdKVCOHsx)cEDzsAQ?PyxbdW&A9eUO&b z{y0X+AAQe`kng45-rByl_kfzoaUkUw_QMJE_+Yw}9#2mcMmvR8k&t@5jNPFl;2zmf zM0RuvB4R#)z&1D*gB-=72bv|Cx74`@>euTEoO)i6bTV4>98|$xN}aIN72Ez+S3qCp zhzx=h%ECe9?gThsi0~<@oDnOcU1)3t?cP<0rKJM$U?yW_9#K9o4sL{eu#j|Kwpz|R(CUN;t7S(idMa2;pi&L;zS|n`Cd^7g5k!A66 zA4H{-u?%$(3OahFi19QOr&20%WV`t$^ph}28g6knzhSxd5>sF9Jx z12SBGg}%N@$p$5tC?Tw71gyN-3}XC%AVjtrXXN~3?>r=V$mQHYZA8Gvj~O8sM98!2 zKvBU4Oi!^Cy?zb$2AxfDnKWKTA`Q`r1^rSW*+3p1^>I0S(0?`{Z5dpFqHag#i6g38 z&mjV%rtu?m8NZK2!;cm7%mvLinQ2zo3chs^#G-DSHn@z-9HXc^kkZc@W!qso{#>i9 zgX@4i;?FLd6@3Mza>>Dk0i;sVO2;Uh^SQF&;5woHs0DLOhZXVC^}=B*dc~RrYch2= zlS0>pJ!H|mK8um)LNXdt_2cSgNF1jOd&-839?Hg=U4*@kKBYV6QGHN4AhYCxYU#?A zi((!#6$_LsBH7ruvLTkJRH1}OJMq{e-AbdNgR6;jeZSM?{Wg8$(sI2Yv@?miQwh47 zD7=b`^z{-YuOgATsH=VxQeKv&(WR^&w%g;288ON6T}NHg=1}o4-uM9|js=$esee;v zi?&mSHyxV^YExwwtL1Fef{Fm^>SqFJWto2EQf)4)kD~Rhex%|?(3W}>Nq3^mE~=d< zUO}I@4`W*ix^PMTrDxS1m-$QMpC8n?6L`AEik)+ryPlz{m=dgWh6A9aXt>k=0}9%G A%K!iX diff --git a/src/chebyshev_lr_functions.py b/src/chebyshev_lr_functions.py new file mode 100644 index 0000000..90fa148 --- /dev/null +++ b/src/chebyshev_lr_functions.py @@ -0,0 +1,21 @@ +import numpy as np + +# from https://arxiv.org/abs/2103.01338v1 + + +def cheb_steps(m, M, T): + C, R = (M + m) / 2.0, (M - m) / 2.0 + thetas = (np.arange(T) + 0.5) / T * np.pi + return 1.0 / (C - R * np.cos(thetas)) + + +def cheb_perm(T): + perm = np.array([0]) + while len(perm) < T: + perm = np.vstack([perm, 2 * len(perm) - 1 - perm]).T.flatten() + return perm + + +# steps = cheb_steps(0.1,1,8) +# perm = cheb_perm(8) +# schedule = steps[perm] \ No newline at end of file