From 9cb6d0db266839bd4f08e71fd9fce50785badb3c Mon Sep 17 00:00:00 2001 From: Less Wright Date: Sun, 18 Apr 2021 13:55:57 -0700 Subject: [PATCH] agc, chebyshev, cosine warmdown --- src/Ranger21.py | 298 +++++++++++++++++++----- src/__pycache__/Ranger21.cpython-37.pyc | Bin 6423 -> 9849 bytes 2 files changed, 243 insertions(+), 55 deletions(-) diff --git a/src/Ranger21.py b/src/Ranger21.py index 3232bce..ad1a429 100644 --- a/src/Ranger21.py +++ b/src/Ranger21.py @@ -12,6 +12,11 @@ # positive negative momentum: https://arxiv.org/abs/2103.17182 +# adaptive gradient clipping: https://arxiv.org/abs/2102.06171) +# big thanks to @kayuksel for suggestion to include agc, and initial code, +# @lucidrains and @rwightman for additional code impl reference + +# flat lr + cosine decay: original work 2019 import torch import torch.optim as TO @@ -25,6 +30,7 @@ from torch import linalg as LA import numpy as np + def cheb_steps(m, M, T): C, R = (M + m) / 2.0, (M - m) / 2.0 thetas = (np.arange(T) + 0.5) / T * np.pi @@ -38,9 +44,13 @@ def cheb_perm(T): return perm -# steps = cheb_steps(0.1,1,8) -# perm = cheb_perm(8) -# schedule = steps[perm] +def get_chebs(num_epochs): + num_epochs = num_epochs - 2 + steps = cheb_steps(0.1, 1, num_epochs) + perm = cheb_perm(num_epochs) + cheb_schedule = steps[perm] + print(f"cheb schedule made with len {len(cheb_schedule)}") + return cheb_schedule def centralize_gradient(x, gc_conv_only=False): @@ -63,18 +73,24 @@ class Ranger21(TO.Optimizer): self, params, lr, - use_madgrad=True, + use_madgrad=False, using_gc=True, gc_conv_only=False, + use_adaptive_gradient_clipping=True, + agc_clipping_value=1e-2, + agc_eps=1e-3, betas=(0.9, 0.999), # temp for checking tuned warmups - momentum_type = 'pnm', - pnm_momentum_factor = 1.0, + momentum_type="pnm", + pnm_momentum_factor=1.0, momentum=0.9, eps=1e-8, num_batches_per_epoch=None, num_epochs=None, + use_cheb=False, use_warmup=True, num_warmup_iterations=None, + use_warm_down=0.65, + min_lr=6e-1, weight_decay=1e-4, decay_type="stable", warmup_type="linear", @@ -86,18 +102,60 @@ class Ranger21(TO.Optimizer): ) super().__init__(params, defaults) + # agc + self.use_agc = use_adaptive_gradient_clipping + self.agc_clip_val = agc_clipping_value + self.agc_eps = agc_eps + + # chebs + self.use_cheb = use_cheb + self.cheb_schedule = None + if self.use_cheb: + if num_epochs is None: + raise ValueError( + "can't produce chebs without num epochs info being passed in" + ) + self.cheb_schedule = get_chebs(num_epochs) + + # debug + self.cheb_logging = [] + self.total_iterations = num_epochs * num_batches_per_epoch + if not self.total_iterations: + raise ValueError( + "missing total iterations, which is calced from num epochs and num iters per epoch param" + ) + + # warm down + self.min_lr = min_lr + if use_warm_down > 0: + self.warm_down_start_pct = use_warm_down + self.start_warm_down = int( + use_warm_down * num_epochs * num_batches_per_epoch + ) + self.warmdown_displayed = False # print when warmdown begins... + # engine self.use_madgrad = use_madgrad self.num_batches = num_batches_per_epoch self.num_epochs = num_epochs + self.current_epoch = 0 + self.current_iter = 0 + self.warmup_type = warmup_type self.use_gc = using_gc self.gc_conv_only = gc_conv_only + + # lr self.starting_lr = lr + self.current_lr = lr + self.tracking_lr = [] + + # epochs + self.epoch_count = 0 # momentum - self.momentum_pnm = (momentum_type=='pnm') + self.momentum_pnm = momentum_type == "pnm" self.pnm_momentum = pnm_momentum_factor @@ -120,15 +178,17 @@ class Ranger21(TO.Optimizer): self.variance_sum_tracking = [] # display - engine = "Adam" if not self.use_madgrad else "MadGrad" + engine = "AdamW" if not self.use_madgrad else "MadGrad" # print out initial settings to make usage easier print(f"Ranger21 optimizer ready with following settings:\n") print(f"Core optimizer = {engine}") - print(f"Learning rate of {self.starting_lr}") + print(f"Learning rate of {self.starting_lr}\n") if self.use_warmup: - print(f"{self.warmup_type} warmup, over {self.num_warmup_iters} iterations") + print( + f"Warm-up: {self.warmup_type} warmup, over {self.num_warmup_iters} iterations\n" + ) if self.decay: print(f"Stable weight decay of {self.decay}") @@ -137,25 +197,137 @@ class Ranger21(TO.Optimizer): else: print("Gradient Centralization = Off") + print(f"Adaptive Gradient Clipping = {self.use_agc}") + if self.use_agc: + print(f"\tclipping value of {self.agc_clip_val}") + print(f"\teps for clipping = {self.agc_eps}") + + if self.warm_down: + print( + f"\nWarm-down: Cosine warmdown, starting at {self.warm_down_start_pct*100}%, iteration {self.start_warm_down} of {self.total_iterations}" + ) + def __setstate__(self, state): super().__setstate__(state) + def unit_norm(self, x): + """ axis-based Euclidean norm""" + # verify shape + keepdim = True + dim = None + + xlen = len(x.shape) + #print(f"xlen = {xlen}") + + if xlen <= 1: + keepdim = False + elif xlen in (2, 3): # linear layers + dim = 1 + elif xlen == 4: # conv kernels + dim = (1, 2, 3) + else: + raise ValueError( + f"unit_norm:: adaptive gclipping: unable to process len of {xlen} - currently must be <= 4" + ) + + return x.norm(dim=dim, keepdim=keepdim, p=2.0) + + def agc(self, p): + """clip gradient values in excess of the unitwise norm. + the hardcoded 1e-6 is simple stop from div by zero and no relation to standard optimizer eps + """ + + # params = [p for p in parameters if p.grad is not None] + # if not params: + # return + + # for p in params: + p_norm = self.unit_norm(p).clamp_(self.agc_eps) + g_norm = self.unit_norm(p.grad) + + max_norm = p_norm * self.agc_clip_val + + clipped_grad = p.grad * (max_norm / g_norm.clamp(min=1e-6)) + + new_grads = torch.where(g_norm > max_norm, clipped_grad, p.grad) + p.grad.detach().copy_(new_grads) + def warmup_dampening(self, lr, step): - # not usable yet + style = self.warmup_type warmup = self.num_warmup_iters if style is None: - return 1.0 + return lr if style == "linear": - return lr * min(1.0, (step / warmup)) + new_lr = lr * min(1.0, (step / warmup)) + self.current_lr = new_lr + return new_lr elif style == "exponential": return lr * (1.0 - math.exp(-step / warmup)) else: raise ValueError(f"warmup type {style} not implemented.") + def warm_down(self, lr, iteration): + """ cosine style warmdown """ + if self.warm_down == 0: + return lr + + if iteration > self.start_warm_down - 1: + # print when starting + if not self.warmdown_displayed: + print(f"--> Warmdown starting now....") + self.warmdown_displayed = True + + new_lr = ( + self.min_lr + + self.starting_lr + * (1 + math.cos(math.pi * iteration / self.total_iterations)) + / 2 + ) + self.current_lr = new_lr + return new_lr + else: + return lr + + # def new_epoch_handler(self, iteration): + + # self.epoch_count +=1 + + def track_epochs(self, iteration): + self.current_iter += 1 + if self.current_iter % self.num_batches == 0: + self.current_iter = 0 + self.epoch_count += 1 + print(f"New epoch, current epoch = {self.epoch_count}") + self.tracking_lr.append(self.current_lr) + + def get_cheb_lr(self, lr, iteration): + + # first confirm we are done with warmup + if self.use_warmup: + if iteration < self.num_warmup_iters + 1: + return lr + + # compute epoch + current_epoch = (iteration // self.num_batches) + 1 + # print(f"current epoch for cheb = {current_epoch}") + self.current_epoch = current_epoch + index = current_epoch - 2 + if index < 0: + index = 0 + if index > len(self.cheb_schedule) - 1: + index = len(self.cheb_schedule) - 1 + + cheb_value = self.cheb_schedule[index] + + if self.cheb_logging[:-1] != cheb_value: + self.cheb_logging.append(cheb_value) + + return lr * cheb_value + def get_variance(self): return self.variance_sum_tracking @@ -188,6 +360,10 @@ class Ranger21(TO.Optimizer): # if not self.param_size: param_size += p.numel() + # apply agc if enabled + if self.use_agc: + self.agc(p) + grad = p.grad if grad.is_sparse: @@ -209,15 +385,19 @@ class Ranger21(TO.Optimizer): p, memory_format=torch.preserve_format ) if self.momentum_pnm: - state['neg_grad_ma'] = torch.zeros_like(p, memory_format=torch.preserve_format) - + state["neg_grad_ma"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Maintains max of all exp. moving avg. of sq. grad. values - state['max_variance_ma'] = torch.zeros_like(p, memory_format=torch.preserve_format) + state["max_variance_ma"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) # Cumulative products of beta1 - #state["beta1_prod"] = torch.ones_like( + # state["beta1_prod"] = torch.ones_like( # p.data, memory_format=torch.preserve_format - #) + # ) # centralize gradients if self.use_gc: @@ -230,25 +410,20 @@ class Ranger21(TO.Optimizer): # phase 1, variance computations - state["step"] += 1 step = state["step"] lr = group["lr"] - - beta1, beta2 = group["betas"] grad_ma = state["grad_ma"] bias_correction2 = 1 - beta2 ** state["step"] - #print(f"bias2 = {bias_correction2}") + # print(f"bias2 = {bias_correction2}") variance_ma = state["variance_ma"] - # print(f"variance_ma, upper loop = {variance_ma}") - # update the exp averages # if not self.use_madgrad: @@ -259,7 +434,7 @@ class Ranger21(TO.Optimizer): variance_ma_debiased = variance_ma / bias_correction2 variance_ma_sum += variance_ma_debiased.sum() - #print(f"variance_ma_sum = {variance_ma_sum}") + # print(f"variance_ma_sum = {variance_ma_sum}") # else: #madgrad # should we dupe variance_ma since stable is assuming adam style] variance? @@ -287,11 +462,11 @@ class Ranger21(TO.Optimizer): # if not self.use_madgrad: variance_normalized = math.sqrt(variance_ma_sum / param_size) - #variance_mean = variance_ma_sum / param_size + # variance_mean = variance_ma_sum / param_size if math.isnan(variance_normalized): raise RuntimeError("hit nan for variance_normalized") - #print(f"variance_mean = {variance_mean}") - #print(f"variance_normalized = {variance_normalized}") + # print(f"variance_mean = {variance_mean}") + # print(f"variance_normalized = {variance_normalized}") # else: # variance_normalized = math.pow((variance_ma / self.param_size), .3333) @@ -300,7 +475,7 @@ class Ranger21(TO.Optimizer): # phase 2 - apply weight decay and step # =========================================== for group in self.param_groups: - #print(f"In second phase loop") + # print(f"In second phase loop") step = state["step"] # Perform stable weight decay @@ -311,9 +486,16 @@ class Ranger21(TO.Optimizer): beta1, beta2 = group["betas"] + # warmup + # ====================== if self.use_warmup: lr = self.warmup_dampening(lr, step) - #print(f"lr = {lr}") + # print(f"lr = {lr}") + + # chebyshev + # =================== + if self.use_cheb: + lr = self.get_cheb_lr(lr, step) # madgrad outer ck = 1 - momentum @@ -332,7 +514,6 @@ class Ranger21(TO.Optimizer): state = self.state[p] inner_grad = p.grad - if self.use_madgrad: # ================== madgrad ============================ @@ -369,7 +550,7 @@ class Ranger21(TO.Optimizer): # print(f"lamb = {lamb}") # print(f"gsumsq = {grad_sum_sq}") - grad_sum_sq.addcmul_(inner_grad, grad, value=lamb) + grad_sum_sq.addcmul_(inner_grad, inner_grad, value=lamb) rms = grad_sum_sq.pow(1 / 3).add_(eps) # Update s @@ -387,7 +568,6 @@ class Ranger21(TO.Optimizer): else: # adam with pnm core # ============= adamW with pnm option ======================== - grad = p.grad beta1, beta2 = group["betas"] @@ -396,28 +576,30 @@ class Ranger21(TO.Optimizer): variance_ma = state["variance_ma"] if self.momentum_pnm: - - max_variance_ma = state["max_variance_ma"] - - if state['step'] % 2 == 1: - grad_ma, neg_grad_ma = state['grad_ma'], state['neg_grad_ma'] - else: - grad_ma, neg_grad_ma = state['neg_grad_ma'], state['grad_ma'] - + max_variance_ma = state["max_variance_ma"] + + if state["step"] % 2 == 1: + grad_ma, neg_grad_ma = ( + state["grad_ma"], + state["neg_grad_ma"], + ) + else: + grad_ma, neg_grad_ma = ( + state["neg_grad_ma"], + state["grad_ma"], + ) + bias_correction1 = 1 - beta1 ** step bias_correction2 = 1 - beta2 ** step - - if self.momentum_pnm: # Maintains the maximum of all 2nd moment running avg. till now torch.max(max_variance_ma, variance_ma, out=variance_ma) # Use the max. for normalizing running avg. of gradient - denom = (variance_ma.sqrt() / math.sqrt(bias_correction2)).add_(group['eps']) - - - + denom = (variance_ma.sqrt() / math.sqrt(bias_correction2)).add_( + group["eps"] + ) # centralize gradients if self.use_gc: @@ -426,15 +608,19 @@ class Ranger21(TO.Optimizer): gc_conv_only=self.gc_conv_only, ) - grad_ma.mul_(beta1**2).add_(grad, alpha=1 - beta1**2) + grad_ma.mul_(beta1 ** 2).add_(grad, alpha=1 - beta1 ** 2) - noise_norm = math.sqrt((1+beta2) ** 2 + beta2 ** 2) - - step_size = lr / bias_correction1 - - pnmomentum = grad_ma.mul(1+self.momentum_pnm).add(neg_grad_ma,alpha=-self.momentum_pnm).mul(1/noise_norm) + noise_norm = math.sqrt((1 + beta2) ** 2 + beta2 ** 2) - p.addcdiv_(pnmomentum, denom, value=-step_size) + step_size = lr / bias_correction1 + + pnmomentum = ( + grad_ma.mul(1 + self.momentum_pnm) + .add(neg_grad_ma, alpha=-self.momentum_pnm) + .mul(1 / noise_norm) + ) + + p.addcdiv_(pnmomentum, denom, value=-step_size) # denom = variance_biased_ma.sqrt().add(eps) @@ -443,5 +629,7 @@ class Ranger21(TO.Optimizer): # update weights # p.data.add_(weight_mod, alpha=-step_size) # p.addcdiv_(grad_ma, denom, value=-step_size) - #print(f"\n End optimizer step\n") + # print(f"\n End optimizer step\n") + + self.track_epochs(step) return loss diff --git a/src/__pycache__/Ranger21.cpython-37.pyc b/src/__pycache__/Ranger21.cpython-37.pyc index 747f841b8d4c67ec6c401592db129f894d4654d6..87b415c7875499708b15fb380813b587365876b3 100644 GIT binary patch literal 9849 zcma)CTW}lKdEOg<#o|Jc1WAz8WhGk|*b*hnj_s(jDMvQ#X>8iD87o2Utd=-S5FoIC z&MqVpT})F;ty4E@*iM?6y5kwrlZW`F)0aLp4}I*D+lS2Lq30=0GNWpePMxQ8GVP?_ ze-;2?+S3W7J=g!7J?B6F`7ht{OSzn-;P=1pM>l@;hNAq8D#M?F!c{!Mdq|kV)TUC2 zbG4;abTuv;t%RniN-J4O4O+~KDXG(yw9KuFgP3ZRS?;dFa-QCpzN1%)OnpNsPdvbSl-05-lFhpBHfvR#8t_XF zo@>a1mykqiq|9OEi?URbB~9t;eIr!%)JWS`nKq|H+M*IF4Xvp36HJfvhf-^La!5?#E^h?sHh8S zN#zqLRaL6vr{SSKx$_kiXn#ztsrzbQ-F_m}BJ~bX!Tp!pwAshs4*ZIN&}cXx!|^n8)L6@+eei>Ik0wb69L(DTb$ zTsJx%Z^`|}NuG$(a$D4M7S+H+qNrIlubx%;9Ll48kqpu!_jMVC2xnMQ`q&@Sa8}A4 zy|0GY*&VfyJx1C*PB+w;5#k*0sDG<`pxsebCDO5%#40afhT8joL%)Aq=98%Pj?hx< z04eJ>J-g*H&%Rv`H*M_9E*p}~a!n%9;dMXc(-?&lxBPC)@jC75W*`#sfcPwWL@qwD z!F-?Dc+CqPq6jX4ILIPVR1;e_@o!4yM^Im>61V9`oKE49yKkTnDzw)+en9J&TwW_{LTB}s zNNssu2ZeG<@=+MgdJqaDsP{Z!xQsc%XnC$L5@EN~^h845sIo59ZIP{2ood^^<+T0g z4nILt(=8g4s_E+R{hm`Btm$cx1SE5cUeFCN`J`s5w#rHFs?>&vx0l~n*ctXPJ9QUlS($hjf?g?r^Jj%QQ~@h%_y4f?cQ5{X@9y&b z|M}=wKYC^TwdIer+P{2Ezh5rb9({NA-~RR2|Ftaij^DZ;@8FZ=`@{bGm%n}Sr+=~Y z!n3P`rk^k0e_Z~4wtWBmwIjdy>5~h~B8l6(4rY@!^;}*p4LGrYoTQIVF}MPm452Bb zY?O|2(F{}XCHkdMy#r~)y({f&xMj&mar7vK)QC)^iKu`y$#82Ov^J9b8_@()woJYj znhf&!jvA(;3DQhXszy{+G!f;CRLX!p6GWiHJ$^@xifGG5)69Sbe;7@pmJ9R9CqRu6 zat_!Mv=^Yk6=yQaBb~y0ii1%|i!z;w=1`uEsAVo9`Vw}fIP)@HkgHgflt&_qCAX;W zXoS@`$D$de$D>)Kwxl=_O`?1lhG9Aa`w~M z+2pPotg*DOF{?ixo?#i%QpmH>{7`41mIEc~EdK#`0RAeClX%OA=)tZUv*-bT8>f&@ zevlYY90$b@L4o%*jt}QTc@FI#qkWp_d=sVO;GNi2Lm#8mcQu|xKla%$hV7_>Z;iyT zj`CU1J&pDf$~iV)RIrW(lqZlcBF}&}N$+5!PmZjX6GCfpVYb>W+|zDLB-Y6q3LVBx zWT8lEo1w$Js=L$sy{hY<3GEJVvu@S1rNWe2x!nzIC|)~OymsB+Xxr;v-LKglHwZk2 zmfrPNJqV~NY=>^su7@6%`Y1SO-`=cOH|=_0SKVe6)Em6r8lTVgnXJ;7z=nN_UxdlI z+me=Wg`dZDkcLERHj#LaxvlG!)GIC{J=nW6yt{V06V_W$Y~1Fa%XVU`ve9lf+qY?T zfft6zgUeQL@>-jFWAxA1y~&qx6@8+^;?cKZ3)SkSue-c;zT3HM_wx2_l)IgCcKa4y z(aVl))auQ>A@8&ubJdnyrAc~6hRWJLq^zH@U-f%O|DWECjowl06`Q){*#~sZdZ$Cn z#G0qm)nVtI15MKuyO zlln=h_#!43)|<527dUToX@EtR`kZFFR>LAhLCTSHAV(tEs{2lpSEgyn4y_q38kX#I zuo@jsf|!Wwqkb_p+?~VfL8s~NcuZty8E`mikQ~I{&T6`q(nE3NVRT)u@ z?K##umE4d;a)bOOq9<3De~psYkO-4`8*aB51|sSC5IPURP1?a;pwt}az{TNLK}{0` zcabPZG_|1R^`we_aS8`VQ zB1~9DkH|U>6e2`7^c)9`H>eSQHR@%RlMWc;f`{sl@C4FPR(msJYdDx?w2lNw>7mmP z-a^6~r~ zGRkmF)CPr&Fyt(AkHU;>Z^3rA>%sYTmkiPiUFaVMpVs!1{~jXA>S`h*`Rlq6Y?nd=^?QkCX8)_NAZbB zCp?boMl1d?r~~Oo!sj+1jwFJEN7K~2-kTa7*=W${yvXSso!15`2yJju`zmAyVLC!u zIAzq*&`}2O(P|)gH{@9uk={r!1i1GQ_ESh7CihYl)NdJZ_h^jXPg1^T?kjugC@H0p zPR{6kNZhqaA5%PIOQjLOr?9E_=&(bpWs5h#Bawu*XmuQ^2g&EdHXRJac_NHQN;gxN0Yi!42$WU2iIIw1h!8h27L_{^wvmoJAR9@e zgXXP>AP&G85yD@h+Xrk=gd>XJ7<1zOCgS@mozvth{{r4SxE-G?!#r+x+CF5U?lybb zSf|@$YvGyU)I(byD*3SZv*)2QQ}REWaZ%(B;hj@#%=OA z?MMRYOvo>gOxZ;No54z?E(dTqM^I&a=SkMI1+Z%*O^_*)&4JYhqzO9>3m!r;8wi65 zAcWlB7_XxreUSlpP_hkxwDBhZ3!}uA&cB5|vJn7$5oD`Tg7gHzg_E>(X{WJ8<-u?9 zE1S~Si|@K*;VKc4@sz;?U!l@FNX9RG%tWFTiMr3c?Me*n#$M?m=4OT%A(X-6#9IOd z6?k-c)zD+f!=1;C&qJ+}a_tpIH-2pB2t%c{(<50;e8+ZagK)l7{teWT-9Yx4L#D}w zPWzW>A)j%##wkWeONxaQjC>#AY*5AOyz&+tE*yi#2&08sW;*gD8D=Q&L>^_RHUPe&tnV57%6IU|1K>Mo zu7mO`%InJ4Aj^G7aNpQfx8Dh+Jl;{CS9aC6Ihd5_C(*uv_9U@r(6fe~RNsuuh9T|Y z_RV#$&fH7wD^JC?(&XQTSCo!SDaD}cr=xVk;y(y8%#fa{(#S@DrTP|TP9V>OxyYjX zvZVnWI?Gf7ysjvr-pEHL(g~Kjr=iXKoVN585~Bi*`wYGHIrJGx57>)88%_a2R2tJP zdr!j}@y>)YkB>Z0K_P$&b*wFmHRh4$kWWbA$@eE{EodtM7^5=irUA0`Cs9tLeJYyT zD@GIej4^{Zq=^cjdDd^8}_L7El@sdyA2=cVu5e?%~lrL=TZpHnCqs z?RZq&R{@({!I{j^h+TdAdl;3(>FaO5hwE^#`&sO02D_g_K0C$@d2*lPiqfz#C!PKY zK(pC>4O}CrP~$&9+e5=P@FN3m%|$t^5eBLivG2P4jYYQ&=K)nxDYN%BGUeen6n;AAFI z{~=_L-X6;y-p=O1-38yY7_E2I_nF5)hcN1(wpom`ck z)gVHEZE;}&ddCBxREPrseDV6W3=DupcG?_a0l>>v3?cPFifAp@@>*@a;{Xcdpe#vA?;ajGRDjha`xa_ySF^nn~6bi z*@3_oCNr>szs`>}%K-PFtF+1Kg1+U9z2lqs8t4PbAT)QhEJ6wBo1e#G zREq$!0h@k;KuYd~5WnE2Py?ZDU+jHhh`re-p0O`dj7$*~zAAPQ1QVnozrPMxOXDdR z4{BePzaKAyl?k`mL8N_`m_hy-#lrYBP<~8Ub;K1ELRUSJsRC^GsuWlQ^S$Oan*^Bf zn>4D8M5I07dh~U`6Ir=WaGp3M64c)0&k3L8sNp`f5gy9bw|DXH}MTaEX7Nx}`(@0+q-;lis8- z5Tvx?n9>Mta-iPz0A~?3|x610bU1C#jsGmjZ|2 zp(KZx0ZZsMg^u=kMd)<+9rTt@OXpCeFg)l2Ixvt2a-Eflkr1?8LUn~&=k&)SLTyMa zL2_BTXa^TThFtv5LB+pK$#;mpuwHiqfYHFa<#Ao&6cSX5+|C-*V^H#vWM<#!ryLKs-|ZNpw$S5H37Jq zaO+L<(-&$2S&&#bJitno*P|Bq5~ zj*|0~EKx##9>FgnSzTQLX!;mcFH!P1B~MU7hz$iXll0}scUu6Ch2`MO1^poh%7Bz_ zcHL&&!>`g1j}pQ==;>NYdrq;qOxE9wj%Ch?L)s1&9t+v#H>=QyZYvuIA43#Mt7aFAwFm9v%AmGVryo{WU?mgKiZ zk&szMTYI%K9k=`b(ni;pcL*Q6G~XB2SY2I-0g?ByFnp&0^v5Zr6WYtqNtK7Xj{3ph zyvWaCQe}zstHaNka!F${FQY8lxiGK9%zt(UHHrVypSI8k1jq}?h;Am$l$o|B{ttI5 B!DRpd delta 3255 zcmZt|TZ~&rb>`md>+84I_U`(z5AS-tStr|UvVx?gX`6T3P&QJYU=rNBp1T`6>qq8Z zr%5zsqwGrk>8cJ>0a6rU^H~Hx6!@SL5<*BIBtQZQF7OY0pu$H*5fsk3-j&jpdo^dy z%z4brIcLt94_^7&a`a>*5+v}Q`J40V{bwIW)AYf^eQPseX0X6EEswlNSdfLb2@6?j zJ+Y;hlZ;*@rO3nYzfULRx9M+_vmk97sDp-SlrB&{{@t&r3lzvd)b?#2L~<0#YFyQ* zL1z_%<|!}0Ockg1zSu5eUjoD;Wy(ldVd>Y&8f9S?*$$Ld50A3gwo%qtoQ-UgvX3QL z5_+Ab*eLXV%jgFNgk{(m8{ei|8Cnj$PBc;u!LGBealve^TYUNOrL*!K-x2w5UuHA0 zyKoA6KQtAZy+8?Zh)_g8=pra$jPCfl35RZxz9Ohd3=~R)Do8~YA%L0)1N4azfV$8C z`h^dm0UAgpApA_ZiC9n=07D`MFf8H#BbyA>L`4K(Ohg&EY4Eq4cz=Xx%qMi_%aAn5 zkpTsIqN$50_Ui;-Qjof8zs>Zf!u;K&lVZjer3CtbNIoKumEBPg_yT)lz+?O$P6~Sc z5x7;HA@u|x;l~=X!*sYuc@UJqhU(!b#QZQn0rOGhYZ`#X5Mu-^81=FvD zI<{45m~5S!tQ=}~8kJSksot<`H;4nP-KyTO-Ox~3#kt$ITpdN$s~n@T9I_pgJGJI| zWrIsSES6ngo7b97DdlRmwXx>vZIhb~n?HrSJc1%d$435L4SY$yA70ppvC9u3dGv`* zRN59F?tvdghUewENH)w5BYq6Q({elV)aO18==0F*;{eE*LgPwQ)hYan2dSToGEI;O%{jj9IRqd0J-;kv|u5>({L=&7Th0zGcHQn4)u4(M2wN=f0Tank3FOO(F^ zBOXD3SB}37;0^h1baoSi8b2R29oi{q_8I_iGU-AD)@fH0suz7}yyAdE$ns@D9feT{ zV=YIvloU7{C@N4c2+F9V_BEk|2vP?$O_1*R7-)i!@;z3!=mWUN7s*ITUs;FM;Ct@{ zz6hO6CzoE-`O|>%698PbQET!SJj}Y&c7s>UjgEDO^A<1pyiA@lG&Nxf$9zEU92(6zTK-fUa%JnB8YpNJT-4u)_MQT8^J%ulVwMJ2-r{1{_ zYILx+dfkz4=APM{0{XTf{b|UZ;epbPK*|cGNq1aCKt*1JS$Y5w+f9p6;QS*<&-4`g zC&H-D_7lK6Dk3Q&QaQ-4xQS-^2}=4&N>91-^Px{+b>;fIT?06^O}s3i_`{xZ>(8Kk z&zb1zjJKG^ud$2>-t_lWz6MtX+*qF-(0*Y+z>OJn1q5BYUuA1ll6I#6BBG2!kE4Y_ z+msj|YHR*N81iS61z!|2{#5>bG6T0NGqyK z%k}Bc#<3Q8HE>vGmgJA8cXY2z9Fmvwwd7y^3!hghOVm|c9jCM?f1NKxT%Xh8)f=vc z^-q47&*$7g&92x`v~0@_U+6TQT4R_R`8j!NW`6TCxY9ES3J7pvS8H@ODz0HNR`oE| z?lkywNXFdHR}f%Qbak`cwwlbPD++@kwOz*gH*oeft7%5XzKu9W77waX8(c86Q8 z2EFa=upGN>@|xMK;wzv6`*vxKUqR)x?$S1|^)`= zVofZ?E?qjyi?~Ja8MY6x1q3AkD=U{)_