From 06b17f5f6d1b1c919e0a8d6dddc19f568b4008f0 Mon Sep 17 00:00:00 2001 From: Less Wright Date: Sun, 11 Apr 2021 12:46:26 -0700 Subject: [PATCH] add madgrad core --- src/Ranger21.py | 179 +++++++++--- src/Rangerabel.py | 335 ++++++++++++++++++++++ src/__pycache__/Ranger21.cpython-37.pyc | Bin 4547 -> 5424 bytes src/__pycache__/Rangerabel.cpython-37.pyc | Bin 0 -> 5876 bytes 4 files changed, 473 insertions(+), 41 deletions(-) create mode 100644 src/Rangerabel.py create mode 100644 src/__pycache__/Rangerabel.cpython-37.pyc diff --git a/src/Ranger21.py b/src/Ranger21.py index ac9cae7..d082a13 100644 --- a/src/Ranger21.py +++ b/src/Ranger21.py @@ -2,6 +2,8 @@ # core components based on: +# AdamP: Naver + # MADGRAD: https://arxiv.org/abs/2101.11075 # warmup: https://arxiv.org/abs/1910.04209v3 @@ -13,6 +15,7 @@ import torch import torch.optim as TO +import torch.nn.functional as F import math import collections @@ -20,11 +23,12 @@ import collections import copy from torch import linalg as LA + def centralize_gradient(x, gc_conv_only=False): """credit - https://github.com/Yonghongwei/Gradient-Centralization """ - + size = len(list(x.size())) - #print(f"size = {size}") + # print(f"size = {size}") if gc_conv_only: if size > 3: @@ -34,23 +38,25 @@ def centralize_gradient(x, gc_conv_only=False): x.add_(-x.mean(dim=tuple(range(1, size)), keepdim=True)) return x + class Ranger21(TO.Optimizer): def __init__( self, params, lr, + use_madgrad=True, + using_gc=True, + gc_conv_only=False, betas=(0.9, 0.999), # temp for checking tuned warmups momentum=0.9, eps=1e-8, num_batches_per_epoch=None, num_epochs=None, - use_warmup = True, + use_warmup=True, num_warmup_iterations=None, weight_decay=1e-4, decay_type="stable", warmup_type="linear", - use_gradient_centralization=True, - gc_conv_only=False ): # todo - checks on incoming params @@ -59,12 +65,14 @@ class Ranger21(TO.Optimizer): ) super().__init__(params, defaults) + # engine + self.use_madgrad = use_madgrad self.num_batches = num_batches_per_epoch self.num_epochs = num_epochs - + self.warmup_type = warmup_type - self.use_gc = use_gradient_centralization, - self.gc_conv_only=gc_conv_only, + self.use_gc = using_gc + self.gc_conv_only = gc_conv_only self.starting_lr = lr # decay @@ -74,31 +82,38 @@ class Ranger21(TO.Optimizer): # warmup - we'll use default recommended in Ma/Yarats unless user specifies num iterations self.use_warmup = use_warmup + if num_warmup_iterations is None: - self.num_warmup_iters = math.ceil((2 / (1-betas[1]))) # default untuned linear warmup + self.num_warmup_iters = math.ceil( + (2 / (1 - betas[1])) + ) # default untuned linear warmup else: self.num_warmup_iters = num_warmup_iterations # logging self.variance_sum_tracking = [] + # display + engine = "Adam" if not self.use_madgrad else "MadGrad" + # print out initial settings to make usage easier print(f"Ranger21 optimizer ready with following settings:\n") + print(f"Core optimizer = {engine}") print(f"Learning rate of {self.starting_lr}") + if self.use_warmup: print(f"{self.warmup_type} warmup, over {self.num_warmup_iters} iterations") + if self.decay: + print(f"Stable weight decay of {self.decay}") - print(f"Stable weight decay of {self.decay}") if self.use_gc: - print(f"Gradient Centralization = On") + print(f"Gradient Centralization = On") else: print("Gradient Centralization = Off") - def __setstate__(self, state): super().__setstate__(state) - def warmup_dampening(self, lr, step): # not usable yet style = self.warmup_type @@ -127,19 +142,17 @@ class Ranger21(TO.Optimizer): # @staticmethod @torch.no_grad() - def step(self, - closure=None): + def step(self, closure=None): loss = None if closure is not None and isinstance(closure, collections.Callable): with torch.grad(): loss = closure() - param_size = 0 variance_ma_sum = 0.0 - #phase 1 - accumulate all of the variance_ma_sum to use in stable weight decay + # phase 1 - accumulate all of the variance_ma_sum to use in stable weight decay for i, group in enumerate(self.param_groups): for j, p in enumerate(group["params"]): @@ -149,13 +162,13 @@ class Ranger21(TO.Optimizer): if not self.param_size: param_size += p.numel() - grad = p.grad if grad.is_sparse: raise RuntimeError("sparse matrix not supported atm") state = self.state[p] + momentum = group["momentum"] # State initialization if len(state) == 0: @@ -176,25 +189,37 @@ class Ranger21(TO.Optimizer): grad, gc_conv_only=self.gc_conv_only, ) - #else: + # else: # grad = uncentralized_grad state["step"] += 1 beta1, beta2 = group["betas"] grad_ma = state["grad_ma"] + variance_ma = state["variance_ma"] + step = state["step"] + lr = group["lr"] + + # if self.use_warmup: + # lr = self.warmup_dampening(lr, step) bias_correction2 = 1 - beta2 ** state["step"] # update the exp averages + # if not self.use_madgrad: grad_ma.mul_(beta1).add_(grad, alpha=1 - beta1) variance_ma.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) - variance_ma_debiased = variance_ma / bias_correction2 variance_ma_sum += variance_ma_debiased.sum() + # else: #madgrad + + # should we dupe variance_ma since stable is assuming adam style variance? + + # stable wd + # variance_ma_sum += grad_sum_sq.sum() # print(f"variance hat sum = {exp_avg_sq_hat_sum}") # Calculate the sqrt of the mean of all elements in exp_avg_sq_hat @@ -212,46 +237,118 @@ class Ranger21(TO.Optimizer): # debugging self.variance_sum_tracking.append(variance_ma_sum.item()) + # stable weight decay + # if not self.use_madgrad: variance_normalized = math.sqrt(variance_ma_sum / self.param_size) + # else: + # variance_normalized = math.pow((variance_ma / self.param_size), .3333) # print(f"variance mean sqrt = {variance_normalized}") # phase 2 - apply weight decay and step for group in self.param_groups: + + step = state["step"] + + # Perform stable weight decay + decay = group["weight_decay"] + eps = group["eps"] + lr = group["lr"] + momentum = group["momentum"] + + beta1, beta2 = group["betas"] + grad_exp_avg = state["grad_ma"] + variance_ma = state["variance_ma"] + + if self.use_warmup: + lr = self.warmup_dampening(lr, step) + + # madgrad outer + ck = 1 - momentum + lamb = lr * math.pow(step, 0.5) + + if decay: + if not self.use_madgrad: + p.data.mul_(1 - decay * lr / variance_normalized) + else: + p.data.mul_(1 - decay * lamb / variance_normalized) + + # innner loop, params for p in group["params"]: if p.grad is None: continue state = self.state[p] + grad = p.grad - step = state["step"] - # Perform stable weight decay - decay = group["weight_decay"] - eps = group["eps"] - lr = group["lr"] - if self.use_warmup: - lr = self.warmup_dampening(lr,step) - #if step < 10: - # print(f"warmup dampening at step {step} = {lr} vs {group['lr']}") + if self.use_madgrad: + if "grad_sum_sq" not in state: + state["grad_sum_sq"] = torch.zeros_like(p.data).detach() + state["s"] = torch.zeros_like(p.data).detach() + if momentum != 0: + state["x0"] = torch.clone(p.data).detach() - if decay: - p.data.mul_(1 - decay * lr / variance_normalized) + if momentum != 0.0 and grad.is_sparse: + raise RuntimeError( + "momentum != 0 is not compatible with sparse gradients" + ) - beta1, beta2 = group["betas"] - grad_exp_avg = state["grad_ma"] - variance_ma = state["variance_ma"] + # centralize gradients + if self.use_gc: + grad = centralize_gradient( + grad, + gc_conv_only=self.gc_conv_only, + ) - bias_correction1 = 1 - beta1 ** step - bias_correction2 = 1 - beta2 ** step + grad_sum_sq = state["grad_sum_sq"] + s = state["s"] + if momentum == 0: + # Compute x_0 from other known quantities + rms = grad_sum_sq.pow(1 / 3).add_(eps) + x0 = p.data.addcdiv(s, rms, value=1) + else: + x0 = state["x0"] - variance_biased_ma = variance_ma / bias_correction2 + # Accumulate second moments - denom = variance_biased_ma.sqrt().add(eps) + # print(f" grad = {grad}") + # print(f"lamb = {lamb}") + # print(f"gsumsq = {grad_sum_sq}") - step_size = lr / bias_correction1 + grad_sum_sq.addcmul_(grad, grad, value=lamb) + rms = grad_sum_sq.pow(1 / 3).add_(eps) - # update weights - p.addcdiv_(grad_exp_avg, denom, value=-step_size) + # Update s + s.data.add_(grad, alpha=lamb) + + # Step + if momentum == 0: + p.data.copy_(x0.addcdiv(s, rms, value=-1)) + else: + z = x0.addcdiv(s, rms, value=-1) + + # p is a moving average of z + p.data.mul_(1 - ck).add_(z, alpha=ck) + + else: # adam core + + beta1, beta2 = group["betas"] + grad_exp_avg = state["grad_ma"] + variance_ma = state["variance_ma"] + + # grad centralization, if used, was already used in the phase 1 pass as part of grad_exp_avg and variance_ma computations ...so no need to do it again here + bias_correction1 = 1 - beta1 ** step + bias_correction2 = 1 - beta2 ** step + + variance_biased_ma = variance_ma / bias_correction2 + + denom = variance_biased_ma.sqrt().add(eps) + + step_size = lr / bias_correction1 + + # update weights + # p.data.add_(weight_mod, alpha=-step_size) + p.addcdiv_(grad_exp_avg, denom, value=-step_size) return loss diff --git a/src/Rangerabel.py b/src/Rangerabel.py new file mode 100644 index 0000000..58b225a --- /dev/null +++ b/src/Rangerabel.py @@ -0,0 +1,335 @@ +# Ranger21 - @lessw2020 + +# core components based on: + +# AdamP: Naver + +# MADGRAD: https://arxiv.org/abs/2101.11075 + +# warmup: https://arxiv.org/abs/1910.04209v3 + +# stable weight decay: https://arxiv.org/abs/2011.11152v3 + +# Gradient Centralization: https://arxiv.org/abs/2004.01461v2 + + +import torch +import torch.optim as TO +import torch.nn.functional as F + +import math +import collections + +import copy +from torch import linalg as LA + + +def centralize_gradient(x, gc_conv_only=False): + """credit - https://github.com/Yonghongwei/Gradient-Centralization """ + + size = len(list(x.size())) + # print(f"size = {size}") + + if gc_conv_only: + if size > 3: + x.add_(-x.mean(dim=tuple(range(1, size)), keepdim=True)) + else: + if size > 1: + x.add_(-x.mean(dim=tuple(range(1, size)), keepdim=True)) + return x + + +class Ranger21abel(TO.Optimizer): + def __init__( + self, + params, + lr, + betas=(0.9, 0.999), # temp for checking tuned warmups + momentum=0.9, + eps=1e-8, + num_batches_per_epoch=None, + num_epochs=None, + use_abel=True, + abel_decay_factor = .3, + use_warmup=True, + num_warmup_iterations=None, + weight_decay=1e-4, + decay_type="stable", + warmup_type="linear", + use_gradient_centralization=True, + gc_conv_only=False, + ): + + # todo - checks on incoming params + defaults = dict( + lr=lr, momentum=momentum, betas=betas, eps=eps, weight_decay=weight_decay + ) + super().__init__(params, defaults) + + self.num_batches = num_batches_per_epoch + self.num_epochs = num_epochs + + self.warmup_type = warmup_type + self.use_gc = (use_gradient_centralization,) + self.gc_conv_only = (gc_conv_only,) + self.starting_lr = lr + self.current_lr = lr + + # abel + self.use_abel = use_abel + self.weight_list=[] + self.batch_count =0 + self.epoch = 0 + self.lr_decay_factor = abel_decay_factor + self.abel_decay_end = math.ceil(self.num_epochs * .85) + self.reached_minima = False + self.pweight_accumulator = 0 + + # decay + self.decay = weight_decay + self.decay_type = decay_type + self.param_size = 0 + + # warmup - we'll use default recommended in Ma/Yarats unless user specifies num iterations + self.use_warmup = use_warmup + if num_warmup_iterations is None: + self.num_warmup_iters = math.ceil( + (2 / (1 - betas[1])) + ) # default untuned linear warmup + else: + self.num_warmup_iters = num_warmup_iterations + + # logging + self.variance_sum_tracking = [] + + + + # print out initial settings to make usage easier + print(f"Ranger21 optimizer ready with following settings:\n") + print(f"Learning rate of {self.starting_lr}") + if self.use_warmup: + print(f"{self.warmup_type} warmup, over {self.num_warmup_iters} iterations") + + print(f"Stable weight decay of {self.decay}") + if self.use_gc: + print(f"Gradient Centralization = On") + else: + print("Gradient Centralization = Off") + print(f"Num Epochs = {self.num_epochs}") + print(f"Num batches per epoch = {self.num_batches}") + + def __setstate__(self, state): + super().__setstate__(state) + + + def warmup_dampening(self, lr, step): + # not usable yet + style = self.warmup_type + warmup = self.num_warmup_iters + + if style is None: + return 1.0 + + if style == "linear": + return lr * min(1.0, (step / warmup)) + + elif style == "exponential": + return lr * (1.0 - math.exp(-step / warmup)) + else: + raise ValueError(f"warmup type {style} not implemented.") + + def get_variance(self): + return self.variance_sum_tracking + + def get_state_values(self, group, state): + beta1, beta2 = group["betas"] + mean_avg = state["mean_avg"] + variance_avg = state["variance_avg"] + + return beta1, beta2, mean_avg, variance_avg + + def abel_update(self, step_fn, weight_norm, current_lr): + ''' update lr based on abel''' + + self.pweight_accumulator += weight_norm + + + self.batch_count +=1 + #print(f"self.batch count = {self.batch_count}") + if self.batch_count == self.num_batches: + self.epoch +=1 + self.batch_count = 0 + print(f"epoch eval for epoch {self.epoch}") + + #store weights + self.weight_list.append(self.pweight_accumulator) + + print(f"total norm for epoch {self.epoch} = {weight_norm}") + #self.pweight_accumulator = 0 + + if self.batch_count !=0: + return None + #self.epoch +=1 + new_lr = current_lr + + if len(self.weight_list) < 3: + print(len(self.weight_list)) + return step_fn + + # compute weight norm delta + if (self.weight_list[-1] - self.weight_list[-2]) * (self.weight_list[-2] - self.weight_list[-3]) < 0: + if self.reached_minima: + self.reached_minima = False + new_lr *= self.lr_decay_factor + #step_fn = self.update_train_step(self.learning_rate) + else: + self.reached_minima = True + print(f"\n*****\nABEL mininum detected, new lr = {new_lr}\n***\n") + + if self.epoch == self.abel_decay_end: + new_lr *= self.lr_decay_factor + print(f"abel final decay done, new lr = {new_lr}") + return new_lr + # @staticmethod + @torch.no_grad() + def step(self, closure=None): + + loss = None + if closure is not None and isinstance(closure, collections.Callable): + with torch.grad(): + loss = closure() + + param_size = 0 + variance_ma_sum = 0.0 + weight_norm = 0 + + + # phase 1 - accumulate all of the variance_ma_sum to use in stable weight decay + + for i, group in enumerate(self.param_groups): + for j, p in enumerate(group["params"]): + if p.grad is None: + continue + + if not self.param_size: + param_size += p.numel() + + grad = p.grad + + if grad.is_sparse: + raise RuntimeError("sparse matrix not supported atm") + + state = self.state[p] + + current_weight_norm = LA.norm(p.data) + #print(f"running norm = {current_weight_norm}") + weight_norm += current_weight_norm.item() + + # State initialization + if len(state) == 0: + # print("init state") + state["step"] = 0 + # Exponential moving average of gradient values + state["grad_ma"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["variance_ma"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + # centralize gradients + if self.use_gc: + grad = centralize_gradient( + grad, + gc_conv_only=self.gc_conv_only, + ) + # else: + # grad = uncentralized_grad + + state["step"] += 1 + + beta1, beta2 = group["betas"] + grad_ma = state["grad_ma"] + variance_ma = state["variance_ma"] + + bias_correction2 = 1 - beta2 ** state["step"] + + # update the exp averages + grad_ma.mul_(beta1).add_(grad, alpha=1 - beta1) + + variance_ma.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + + variance_ma_debiased = variance_ma / bias_correction2 + + variance_ma_sum += variance_ma_debiased.sum() + + # print(f"variance hat sum = {exp_avg_sq_hat_sum}") + # Calculate the sqrt of the mean of all elements in exp_avg_sq_hat + + # we will run this first epoch only and then memoize + if not self.param_size: + self.param_size = param_size + print(f"params size saved") + print(f"total param groups = {i+1}") + print(f"total params in groups = {j+1}") + + if not self.param_size: + raise ValueError("failed to set param size") + + # debugging + self.variance_sum_tracking.append(variance_ma_sum.item()) + + variance_normalized = math.sqrt(variance_ma_sum / self.param_size) + + # print(f"variance mean sqrt = {variance_normalized}") + + # phase 2 - apply weight decay and step + for group in self.param_groups: + for p in group["params"]: + if p.grad is None: + continue + + state = self.state[p] + + step = state["step"] + + # Perform stable weight decay + decay = group["weight_decay"] + eps = group["eps"] + #lr = group["lr"] + lr = self.current_lr + + if self.use_warmup: + lr = self.warmup_dampening(lr, step) + # if step < 10: + # print(f"warmup dampening at step {step} = {lr} vs {group['lr']}") + + if decay: + p.data.mul_(1 - decay * lr / variance_normalized) + + beta1, beta2 = group["betas"] + grad_exp_avg = state["grad_ma"] + variance_ma = state["variance_ma"] + + bias_correction1 = 1 - beta1 ** step + bias_correction2 = 1 - beta2 ** step + + variance_biased_ma = variance_ma / bias_correction2 + + denom = variance_biased_ma.sqrt().add(eps) + + weight_mod = grad_exp_avg / denom + + step_size = lr / bias_correction1 + + # update weights + #p.data.add_(weight_mod, alpha=-step_size) + p.addcdiv_(grad_exp_avg, denom, value=-step_size) + + # abel step + abel_result = self.abel_update(None, weight_norm, self.current_lr) + if abel_result is not None: + self.current_lr = abel_result + + return loss diff --git a/src/__pycache__/Ranger21.cpython-37.pyc b/src/__pycache__/Ranger21.cpython-37.pyc index f3141831ed5d6b8e177e568fde7862de1e5f0c04..d9e095461d865aded18b3b0efcc1e2ad4f03aa69 100644 GIT binary patch delta 2895 zcmZuzU2Gf25#HT99+$@>d88{e^3<7ThX_oMNyz{28tH0NCt^Zxz&l$L{!=^RZ3 z$Jr)ZT?xL!PW1kA^2)^DAD=5-3JuS%t#3M^vDt8)n%uB4B6xD}#o^Q7^TsG4K2c1? zm7%yw3Az{S+CJSSeWs|g4;Up%Q>3b?1W;X>Kw~Nc)KEImxQYSg;DJ;VD%AsKsx;7~ zK#`@AKvT*RWpY44Z@Y%Fkk<*&tRhX#`<{roOc>p)pA+$2 zN(peTvhR}j*suoyH4{(z&`!z6{wq0X81E7RaW-w>$@g?Lhn^|_4K84srg>;ke4k1a zO*zm)cx7~4M$g99!pM5!_8X$Ip|r(+uQwY0`* z8Y~T3085+k;DNDneg+(bR<-ImK0wTOs?~WWmrxjs`twx206mF-lUXD<=XUV1@sW)Y z$}>Pht<`Ylso-Dnak><6{yF+qaFu_Rp9LK_SPFj3EA;K)cl_WbN-mHqeC_(vMMH$)DVE0{Qmy|a2&!b!L$Y7VIQcWy2|8aOqFpkWdpz9lq$1&jue@vql*@ayigbFECjOE~M^@p~Z)n!X#eZ zRd2gh^|nIlh3wwxoo8F^78HT)R%QBP<#eUtRou2;*=V;qP!galSKe&+cPbuS@t#w` z@mM#!`j?{T_m}?mA9&{Hf?uYt)02UfK1MGFXVS;%eDLM;T<^0O%w;5aU4}-j({Wr8 z>J8s%$x|rOy)Eg>MKpO4`&y^{=Aee?qUP5^L%) z8fwrF`A}G$pBY^5aWM9_rNcuV8lF5Ee2^|IeF6Q`W=K174d=rvT1M%*j;)uF+(2>@ z$ug2sP|jR_4$G081QJ?N0-bxEYVD1B$Tsf61hv+>d=)KTLkmqpAzZ~4-3hvxYd!Sy zUHH65K!~Q(JT;+0=~UMXv;=JiRm)W8=NwD&6v$B&~gf|Ve17} zf}Wk?Q0q9&(Hv?tThGUBjYB1jUK~NW-s9BLc~lxXEwMr@M=h!|bD#!e69U4J@+=2a zVKE&}8aXnV4I0jE2*}n<@L+2bh6Mq0mS$W(!gf-x*4(-y&z@QIe5W&?T3ua{d046( zMH2Y-c(7+L%wb)R>T9&mas|6DBl!xDFz&Xim|H;ZpX{k#RF=Iw#+q~DescNU|+pfEOIZ1v{a$UyG8i?_Gk4gCM9ZHCNs hLo7%Q6rP#=^?L1H|0J=K{v>JB+9~}>MClJDsHIJ#@>8{qS`;W|n{kX?$985n zRa(YakO~eEDb*aSBKy(tCAzEm-kbN{yqS6P z-urIy$6d)&iG+pV8!s$>m>s>7%;9(M?A?BRL8XY|y0)N2RA116k~&&87EFw&NmZ)d z#tX5ti0ah1ji^ypYSy~45P!J&3C2UhAq8xUqogkGlf8Ie{6g}b*+oZg1)mGUfT6(f z{{{#UCtT$U*SJOThSqihypFnrWA5|_MqKBpq;LaJX%T>V_3AAzs5To+#ym7>dC6O?2fp}9&z%?t(Hkx#t1Wg+y3nqb18=HNrr8@HVyDF)vEBHlu&kGz5fH>+NQM6=5pP?s7rzZOcprTo zpGKSb>eug+w$|1+@mc}oI#u#$OEW0Gw@!5eo5RCVe4Vhz6hmHWCs3f&65Iq#QhkGf z%y@y!JRuXsVb20rCRHs(V;iI$lO8mbN7On3i4Qh9jpq>@BHo6bsF6qQ1W*%bl3KhY zg<2kEAWPd0Pju5f-ov#Fq_V~1IYjLZSr|!>+aS-vR#IbwCb6MN4Tm7KT_`KJJJ3T% zP+MKOc10e|*u?%1P;97V1cNjg)hBIz?YDj)VfLtPa@*x)8vfqJY#J;MF0{?*!Puuj7~12h9P z3p5LK04%q|s(FsOJNN)k_Xx-`AYtHhXEzNO&G3Ov7V*>w#HS}~^Z(%-d@h`Eo{l0p zBxAJ2J6kcz@mzOX4|PX*wujq!*l-9UHw;^i@j;&F!||<{%1c~<+*x@FaU_?rs5tNr z!;oPW7}3Y3#WrUG&xo_mtKtjiV}lKYyf`kF+;ezRJal*CW8#s!2k#O8xP=c6fs(x@ zL8g4Dd#zTvK|{3~l$ZOJqWUW=VEd%WNok@|FYrReqck)D7HRcHsW`%-@-P! zH!5N!J=c*|pTh7z2SBQdv)IC}qGDCe;bGtjv=Lj0nhCKKo0vq6^scI70>7oX#FmYQ zxJr%;gWlDl>3}4S(*)#-OR}1cli0>8v4{!nr~ljFVIsG+*v8Uz64X1$t;9vIQ7NS9p?%Yat4 zE?>4>Q$zSge{#RAnl7Bs)CrR-ig>nY_dUmhCR_aIK(iH8mqV>j$o%=}S=o884b{bF f>n0Ic}0kmq*n4@SB;~! diff --git a/src/__pycache__/Rangerabel.cpython-37.pyc b/src/__pycache__/Rangerabel.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..205fc8bc682b00e9dc5095f515f290549e77b965 GIT binary patch literal 5876 zcma)A%a0>R8L#Sx-EKd|^Y+Z_qe&p(Bs-fdkH8YLkWH3{mI)*)0w$s9b+u>hX}7)A z?b#ix&4F1d7dGqxi35r>!;OD|BjSJ%aNze!g0D8J-I_<8S`#amM~YgV|%D^J%-0h~fMmM+HgZ{;@ z4dUHfc<+SWi=UN&=!S86;VOU(qV70IyMx$!NaugZ0d6!6Wr}WJ75Bn$h)%Pp%qWbN z6?KzTSxI*sDk~7eS5`j^Vr8eJVH7G`2Jvp#G!?(AD!XmJJ&5=HK^z^(5_bI&lXW!z zcyxRFV*dX!k;_X@Urc2CwR{lV45RJgfm&|Q4j1~n(*ti|by5RhX5F;7!yhpm?s2&U zOqbT`hl{eP+$*(A-LHz;J*Q=f1yR4pTDDjeOK1yXS*)Nf zidC_O)(J~T?lBXaVqG-uL6EKT8_Z&@3fR7-Dzi(_$$w<*{_+>}{N>8-6K|~j_E&%Z z>y@`>n71!~=c(VndGO+MJ2&3`uzr4h zuG=EkGvCp$lsV|Ty9}@lJ!F1G!)jJSzm_f>)k#Z*xdN>OW>HvxOPXdmtDwJd?K$$Nv|8ArkeQ;e!5VBU-=h&1vNGrz`l}o1rYIgls-QsnC3km7+SO1Rr|Vj1u?-jLq3t zkOkFunzk|`shs|x59J&6m3=czgG8C(Fi{nl&hD+$7hyX%kYpvhxe)(+<=R8?0Fpe? z_|i-ryumQ-_F*W}lVKnZygRT)uQQ0E!JTfr>m^~DqD?NlN{G%6lmu|Ica6b~@wg z&ZzIbI2^QZC7>TSXm~S7+qc5R8{!ZlO<@K~Z;5U@RdzB0UOB$sjk~Gut8zT*`_m=L zrGZ|PsB(TAKRp;i!$%2hx-A#LFI9%xOJal%&b#fAlwdve9ik$e!s49s+V_<8_V6sD z7zfbXtA$ABV(~jcI~_1Xk7Sgn)%`$rgSZ{~38vs-+k3d5vWK!8 zr_Cj}&oJt!g5EEYZ;TB&rF>gD(j#0xP=)#rP`FM3^K6JtDnPQzpSh6$n zgW4BTFi9CaAI~^NOl`${r)kJXFrzBI4;zDerJ?Tw`AI_Bdfepld5q@na}~n_ya|1Q z@$tr~bG)-to4!qtjkkc8mL?qlcoa6Fn9e8g&e{VuC}lw3*fZp%jHihG#D1DR<_$*ksSR%nFD2}^%tQWdgfaN3!p<%}hUafrJ zG0dhLMB_>>eUB{78!vkCAoaR^1ZQ$-q1bK~v<}FR(@GOIATR13_E7Fuf@l=JDCIyl zZOt#Oolw{`91`meC_d#M$azO1&_z0e;AT4n_w!T*u}J7KDzQIdPs5U z)mMBB49;-l9+HY@n8EyfgiGrYLVk~4QsAMcGp939YJ_l}A~nGLNhn3%K0*T0(@895 zkuK1z0Mig|b>Ieu!z4`kEm0sNm{=*2m7a|RxVXpU57Uz|1@DYZ2(!Z3$lDU75*mY^ zZe7BxAdAQ9e*)vkKMG1P9EgYYN}pg8D=r91RvetDnjsMoH8LrkWYGEDJp-x3Bk$0I zZh`2gsR#+UK5lrUp`Z{QNkqmZ6duxM@>DwZ>!e`3nuo~{LPE5kMa}VQI!J+u2eLnp zHW5#EqzP#}{12Xo|0eJ+fNM9#A9Nq5$NkLnFJAX3S|KVR*r#C|hWw-#hj-{4h}1R_ zx?_D3uR}%2G$#;n3v-0cvivZ`cCL#y>uV5wkKalsFl@8*`z$H<*Zva<(51jKX8OJizx{ zUfx?3rTYfXfIV!sfjs&d1X55HMOk~e>ZC?{;XDgyVQ!PUa8W^keuFK;#3qZ`!qHk* zJ?6dj%*9-_!9?Xg)f{!;ar#~ZU!s{BW~%p5hG+F`af?lsU|LH@8~Rq(?ar%oHt;g} z4S2--A~NzCnWN6k3&LGzGt&d6o;mZF>B>K)TVip_#WP1|GmCgec|dyZ=)5-MB1O{4 z^6l%dLVkD8U|m5=vOX!^{ZlT%7uf5+dq;8)40-v1ar-e)PUVu{eX;`TCGcYv?K0Xm zPrC0vXj>}WS{iNX933N6J)AM=pm<;-Mc!WK;AzbNO&PH zNc-~hphBKl?T7t=Jn-T4K&pyVwfji0%O_E=A0S{;)}OmeN_w%*4x-^L%qUyOweez} z7<-gydr7b#ikb7&6P^y8xZ;sEX$IFy6_#?Y22rHTaU1188?^-8MwLRiU@F5xRnc4GrwPRIM1BnWDVruE zRq7^w&NBQqfGVZi&r_;A+8ftDqb%~7sG)*1P!?*2zIM3E#pwo#A9eRawJ?-n63YE> z%1-$~VjHDXd4g`CQt_orpLOs%Oq+(vM7@!F^Uo0>Wk~W>0y@RBlG`%Xo?6!m@;WUx zu{hsBboYJvb>N!k^Bkqvjt0p{;)foz(vcjcFK+fy9pMmN9f@~oVj zR#-Yze3_g*=V7O}e_Ab_Y?SXzdWxDa9`uzvJ#2pfdFx=OOTIjcK`5w-PDMC4um-TRb3?A9y|E)3gxerM^~H^AlyFz6%#VXUYANOVKDaZAsE;b^ z_9%$*8C@1i3cy;s)&ORoj1mfms-m?K909Lqgd-- z5Gf;$l{+`)*R&PMo7fY-EDVMRl1fFnM}X{7m0p_t;G;i0rzCWtyh?JO%^CL`p%MZR MUM^e`qv