From 8a406e1f9a7b1117d676a4df68ba06514d5295f9 Mon Sep 17 00:00:00 2001 From: Amog Kamsetty Date: Sat, 28 Nov 2020 10:09:38 -0800 Subject: [PATCH] [SGD] Add PTL Docs (#12440) Co-authored-by: Richard Liaw --- doc/source/conf.py | 32 +++++ doc/source/images/sgd_ptl.png | Bin 0 -> 61827 bytes doc/source/index.rst | 1 + doc/source/raysgd/raysgd_ptl.rst | 117 ++++++++++++++++++ doc/source/raysgd/raysgd_pytorch.rst | 3 +- doc/source/raysgd/raysgd_ref.rst | 45 +++++-- doc/source/raysgd/raysgd_tune.rst | 2 + .../examples/pytorch-lightning/mnist-ptl.py | 29 ++++- ...{ptl_operator.py => lightning_operator.py} | 24 +++- .../ray/util/sgd/torch/training_operator.py | 44 +++++-- 10 files changed, 269 insertions(+), 28 deletions(-) create mode 100644 doc/source/images/sgd_ptl.png create mode 100644 doc/source/raysgd/raysgd_ptl.rst rename python/ray/util/sgd/torch/{ptl_operator.py => lightning_operator.py} (96%) diff --git a/doc/source/conf.py b/doc/source/conf.py index d710fe7d7..5df6e8be1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -44,6 +44,13 @@ MOCK_MODULES = [ "mxnet", "mxnet.model", "psutil", + "pytorch_lightning.core.step_result", + "pytorch_lightning.overrides.data_parallel", + "pytorch_lightning.utilities.model_utils", + "pytorch_lightning.trainer.model_hooks", + "pytorch_lightning.trainer.optimizers", + "pytorch_lightning.utilities.exceptions", + "pytorch_lightning.utilities.memory", "ray._raylet", "ray.core.generated", "ray.core.generated.common_pb2", @@ -76,6 +83,7 @@ MOCK_MODULES = [ "wandb", "zoopt", ] + import scipy.stats import scipy.linalg @@ -87,6 +95,30 @@ sys.modules["tensorflow"].VERSION = "9.9.9" sys.modules["tensorflow.keras.callbacks"] = ChildClassMock() sys.modules["pytorch_lightning"] = ChildClassMock() + +class SimpleClass(object): + pass + + +class SimpleClass2(object): + pass + + +# ray.util.sgd.torch.lightning_operator.LightningOperator extends +# TrainingOperator, pytorch_lightning.TrainerOptimizersMixin, +# and pytorch_lightning.TrainerModelHooksMixin. +# But, we are mocking all pytorch_lightning modules, causing the ptl base +# classes to have a different metaclass than TrainingOperator. +# To fix this, we replace the base classes with dummy classes that extend +# object. +# We have to create 2 dummy classes, one for TrainerOptimizersMixin and one +# for TrainerModelHooksMixin so that we don't extend from the same base +# class twice. +setattr(sys.modules["pytorch_lightning.trainer.optimizers"], + "TrainerOptimizersMixin", SimpleClass) +setattr(sys.modules["pytorch_lightning.trainer.model_hooks"], + "TrainerModelHooksMixin", SimpleClass2) + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. diff --git a/doc/source/images/sgd_ptl.png b/doc/source/images/sgd_ptl.png new file mode 100644 index 0000000000000000000000000000000000000000..232f6e0d764601a716d3452a9c561b6b60a33f8e GIT binary patch literal 61827 zcmeFZ^i(k)7ebms^GVbthu6hvBT^pKX8?hpyt2x&J`U`$#@ z4j6o1_4lwT|85D;AJCLkdC zdFux76AK^OdIEw_0%e7ldcLTw`J29aGhs)2eQeGY57U~Z+_If029O6wHg#FU>>9OP zS>e=T46`H~aV%rxZ!#0Fe{{}_<7!kBDE?JnN;mwP&g(h_(d|$7ABDBP@)+$9!n7Iw z5cyRc`}EQRc1&mucOA{~HP7)F#ifN10uTScfBzSO|BJx?j|j{(K7Og3aF0~wHuHm& z^ZupTB-8ir*U;;-P+4(K+|ttLLRNWa)v})<=O<@lAq&d*#i_e@RLW+$n7{rBF?i3U znD8Y0iM_wHKixIlbJqFPCo7jLcMdN@a%5Mh3z`2~FaZ$YEXZlMpoM>reARJ4y)I#$ z0>bAy9%hTFlwN$RwEsIW;S~66Mt0KgS_TPP%(&p>{y0+?)0D5p_0x`5tsJLDu!&@` zgT_M42|pPD!3!}Q$W^D^+&$*T%_bU9^v%cNo;n|{v*8ZZes@U&-*{d>^nDWhI&rfe ze>ZR73%2l2BN|w}3&b@7f}iR)ZVqQja*D5ixyY+XB`XfUFGmPq)t|ZxYGIet&r7S( z@%MaATvcv8e((l3c&ACI1*|lLvwU)*|E8|otvmwYLk^7JyQpSLadNhpT)tUDzvp@! zt`!Pk&S}NVrr|6xPR?~6@LvkKTf`3u2tK>eEG_vB0uiOf=-zpr{y#q+rDMoW$q(U_ z(j;ULzklZ$@U<5Cx?c|Nb9MjsNX-U!Tp5Ye@5^y^1K%kK_4#ulvH!;Z{`%D&SHb`N zb=rlc)9>IMCpkUTC+(L^ss^FtniDrlB$@u0Z>UR0H`jSaRq zC>1-4h^eQctgG)U7@v|Mr%t4?G|+gJ-Y0=S8@99^qqfy)&)X)4ygFds56@Ecs8^gE zFiPDJFL(E?H%l)OZ_poXS&r~hgp0+|zO8yja8O8;t@fXLaD^o5n1gvN&CT-he$d&b zXW#N9^D9+a%zZZx-OFQzxh@iKKJb&cJr)vYo-}nUX8fVOc6=XFPqqDfn(%Dp^O~A= zG!uSwYasO|!JTLSg72xnx1DCd^WGp09+JC04ec-C#%V*3#>>r|f8&S#*ce^POg&2$ zXLh7QQf6DNV_6QxW>w!lQ+!C!T1mi9OOZV(_n))<#L$kSi>U{dFwIlSd+R;4_67&b z<<}ce%%8j>CvE!Js_mU9BxRV>u=eTXIom^m|K5#%B3qxJ9aQzXsF>M}<1NFdXSCH` zwvDG*zLCEkLbS_bi{LqCftM|}xJ^>jC2~IHZE3(KE^4b@EcBaQColtR{cJA+^0z_f zDw&AxnnH!Ss&iKiI%SZhBRR{H!TbSR0xxP8?=&$y6Eh6(PBCj(XRF7lz;^Ggj{kX9 zw;a`Qd#3R2o|Q^bdqmr9g4S9B4!;93n+I?Hl~bCtd00E7OqX5~&KIqOte?R&&ob@U zR|^Pen~P2G?q;p(C=HmTjz3HO>kZn&uYNGT`G9S%f+6;6skQYm)_aGO;4=Z`Rk8ia zvu!om$ym~}i2lmSBr{Mf-uDsaxtg#SyFE#S>$@IF$84_+Z}pl!o&dks@3_SrRex%= zEV#-zr%&*+<$&xhg73l76J}<$|9nk{mUd@b*tv7Nnb*TU!QLk(D;01lE*!kcUj>&_ zlD2r8)XOtmkzqo$czOlC8>xM`=_5W_5kH~&43S6pS(oH*JK7)h-z(rlZcxxl%Yf*a zZ1hKcQp7{TjY~6^^*DC;lEu6{@Y`ui&};ot4VZi$IC;nUslO`@|M~hR7bAM|BJmHj z$vu+r1%_(rlBziuw zgZHY}_d58Sv8eeYeIs;)vs?KUp}~iL zKtpslkf|Vr4c!=IU#knEur=tF&G86$p?zqz1xua{=F#cm3wFa(%wwx+);F59>g+Zk z=wm!<(!q?m7@RK&13fsXOC|h#JSH340i^tDzGB~~$fE~umM9JM5Fqed`;ZA$=fRTH zX4kqe2|gcF?NiO9NV4sqVhJ8!G!;w~7G5X+J%$rGz3o~isry*KkNzPmKysG=NHj@_ zi}(Se*!eNygqr}^{tfxh0WoXfbwH~+Z%gb*6Wn@Bs9_?p+H*4i8j$2DvGH?O*CD^NP13{d zN1KaFZtme)0QbJAlmB4NX;Bx%7<(=s(?m+29%#hhbFbYHD2kcFZ8kyQmz|txjEkiZ|}CvYJ|oTh}mQp^IgQqk1QJ zZ6QMOv^wuP<(+3$y!NT7O*}kWmjUl|(|gp?%%2?+%}72qL&8jWm^X)J#Kw7D7;>sE z0-6<54ib@zWVT`7dEtXC$PE`?>VGJ9{ShkJhxbd=#ZG2^NBO=7C~H>Z&f|7}Tq@&U z`HehV25-IMk#oda(RXLf!QY95a)kXNdckO3MD(PEhF6)?f#9^iu~7QW2Ns3KrD{Vu z14YVyLEw)5>i}f2oArKRxWAww4SU^>dLV)1T1806jZ6qlvAxZ%#GhLujr%}dABI;L zH-;Lhu(ms|uHeRli(5K+&SZ_K?&!_th_lop;Ja#Uq{@UYG|nmfLd)Cr&wn6 z4p31)4`~nnGf&lmn^GDzyrl>^RoKHDwL{>gA5d%R9+8lFSk@sqVObTC2xP>Wx#Kh7G6{WgBNktQ-*Qp3u*+$`DMypwd*P3Lc8 zP*u;MvY@HUTTLJ3!}xLq3=}*4U#BW!0&5pG)1oz1axGLslwx;D@B^6-x_CnYox4-wp+QPJ>Bsod%pR z4fG){*JbG0ZduBrx%*EVSbAwEr6VQB_n~OIQ1o{e6uTf8Q`SX#;1#Qx_Dx{D^OIJx z-L0jHa)xpmWTle6-}H(Le@m{?RES30g$G&oSHAp#F9)hsfufH4(xlP<`e9P@m!)@qzUK58HtzxF53TJmSPrpR#CgSZS z74hVw~?Q)jj+Z8hWF?-CIeC=pzkZ|^3wA{Ba>uhbjPeR!=F59LCr0UsJNzmOd%qMR1`HeorA5T?BQ2)qSRA(!z zfOLefoskF2+%xZ*3sy_6nGW5#F)!NxJ77M29Rn~F0N?r8PJqEv%xis4o!3(a> zj@1H6_d@{DhU}5!6Ugo}y zzQ&)TuZsK`s3Hb#Pc9pAJ^nC%x5HPn3M-s#CXBg!@ft(-TPV$Xgr|y}Gcrqgvwj~H zjld6!*lq1qDa@7+71CN=@^!ru8e*i1vv1n9SYt>3N z#DZ!2=h4lIFIlNa>Qx%Tr>Hb>3%->ZJ(083TDiwcTV!({-D3t*t&)Zdqf)mCsyYdL zhTdw2Dl_wS4K5G1(ZEOpwx}={h9ub#-UXG8pZ@+qRgY=aUlBX&@~C&YMCie*-k7m}oQCl?X&3R& zo09bQ>0Bi;wV2h}o-=P{+lh2!lz#_|B1EyOZ=6nr`3>=hy?6HP`gn zBq>Tr^6CbRI6=j(ewk8u8nev#Ra_tr)4oo9IjE4o0(!k^3GQg743 zPac+wzs-KHx`($5=bvNI&=@)Bqo3JusA{NjJS^{nPmtT@-B@m*#hi2+RG5#|b>&VL ztF+Xaiilpeo;uo&lYCXo&dDs2nVYDUd(irqg2&$mVI`f-Uws#ZW+BFH@vAV|2|b&kCG#DjE@t(Wmg%`1)j3c>S6izB zfA^i446V>T%~;lWTQH*fiTD()|9UB%Jw?jms5G^5cFvL(Wt??h)j=jbm}~Th)0uv_ z@%6X2>$MFGnT2gbt2+q;4v5;@KP z$5%%3s(bO491zWV7xf+EF(KtDqY~7Soozc2{JdN%l0ZQC9NvKz#HNe;H`&VOd~K7H z&RXa-q8?^#WNtz)y^4xPeqDF0)jjVOKjL};rb48cJ_d!$J{l$H3O2gFMZ2Vx>@K;G zcW*7{#{VQo|By=(HX)LlYLuzfDAy8faGL6dRBK^6EjR2*Lr`1kg z4oHy#6nhml^7CSlS93AhKM<8Wu4deZs8Gj%7DLty&Aq}+cI}T$EOOo~oE^xJ#NQ*; zaQK~hm7Sl`IjgfW*YPs~-Vpmggkvgs9r?yf#o>2rEY&mG&s6#!J?_~4?HgIk`s4*- zL|#2QkW2MP!M*2xY*fE~y=*N*^6Ty^zUk6>LvCEEh}B;3B5*wow!l}<isNVk!i`>CVF;c&ZPFxPrS;D1ZLD zHuY?uXOw9nGmX4l(3kC-1ffJ~Rlf`3nT%4po-B9xkU$1JGU-f4y;CegLhpPvFq{b7 z=t%XWw>TM3@&P80CbxN@4=|5hsQAtMww)-~>u*h|W~*d$WEvZeNlsTegFbEJPtoz; z=YR>7OE)|ml=6W9>Mj8<{^1oFE+uXDR zxNx$g7t%uqq3bg$D;n^ZT6Lk*`sf1+3J>KCxOg^P!o8tlFp$+qMxVaCdcq0->OkQd zyltSU-13`=-H*FauaIH`nReRDCj>j zrbkdXQZ=Gmuu^G46ZYqGy#VY21MAv|uUY>qgJdCv0P=+((`1~;b7K=sTG8QiW3A2P z-F4YUIsQ&vv;3fSPmVyh7yU+x>JFW!*LS7=1<48Np;ha<^G&e@QLqq%aqS<>o?dJ8 zU{qqnT{8=%A8oSTHOp!eJ#8!ZkP?lyz6%g;nkQT3^v~c=VH|%>QvO5Wh-Bc&@Q?OSrKiUO#hYx%it8Kgi!m2S>#gOj z53E>OU(COl$_sVjle2Q$h0w0!6E0a;(iHIIj4*wh40jk`C*z)BYK}+M$Ava*1V2C5 z(&<$N+9v7a);P0_n8%~Q7j%6CoCXDu=-3%mgLZrmxlBJ_AFKbH^?PD5joRMYWoh%3 z6LpjJLPC9id({pNDTO94#irPTB@w(Kf>UfEnSXRHgFYb6dJ#x?vWJJlmMYTf!Z*@a zp1)5Np9zgqHyrLC?aJPJxbD3u-p>_6VM*=Bdaa)R028PFI&C#eALtgoye=JorK@YQajc28xo&GNuWZn~QkuedO^w-^ z)t;s-rl*!)s9he)7r*^TLn)NvVmkf#PZ#DFkDp7dx}R|u`$A`fu<71_REn+(Gy3kE zf64W>35<@2qycwQwCxUC`<2u^sQ$9cSz4P6AoYVSQ)ov>Zvn#n#?4Ww0dYY7c>XDA z%vY&hULi88ujR$))A$B%cf94;MRsY;nQPr&)#|l06(tk>kTU_PBCqGAH2Fv9X8)HP zFq;uzg05GW5aE*Nfn}6lh_Ws2zc49kXWJ7Lt7-jdVIx~-yzsS$*Ci*H!$nira4xW= zOKH;CZ-O)Hbn!Q}_V@rS?kEAFrP*axb@9%J8Q^5f7K_^wMYvLaJ|3gL%Of3%jVtjV z_A5E0+>jmt;VhY%o1o(=T92ii-(-bMGZvANHUF+8Ni`UfViv$QS@@CQxzMb+t&gGI zw$l2oMGSL`^_Gg$5a}pl{gi?M(18n_Sim1tdkX>C)F*8)t2OA2q;1&B_*9zwaL+`f z_Ydgp^<^iMjMLoqvnyylJ*dSrf_K#`U!RQLn{_{e?(Wt5k9vCJM3y1?8yZYi^C>5h zVvGJ8Rk*`gVgNy`g;K3iqZj(>tmY;v{#xQQG6g$ zM*kPu{V4sJEuQW5$+@*QN=p8a{-!#GZN4`9?2hfUMCwGDd@`yfZP8t1s^SK}-%TO_ z&klauj9QpyxA0&)?s1U9h9_g}b-`N)fr)naFeW2AM%nmjU7zBuhZNa!KYXw5_6^T& z93Y2?Jn4k{v00`;{}MZoVbgk6Tu(CUO_?sF9L3MGy3AR@((X~gGA#e z2CK&kcim@QZsuMfo@NEQ`{n(aoX?A6tX^@-_lyTAK=>2C$>3rh2w_)emM8kMP7k*! z>8&C_3M&X_^6ygj0a{R4Ewi6Bhu!LkiC4!5q2k-|=m1cr^psi677ouDEgrnAH4KyZ zzRb4dZ5Lp#(Zt}Z$;OwvVkJpYIO%?PKz=Ek@ni}JsSYa9(ni`8{1M&ZVj zohha4)%<^@f31KwhOuk*e1~Fg-j*cm@2nGB->_Cp_d0oXrpSO>B>w3raA2Z%vb%3# zl9Gi@V5E;n=eqrgs2?**n$t(mytX!TjQ7&rtNpTD0T(u>Tme=&C%!fAh&HS3jowe! zdT2gcS#O=5J4diLly7$F94Z?JE^w5zDu8qLHfb?>KD@G=T@&0Y=_&ZNW^}0AI_m3d z)iY#wr+M&$q=ze73$xl!cbCm>0Tb{z4GgD#bhj%+`zSiSLn?+bHDlOoTd2#dVGH*j z6Qapf<;5T4>w@+;E@`Eu0ot%^R71l8jit72oN!d=&{H}L-Zpn{u-v8s_IzwLZQnx? z%oB8^INLk68c1H-D7P>N>WBaO$}=|aX_##$&1^|k#OlAN=6d$fsPDI;-^QpP&A)8L z&ckj#5xdvr@;edvUUZkQo?&subn#-SdOTIAixG22?14#w6vu_FJPM$7eX;j|0?Ji$ zMV#eVZfw1vT2e^+gliz=?Dj7+Id^vp`x9O?s)Be>;%hx$(-;{rS8SLoW}VAfP4ANI zGk|Qsu=22g-@5RAWA9_OCC6B}pY<0N4Q~2z{5#4*K|CzPS9xbE4`jiMsBH8csKvx{ zMzh>Gsu~I5KL>Hg4OctRG6S(tByo*3%VDJWOytC4JW@GEq!X$sdS>I$dFx=PK(;?W zg$?8Ied6sdkNK}f_`6`OnZ;(@`NvMD1Mg!r{#0vQW&2W5b2VdpkMwb+e&&G%TBy5U(+Q_ep=2&`P%JYT$ccEiG z>6G&=sWwsxO*#Zj)BAfZ=6J=hJsae^ixP(D4!!%Jt-WLC?7mIAm9&ayj8D89%Y!$jdU_e{Ts?fSdy{vq(v=grTI85qu!H5L-Clq!WLn&K8B7Rr z4|uT`*GMwcZ-oX0+25rc#(q#p3v@O8aMWX$RVy^dlIcHOca<%c)c17jdb#OU(Pp)2 zGRgOV3UaUH4G@i&^xsP?YV@M4+)O~{di$BAdN9&!4LYl-%R^!uZMH5t95!xxw9P&4x>_sx6`R>AnrOLqlLK-qcTi3oVi>! z99Qh5X|mG|L(UGIa2Zu)dpPGA@CpR)UbR=Z_qa3itEeWL1c~~D? za<1?G;rEfUjxw#yHb3P4P5UYrUeDd9Ad0i^4bw-%Ef4vD5(5h_1zR&<-8!Ch*rfEn zprXScC&yqTqLRJ=CXdxq9D#~fNcBA;ig#QX>x}_!d5e7B|D}|Z-;=w%jKSC-;Jp+ivpHC3_s*3 zE7O(=86HFQGb;{HijKJIzGZ5{Z&X{X!KPNZ70Vc`yMF98T~c`L(c0&l6R57OeZ=S@ zP8YD4(Tmd$TWLn7=Mlyk*pXf*T~Np$#^H>v5VFV_Wd2#YrL6+Dqrg?YVs_^W)gcHNeQt;BREM}i4L z(I>1ULz8FgPyZg#Lm!uRnZgjPWq&N7hT!TV`ZapIw+CJ8#F_pWcevKET(MQ%ibm!H z&x&sxqm8GtVl9$0YiR@YL4pM)$(v;5*dn#D=N=4yXWr=0aY&?!Y$Bw|%r~te53HbxvwkcoNq&MaVzF;Y(q>w5@|J- zHBf&k0J{h6IGD5jhlWyTv$8FREjM)%)NuG>vFHiS9p`;tM}IAVQ&GyTW{5rLpgevr zG7^sop_Kx11@qtyS9Q$jecAZhk3l&Hf#5TzuWqmYsv66UAjsWc!t54#{h=45ln2aW zQ0d4_$s#S>_aXS$Anh`7U5eN<6f^a2DrdiyXa~Q|GAqA8burs!NH&v4J9B(sq>&~m0WM7sHp`0mx}i7o3*@^K$(a^2GQ z_xihg2`EHW;-`^COQWd9vH);Ps>?HY?q$TMSQS6SsUU9Yo%CKoT5U>6Npmk&U_D=a z=t;`X)1K4Z?U=@o&>C;_k0&L+(o#4KDkOMbRq<Z=D7jaMW3g-G3292#Y!C{h)Hc?0CSxV^9iTFXnyN|AxhK9#OyB=gb zlG5Nm(_#s44Yv>K{WXuj+Klch?KMVa231o#F?$c3*FXNITtX`g#LmW(3F%Vt{miaj zT~)+Z{S2s9lH`vyrR#O>lNQf<41+BS_2HnT{f%VX(69fv>9CK;|X!-etkw17ZxSc|y~@XWAQ-0Gy#S8~{9 zM8JY%zMiY7PmTCy&Z==+A9VoM9^B8RrB&PhL|`LiOabsf@wTHG*E*eX4%-m*h0GdK zueEb&KdA!xP?oJK`cKP8&jtF5oTpVwN|a*jl=n6x+PN0;b}|!-E}WHX?poFVBa2Bk zqg=!M&ub?eo%6l+$L1zzdDS8PL)Ft0u(Kdp^91*;gPMIu%F9njJAjktL%ja|ST=wl z)tYE@31nasA7;J2J*BF+3&T^fjI%gVW-p&H6i_yweNVqb@ z(KJvh#I8C*r|DH@E4U>v-MGFF8R2MBd^ERgiYuxY1{?=32#X1lxlbPLx>y{YrxqD= z@o)K}Px9#M%=ti)w!t=J@g*CtAq(7N0crQzWRP%ifUZ|LK>3S|77~~4TP9LB)cWuR zQ6KtCn250IKJN+jjt5q;Yix@B?smY;6f+>!!;(1w{un1Tw?SF1Do%0xN;$kE+v^B) zYZ=zg)}dPa4?)&`-)O;(9w2p1^P}_MdrHf8Vd#@K-QXsB%%`P7)_L!vz) zzLwjgXK2~fu8+FsU@m*6#BtMprv}Se5o0<1C!I&;3^tTrX7!~R)Q(h2 z#c#_fF_Nh+RU$%dWLJcOjn>n$3DXMR(d0E1c0WF{#(|IOg&Xvb{@RQ(7Y-LRt8Lzy z8Mtuu>R?%n&M(JC&)>Nv z!c{5bIl&Dq8;Mu4Sv-qX7ru9U{iac-KznhMtg`BwZY+H2<_FUhY^z)xIFUPM%V)DA z|J=E{Zih4(*#L{O0OQGk*@QLytW0|84qK#$66zou)g~G;I&sW5mzRcf=!a^BlDqD$ z3;Jfu2eFJ)b=vSKZflR8GikM7P2i&P0gput7lyaq%m|6*XQmcu+!4EJml~>k!aAJK zwve|4=ree~h_-dU0y~eQD<)eN*o66pPX^@%*zJ8Q4S5aHhk`mlw*X!Eb5n{%Y1f*h zB6<7_yYbQ8Hkp>!mu(_~jR1A*o=`=%6Zy@as2zd}wyl)I6(#1km7c7#aZU2j0n?Dw z9xdF$mIfY{vbbZ(5F^Z?gonf`mfqJ1oPCa-MQM~n9kRN3(n(D zX3#f9HJYlcB;7~Y9yHX@03z&XkAlk=ehk-78vVXq|IDtm03A;;XHd|Nf zrz*9NHC!$I6jM9?3u1E>dc7un%f4wRrP?F3InF%Y_SjPiV|ddmq&{XU*0EX9xBpj! zwL|f(sojQ~60zddtBiMRC%5zM^`WOpJ@SMs!LN5Fi8x(fG_7kVwIJ zM{I^q5CfCO(0HT8U`99Vfm>Yt$A0Au)G5%&i0Tg(`LpZ+LFW1B8?>Up;1y%-#mOQz#7?{)k^d%>Pk|tBJ{1 z=JRsbslxu+5%i-pd_cgLa1rKI#AGcROQeRf=DEjY1=Jo&aDP*G2%%}1J^EAiZHzz+ ziXO2WfeC4_juh0`$+{S*4+3UJc*`PCk!B-C`3}+nQWB~coR8%d9%j8&G~?`vHg~Q% zqP6UDt8yt+oQ!QOKci&WC{)%sOcR>NJ_p3N?x|hlMkc6F70(gxS(hqrH~Vj~ku)X< zO>rro%~!WZev6+3q1@l^KEo*l$^(P?^UJ?hXPM#GKnbgQvmiQY{SL_Ya?SpUaVP}o z=_czVO%l8*0}LOe)^V3TC26TI;NO0#5G2*J&ReMKvc-C z)``g`{bv6iQyV=kp7rISjqY);lmb?vtG|;MTMkYgID{ zNs-kj`A+asH&UKgP`hGy(DP!=WNL1s3Z!N@i3c>f5s4uUST zJsh$4WTEfM7K=-?s=|;fLg~L*{}lnVIERlP*k}hp4Vg+Cym$hR4zXCr_KU{zYg9L_ zI=j5Lzs~B$KfBtGF%1|o&6|G!mUOBIENx%3aIf-JpHthMVIB$%#LU6P3?4MewC8D& z3J0472;`p}l@c*HlV3I8HDe@pl#4db=ZOkb`l{HkyCOaINfLM(J4FZ24UBgH!k1jq zYK^SxN~3v`aDVv+EY&Z-;siT>4d{OwLcqiCeg<`PT|c!NC5X%nr-M)`vDpd$t+-6T&ve4IW-ykz?B)U59+Z$#^ zN2bSHL8Z7DId2!4Oc37O(Rz09dEFN7yD&_~*x~?neIKN`P}5ztxMJR)d-bv0*0w(H zDi1paNNc!;D(YjE4Ht8urT2^t9(8dVW3Rl26o2n-=gwiz)1a%J2Q$M6ILYYNSd5jf zi!A2YiaeTA7HZf5p8i~U2B-?Dn;|}_djRnwm^TmQu>EoKRgq)NAyz8Lq4?x?(sJgb zs=HUB0I;gia5(F#=3VTvw>?FzM2^^UGPzZeF?e~735)q>pILC5G9AuOWt=W9B4jjx zrMABu@eYkM^yNwpmP6MAjqfKnGNrr<=ZNvq`A|nG62W@xB$a8{QC;-SBcF^fAXxTd z!x}ST*uMih?MZ{KxrYE!u75CZQf)45lbajOs|W_|Abl#fe*=bXv8>03GEq_3)~n(k zalQdWbG`6D)0y#NDgV80aMyZjfy9(l1k1*$J8<&`Z>17pe`5|EBj;i6p8`U)LdRI| zIM<|DaF(Wi3y6pF7c(Z4RZBhr_9cZE?%S5QNvz!B7h>GTR@&Th>L4E*BwM#48(N8n z6}STadV*wCizBC$Me+o`*(SHl?bNc`ecEUiy0Z;H?(+@5PuvD=IlfplOtaaT`TXJR z>T$d_#|rNm>}1%OQC>P1uJWZv$3?^}-oY@J=^-8ndiAad?_;wqdTW}UF>h@;uzs0U z#jO9ZA1^p@RQ=Mq|@X(FO)(w4vt4mn4vp;9!1* zFLAtakliY#wvbtXbW4hg+nIA8U&|w4{YiX$>cHhPZ^5_g*JK(jQ>k|@-k^m)>zE)ZcXxtLml@f-6e2FH{)fj`(|~OQs17-iBmgeaNwYZ^mHG#Ik zq_@V_BndzvNNBLsnK|w8P{BbGJy@5t&nOjcSN>&UCF&Ws$rs}jd9`SUy~Lz5b{{X6{pp(F`Y{%8kblM)cto@tUmm@(j*=P8 zxOSzcnq8@>^0B}k#LU4T^3LAg3y1UUc-ZJEP0xMiTi@@&3-;j~^xrR)!gR?z5id;) zcn2sU8H(R{(Q)g5`4zCJ{as^S=GY{43^6e+?D$&?VicOM9DzZLI?8Cqb50YRszAfzL8%?5V|3Q8DEfGzQ*jgtG-PV^I1iaBXW8T$4De8j>4NVY9V;f04vemgh){`yx>ZqQemq0RpNdeTGdL99mg~xYThw>#3Li59c?&S_> zyzTIeIqn?1`j7i(qR<~91-?sl5BtRWDgy(ObbI4Qe~R`a6<($&X5SWyn$}9rH{Wmw zj|Y#&fq#nNXByUb-+KTSHeb`$(~)|o#mN-8L)o|if@WsIE5Y?#YnvH(<>CkcY;j$s ztzdL-9$mTlt*ltKZ!;=5k#VD$OvixzpWI`Mpkx~!Voc{FU`Jf0P}gqh4Lip^Nc;E@ zLQ|#H5$f%=j{p4ej}6;2yT%YgcL1s^+S#M{hi#IxfD3p%hFxkhV@%YNOpKf3J1|%r zg#IPQn%@^TomxSl9s=3)$^9-ehG~|A|MtkLP98ZSff_B0b*US627ELqtQy0cp z1#E+-99LTw$zYc0udZtk(PUj(-Svw@WwMY?aNnbU%&8 zKNt$QB`p=My3a8m(0%*?gLhHg4G7X8JJ`z5L%!T?GV$0X6Spls(_91W1VwO7S>qq> zy_V?hxVp-Eifqnm)xl3bL-6Svnyz17wP>}r0=vbj4L^yE$?XQgbnd!?r|5fg#5L79_>lPy0)JxA65XNf=0m3r>VE6A$YP9C1E($eXQFhJ$ zcEx5oE_KEzJpbeSGjrZwG#4i-O~GakEnHr&w(DMm`_j270`y^!bSm;E#$=bvv6NmEvaPRflaj0wYhvd@j}RQmNVc0KbC;vCF-m%gLvQ z>4_$E8X?t8yPxXe)D7c=drJ-j{1OOv9NJPs|9+TCYppI60!Z=G5* zDJ-?&O5WexF^G?{<2)mA9-J;be0m{JcJyKF!>bOvb;T=AR&z*5M;RL-hKYy0un+sZ zRQ}{nZ8yNdaVJ#OcF&W{I&6BXMV)+40S9!85)cYQ(|3jds@sAcHRF{HYvt5)ETmb6 z;p4Tt&@9^D-dY67wbOV^+-;QXI++ ze`-!55aluNMtw*HT{5r+Z={&+$-^wKZar*%TcXA9S9gkh9bk-4ZoB*w2Ds#{GmHJI zelO3$8b#3xHul?967r3x!?WfO7{GM>kgx`@Lj+0~;z##LE$@hMnQ%Zt*{qy4E^Khi z0j7J7?dP^v*2Ug;&yv6F@}iHwQ&iN)DI1E74(fFF*%1Lje!&|Me7X~XCSNqe8?246 zwzWf|-6qFGEP~^Nry7`sP}+@xN0E+#lD2~_VBmtoYHIWrL|7+iXdv2}KI3=4=F=>h zsgkVDw4i3Ha8%eMdtg*81k{GC=-tRYm5yey*lCjU^Y+tOT5SMk+^LwMv;U+Nhof_h zPp5IFnGKD(TO%-CJSo#L_hb+GMj|yW?T0y6+X42-uU^oOzGX*8F~|9QnUz=TVLl<^ zIUuUg4z0H2O2tCc=24%3_lii?qv*}e`uDdiZPBaZX)aHsud8GqLOz`6;8 zNA5z6H%3Gmt=Epf&*zG$m-E#!G-(D~O~eB?Su`gE6i_HA&}cydiKeHtp^6K$AmUYs zH51|Gr;uG$v3PIs2FTk{cxnQh_@vC^ocf{8m;!*XjU1@$F~V-%0PZg>*AJfTGg2LW z-UN)vVvKVWs9&}zi`3D(H}`HUt^#jyY{%j*GAn>RM2pK+B>xE{Ytt9cP<(eJ!IV6S zB4V2Odxobdp#d|oH}7=SZ`FM)ey$a~D9cg`sgUO}0^XK+6At9)V3DnsR?;Ql)PA6T z>4AdE)214zPmFxs!mW_Y*@`wz$LKug)ji`bxcixdE)X>CfWJmd z+zbwSF@57Ehscw2_l|(e)_2DvhGL8xMn_vEC&A6<#+-6o!ung$z;~R<9oiww?#+JNBtEl?xx{~MNO$8PC75u{&!A>^s6(wY( zl<nV9 zPr4c0KHkDR;CsH5B`XiBHg%DcbEB_7rBQHL`TQ|k>)XrgkuG{4mA?d4odvawh+2fzfwTH>PQX7t06I6;{*QXT6r#ED@&Aj;_{kanrfKrpi zKlgo3_DV_9aa|jNc+{a_GbFBN;FO4Gb#`=^`gSjT|62m?GR~}GSsruPemoMKGp=SG zD)cRGQ{*fl1vHrFyrY3`mSeG(^R?@Mfs#eE<8;z4wWvYKg0{hB1*oUd)y}Jsw^l}? zbA$w^YRIXJ8YmI(#DM*WncwRRJrzGj!wwfgsBA(5w~(BaJ~jKnJprcv!(;U;Mu z@~D(DoY#6ihOf74=vz<430)jOgMcpM_Me;vnX&BVuIku&_4EB?o)_nYvpAI4Mi*_W ztg@OOcD_E!T7m)_x~Yzw1D=*Z{MV-mq9=t){4LMB&h`4cup2hbI{ZCS@Tnu(l@X&G zj1Y1h$$qo;Mb|XV=FGPAG2m+{ z@ZgRCd^@Y$iz+{~M6aJSEXD9d07eRIhVjO#85P#NDtIvoXy$but7I%Z`T2Elmj-h? z+47U%q&<((Cfq8FGBdX3#zl-Ix$WV>-KNv)1!Py((Q>vAu;d2y%Y9HEAkzdy+()O( zYa$5{E0rU2euchR%B->v^8#HBc$C{EUa-`jvJ_Rcdr_|%MPOUZ=n4X zHS!*}*987|;$LnE?5&sP3ORw?fW1nI`9T?1TQ1$~CS%qp48+y|2Ci)weTG`PI7}S4o9^?erzmr$LwW{L}c*$zk zRnFE7P*f?42_x5;OOFQ?43`q*>2mTqhdj2gW?L5GlezYiCK^Kq>8x^llyUS<$-u<& zVYY<1kHc)B;rf@{Dw&HR8ImmVJ5d1gaOuRfCun#^+gPqu$<&EVK0&=gP7P%sI}_~( zRta9^A7`0y-t3o}EPTONL?*-~C}QzrA7VHbEaj`$r6$F)`Vk-2o)!W9l-B_fMbI~2 z1Bme80Ew%+I`%JPWIWAf$kE6(zB22xm0#B1oVVtUlG4+UbkuYwhGK9fA7%QVJXO~# z3x#aQ<#QfUv?Uo?Z#nUM7V>-vee_7B8niRlCUr?0E^RHm!$-2!gXE5rk1Rh2J6wdqL2>q(u0uD#xPpk_?l)^VGv0@Or;_|0sQmYBgmcq1ja` zurG}xnceGvKg(`&Ku$W0F!ZJE7$4hC6Hn}+2X%h!DZx@1g-*YIY?6WESMH_(`^ax5 zDw_{-n)<|CUBqsCptAP_atU0m!f5|Al?ffZzr$$$M8t)|j^=UJS1!_~2+seUVnF3H ziI}=<`Oxbham(U6galP#>NR?5eRwHFpUYp=Uf);d<;6;TW_5`E#X&2Mt-D!b+kBHt z(QR$$SB4R3+^s)vPvFR5!ZF`lFTQ#a{gsTc?Pj69BLa5%IWnAKvvzICDvqf->c!gp zf@-*+Jx)x4@YZ!Af*XdjJC6y{D(a+i(!E#0QV3i{yDP~-tO6e&-)9K%je5Ny?e21# zqCtq(BE5Hm;KtM~gQL0LYiZoS2$TrNCppfak8geSNPAW)Md4M2akc`bvu*xAp1y*u zs;=w$00%g9cY{cGgLId)bVzrnbc1wv3rKf&BOOYoNGL7c@5cN2z8}DK_FA*X9COS? z6kQL#qCa%6<|JzG?jDp;xoQ=u!)I<95LH(L`tE&%6Hf#*I>Zzq$S$7IlIV+j7XHR< z{NNkyg*=E89jJM@yvGEAa%%DBVEz%SAy>vbXL`wwD;|<$tWY%?5D-B+g4|4AbjlO7}EcgW4Ke|j`C%9 zruHp9G6)=Clp$x=iEm={J#nc?{0IaoSztP!f!4g>en;+ik0??6kTpxR%gnr&$gF;MBmR26#R>qC1n%b}ETa`~d<=pfeH$Y!$c!{PIkpDL(ddpVYm}KF?$^KGDX0 zeC}dPHP3~Fc0|n{U8&ScCN_=g1_^k{Eul1Kuj|!qy)il~c+>2RRFwIdR7}ojCD38x ztrpS{Ei*Y%OW6NnU_`L(!hH9q(}>a<@^}eIp&0%v*W}%eToTJk^wYpoPB71_6<{E6 zFreVvGW5Uk-IURH!qBN0dm+g>I#`uXxP^{rboZe&z0-Y1+TGl1gUzIv?id-uRRk?# z{S5sW-D>3(UNDSRq^A)UlO(15h6LlAIH9^C-~bVFd34Uc`dV|X=I0v{!?hxGQYCB- zWH~DGQrL8^N$Zs6M+0lovE8mZ=(rnHDaB)AEo9v{uVMI-_&B5(j5%A;?d+O-(qr@O z>_NthF4U+kE2PdPMLcYB4o75fRJTvT1G`YwWf+PnSjZjeqasRuuPfO{74Go$LGu$M z(y`hNXL}pS@FY33$UlRSHD6WS)=f7z8g{|rA-Rm1%8aLq;s#_T6O^4`ghZ<#dT zu?q{aU+YU3c#R((NBWu~r>^g?4iKRA5eIH287123~_@X=a759*-O3FNU3EU;cz|rzR_M~8Rr0$>xnF9EO~8`flM8BT zR1dX*e}E!={?Lg&aMO+_zHRaO19z`8cI|TaM^EGlp?}HZWjwdnQf?%ml4~=kJ(K(! zZ!!J+3{!$DsGiF*y4YNuuPoH6Abdx>ONf_PzwIslSV14CH-D^>cv$hNFnwDHv2k5C zg1!>j??uOgPib`RDz!mAST&!cQF0`U*yB3U>EZXa$X+arbe}k+JGt*CJUxk2HtFF@ z@vkb0Oc-Bx`qQzYk34(1N75x#h9>rD>p47o~`27^~Bg`vkDiyA%7i8!<_k z7Y>{VE@r581gGd1O+Ipm-_THYO2pDPj5J}VWtS$e5)=+HecTfzk1vKSfOMV38G9c2 z#N8!w^r#-HXj}Bc@YPW>wlW|5q?{%&au3!zRlKn(+Qr3>0LLP#u#}}|fsXkbU)Q7k z@%IJR7Ca6!x{I#wP%F(ub*SGKW~lBN<0<{vaHqb%Rv4(gQkp6{;nl(?WUI5iR zI_Q=HTx5(#vEoNgat+r!(`!@zL2tvZE@)L?za5t+xz)QroQ=>Bw&NWwY)cRy{A;l{ zDjF|XeJEwTKN~e$F8ek9$NAU}SHnFF4pJS6VZ7{Z6lq;`z92K5U;cl;zfMjshG6o`1h5=)~J@C)!6_jty4;4`>MEJ7Ex;G zf7hF}M4-b{*7fp4AlUqPw|{-0hC^Mh6{3tG;)p_#E~?kpHMjaRX^g|*l*M-c*IhRp4$r#mK867d1JdCwNp`qtJhr_}=e<+vnv9FV z(I2q&4ygp*;DGgFaDaHzOEgh$6$$J8Lnwlt!783$le`fHfkS{1VDI3mXXCZ+0?1u# zh7;LUF-NDJW5<^G#Lihme;*}G?w?g$JXOO1Uw=7mnV(IYuhvMm zW`_(?jDi!4@*|sH{@jiu^oTbmx;sur2NcFnVp!TS#c~3e z2D^{9QBFx);I&2Fz8sr?Gx>%d>ReUMESdnWgMS#ocG%Vt{^iJOz2(u#&F~0TG4%Sa z*jHro);;D=%BjSkcu#nMvNT5HcE%DotOd9!9DNV%+7I!uPyByX3J9{EjSTsMr%UTn z9HU&(0&lDeg^S32N+rVMHjNp^3n%q*Gq))vuWM14>aj6{vZ_9tOSQ^sX~Wq16zh}k zy+5O*|J1AS&w8~PWQ7_@0&u2`&qw%%_quaGg0kmK!MihiwBgL%X{!0}dYJy+_w(FH z|L019)^@G#UrUjXUC_xJj-TW?dMpqg=PGNSH%D*S>cGhya&ET)m>Q|?voNGSXAe~h z+g-zQ*z*-08ZS!Kk}+X(5TI0DE^9}aHr^4&g=h8u_Z5yzT1w=%!}s4TrlbvJaZ+*0 zEcrNH!>bFLtV`YjkHEBVGF%o8M0F(6c%U-oLjDDASn;m&C7BY!&#c4kP07(MeFTU_ z%3&?W(L);k`#3Bl-cCBgj0R+#U2+K;LDO5Wbf+ljEp8&?85U+rsUPv}LIOz2KNKzL$N>6DWM&-gRZ;Y35=1&wB?F z9PkB(6@xG*U_tV~kZ~<^Z9N_K|HfBdakn?7_+$WRE)CX?&JnR<`s< z!Wt^P&>R=!c9-K5riE9vDrX!G%P}$sGISs)pwCSORj^pzsN^Qy1bSB1%yKrkULCp* z2UN$^AXUsXDa)kL-bk}ozq;B(U*d|$SZ}i*NB+-aMVKkCIcO;4*ydBkdC76NkDRf7 zx5zc}eQ-`dt$}dt`1L|d)hYMIh7q(NCs^4N630k;ep@;SOT2T#WWQM}K0<7-!aur0 z`tK`Rlxl^6&WPM^{<1-{u;$B{93nk?0r3j*?hJj&FVRm0k6XsuI3pMxSY*`**Ws_; zX=Iaz1!d$7CC4?MDlJyxEKjv^-1fGx$G1i-u4N>v3OSR?k(A4DA~qQK0*zE*=>oES z1Qn;?(MQA#F*Le{WSSqBFt@M-F$1e(bhg!oVyiQDY~?L zXOs2RNmH`DgDf8$vGTE8pQ26#yJ%3+^a!VN>-MRp$~y$-yZnY|f2dVoX)$Wvgxx|D zKzbSlZaII}M10I*yy!gz|129^hYA8Jr^DMNC62DuJ;s_FH2sw4T%W2&<&-JP53uNZ z1HM6AU!zmS4TbUj7mkk&i<5q58tESVYi~eruahZ#sDlUglnbQdls3%ks5am2mR*EI zC*t+_!Uw3n@MDLs+DD6W;6*5Rr#QHU>>(+qw*+N%al?+~ic0m>2srVbGyQyl24}(g zsg)0ZMRPjw}^io(z^*OF=DB7Gnc6eH3jX?BL3Tu{xQJ1Q^CAoK;W`5Ap|W5&mwI-qO_MDxnVm>|aZkyTZrnba$K7GZeOg zVWqzEz)1N51F5ca;gG`)_X!=zkv{2SD+~4~SdT^giGmDTi-7X%j_(_RG}dz+25Pjy z@pg#C1@CCNaQA(7P8uNGy2q|&^j$#KD-`MBM@VeWnr{$@cLzx8=6rv2?+@_=GEjx+Sfb&#AkyxNMd z5!cF@ehoW&;0605pzE33!UC_l!fA>-di($Q&I~oCI6ti))#fg(m#vxbp8P!wx*kWx zSd-+} ziXKF9h{P^uXGhB2N$-#NRS>a{4%BBJxM@AlmS=oXE5Hc>wVtPndn^+-_>E;GMb0Ti zN)=B^M(l!uY|IDAuMKnxOc#R~D6R)Q=xN*_0k1f&wrNy9Y{MC&P}xN^I4Zuc0}Fks zj5Q_z=Z)7U1lYOSw>lz7be$a_%M@+oq zhrJfOG*IQ_GgnDbSCCHF&Jr?Lf6qU|}LZt1{2b zKohTVIDR{znJjZl5`Z^d?Wx^qylnXrm5F$)ACgbpKD;}F{1eexE@@~aDb}m=p`x{( zxE)CQfbp3OgfQPFgU+C!-)4ImCG|y_A`AJ44^iuM%@$8%XQNTqd3KLRXS{==*P;CZ z8%9gvqN$09&^I}?u70(W8bFLUzRZyF@T8->=keZ5kM$ECMew^f6TV7xpDz;&Oyv8Q zcROWHEDXR8+vU9P;_*H#<-%_vv6F#I@uB*5B6a&_GLU;}s|f-2pvCzh;BQ$DjDvl) z^PaVy=>0#Oo$A!-ToWDy^j~6 zZ3r)`E1yoYgJ24(Iev!rQWc-$Q zd4ce7G>%D{p)TQik_w0f{#Bv2x?gC>s|{$)PwvOG&;Q8SZ?4cTm`L*89Lw=NV6cRF zv1vtmA9Y09ee`7+g=X(71NHfrY2Zw3K;6QE;1(z5JM2-BGN1PA-bj{6=vRC7IyDj! zviEV>L=S!CNV*8^yH1;)AgI4s_!av<4yf7aywEW#6w|)ED1%>A7_qOrK3u;bDay4K z`ZrtYi$#mYy9Kf%eRo8pF4?5t^QB6!qUXCcY|ygjEE`f^ zYX&)qN2fDhPn~?^k*1`h^t&$#(1DYPIy!nUIavh1m_MGfAepeRpjj+dMjDxehigwy z6bHq{l8bZ1lgCmyO;9e-%Eckvrcd-DXD?ef@>E+quv>V5flhPR=(bLd0{duodua5( ze{EWZ)L8==QP(hDt-Q0IXRq~8_hHwaH&8###(k3B4E?TRv!5!(|0f^_WL+TsIg z54g8U$oY#NP9S$p+;$0gxKKsc;imz`Sf@KaduRT=nj034_;`WUw^O_r?4XKmo^`kd zB_Mz%by3&NT>fO9zSocKa^IKJ;AVIu3VP6kBrmKmCeX!)q8bI+NAtfQqIxsy`Gx}!g4h^~XuVZTl=-`aIU zQUnkZrr6tgIHEhSscRt8P`bu#F9MlgezQ#XSAE14#f9{RnFkS3fm_PQ{D=^H%0N9p zhqgm#qIlS>jbZD57owU;r;=h-kMOS-m>}d*lOosZ9M~NjY8!Ch$eiQL4~gX(B*F(R z_5Y$DsX`c3$e|JQI5Z+LH#1LfZx(<~+Bwvdp!HG1{`6p#v99RUS{0t0!KoCHxIQ?s z+spgNFEJ?G#6;ksP4=ytqL=98trQ*D^CBDeYk$JD<8|uX+UM$XoB~xtK5uI^&YNQc z= zOG7?J{Wfr%>wY|L;>dzV*D?KLt{+psiBPlx7)c8B9u{4fqk z82wDKl>UAMJ4o2Z2z_l%=C%vw3<-qHT~q3Y5N-JGWU1!7|B^umB`Fvk8p18kcNu*7 zGh$#oWF*^g14{rvSg>mj01+)8L?r4YO-67t|Bk|CJaaougg`!@+c_hMhy{IXPpM-g zMsCd{AzI&L+G1vNF@gQYa~s(~Nlels;xo-40veBgR&)m=QYrQ9!5e}@v4?xwNWaz^ zhwaZYBo5N_ex&Ab-y_J=p^c?rg0`%0Cpn@0h-@rufhNi>RlLI7b<=soV`8>p+kfM^ zs8S+lm=XY9Wle-7M+S1DbH##e`*@vohUe%(K=J^{cLJker+K$o&xkazT|tg(6=nbe zHXlL;F&S*fXP`fNY_utU=w^-?tZ9>5l8>vgC_c=``7#RSY9 zqWf!DRB9`}d76E9lWn9Tv+^}NY7?yW#re8Dnr`P$hzOo{d0B(pD#HIOQOsba!*)K( z(dW5k39FL)O068I-5D;H@DC|AhzMW?R7OOU91hcP%&`K1ipBkc4(&7Ezr=vC@8_KpqD^3m&CWQ(-d(l9&##Ecb}hX1G=_EJ}dPlF4-XANWjwAlCR%R|FdIJRzAjJHK=7ByPJ1IK=c%uXQ zC(VH_co1?udR5Za`we!^DDH&%M*E&=EwpkcjJkqu#ADBPhvOdoolMp>--K-aZE~ZT z`5Sa893jN(KO1jOdfeJ_P$$h*YmsFKNUqnvqpp^j_AH@??$tz?F0!N+`Br1Pt||8C zo+n$^cz#PCC&k=*gl6MEVc0=E>(96%DxB?K|F)(XZoh*eefsmgiYcQthqzoEN#)?N z=k}3S`^k!S#Aj;(d*a*ZOuVolmiWV90o?Ba6ubky3XWXhmRk1PrwNhU9~v>Ry)33c zFTH?2ZYc0{$7E&i@bZcA=g;H$a#h7o8ho4pNj71baQ~ds|Apbf*w656&yQfTRk#zL zVh z&uw{*)zvL8F)P_13SFTgy>6)0L{Es+ogdb}k=lUA!;80NU4G0Uhee8K^qnbs?F0=o z^;u~lfwi>IfV%b}`q)=UZ5kg^i3=>)9DD+pOK~qu<7m)aAjot7aP4ux!L${^In4*? zKBn?5p4DEP@mecS#U6lPz&(`(2%Azjl<2j;<9QZIZ#-(!aLVQ?0RJ|(QAe#}9&Jw{ z^YSd)y3fXQAAbLvSPP(mfK-Bnu20oLhl+NY(jdPdmEMt>jnIrHsY65Y*)btM&+B9B zQ$uykiB$(<_T7SH#2kvY+7Y9v`iAqHRg9X3p7fZ9^on1p{RgigKa%sEs}1qZ_?V3S zND2ORXH9fUYh*^UmsrVnl>lC6)RNTT`KtUeyod~SCAyz18nw@>_Kt=m^1bUe6}IW5 z9wri!am9${iKWSc(jC@W0`5oR1jCxcdb~-o0{uCN)^E^lWh_i>0*y?zZjuGVY z`{1;Mi@f7Gm!7A)=-BuuV>0PPB7TShDIR@-GWDCn6c+f|b$jVZR3Pz~AN+Mo2n>SL z4zMIR36nWyT~Qv>?lsl2-P4RP#>2dIi|f}%X{#RlYeCmA5$SOINrOhHNdJxG7)k=| znHZcO3#>`f_!+%?_U}&{*q5I~q8EIv@Uc`8Iw{rp4JC{W0<^qc%-1Wfd2?wbU`bWE zjWdE!mhvY2-qaK~O}0>Em?26 zSs%K>bY!aaCP0{g;BcKa-QpS7}x5SGYnL(YHmWJuP-@ z=0uFUjIT{Qsv*)W1f10otbF0KTS-fvs_!CEMu3-~IlH5+SbE@Xjc=`-1^gT6gdnE9Z_LL4*m>zmqF##;Ri*SYNRaF;)HY!pa&F2!D-yyr9&5l^E}A zp3I+i*P|vyh%tNtdUT((=JL^ zr`ShGbhuOJ--SEW8)nN5IOtFH)`XkqFS2|o+BteFhbXA;0}K$v^>hCK?BNvn(EM_Va%Ja}oxhH4tVGe=9!A6tIhGy= zWT8CcZ}L)h1yM=%^Kq$PUrWU`LRdNsFja4{+@S@CV>gXAH^p(;0-S~+e~%AK=Q#wF zM!$HMMy|^&x?lIReQQ((Hnd$E_<1Y`j9lN6UA@dNEky5xMe+RE@e+w?yt-IPnf?ZO z5AdoDM0YlVC|5O(4cDid@N7Xtn|(xRZFn$IyB#_c7peSOC+Tz>q2K8$zCpj89Z3~H z;KW!fzEWE)IffezG?z>yuhm^B6^4Ld|(7whrQ1o>*#r8dM7SE&Jf{r*G7I@#T{xOwvLMVEy2#i?}5z@Of6zgT3$Q zTeV3suQQ9#%M>DDnrgCFhl0EOMPrD9P}YUMc2l%^Y2dEl-!`SbUjRU+ls>B9)7|7) zjM{Xre4>T^m-T`Qa(ts^IB&3Ka+%BNpl}3lyMn9Vb$rg>ZzM2suk4j5F69cbjSKWi z_uC*6X(Ce$&qM);@*T0z+@e^x1qsyV&rsv*%+lm5EqZZ>+4s{Ww)(u}!ksD@%X;SbzMg1cjPhDYQxj0_F_VLIt5PexA85}kJrH!Zn9YdzZ^_90T_6CwDPn_hnc}Y7f=w}O~PzF zfFZdw#>w)OS~o^!$Eo=9OUvk!(OQ_p?H!+08VkJPfZJIvm&u`7^CwrP2K!r^+`5ao zA@r5>))aFcTVqqAnR-WPd-lR*S=#HgOgagf#M*0)ofynmE0eBoz>72kywJ*#E^NJE zwdTzfX{TWS6wn0U&;l;iFEOsNIJEP=6Y|cwF1z6(p7ot3n~P63M6xX+-FphCTY=M- z2Or)w-X#i>IW^j5g77R1?*yfG1})_F5teQLdaHmx#eEuIE|$k$^I#}C8-=NBEs8-P z*!0h~z~)#}8_UfZZ&?$JVFCqd<6(R@EZKUiCR26Xx{RUu{FvytM}*0-4>V!FGEhcC zzS`=4nZh~YNB^oi?EazK5FTA2PEo$v#SWQe7Vb6_nRSVuZ0{;*M^&pTUcWbpGG4?oo!`rgr4T%q79m9J4E{0DFm`q8pTwn^o&Cn6-- zuLp{^Cpa^AFXHQM*59mg^U$>{stuEzQ@(vUaY=JteP=rH>8}oecsbmhzz-BkQ^AH1bxi_8*Ug8sKqhoJ}ww{-emv1a0xjA+5%19b{ktf~osM?A^R=IG!L`ANojvj#`N+Cw# zS&D*RVIlS(QuMj8{akGS^oVnTfb6ssy*o4C;Sj6!H0jXSLfOD#-4tP;5CTxCrsaiM zpDTsad_LLkN1y20FEw3=quT($ivb(9+(jCy@t$> z{7vKLcLUzILXCj1EAUy;m8qgd?_tOXLlt_Ba1q-kFdapg=v!=c1scPFj)Ts9_uEXb zAciG`NZ*T}8X zCM{Tv`A7!*trdGS3zJJb-qm>{yt$KnplPE4_7)O3^8j| zz*zqYWD6-~>e#OK)8X?B!Ql7HZfD~`jDUS1mz?p|_3sGL&f}fo3mzJ*^b7W*lqcFP zCztNiJIJ9oY>JlAaKANn=t~^IJSc~qr@EA~EI5Z-qkn}Gsb$MaR}9=T*I3>WgW2Y3 zK}?16do6-R#5z~v)A?EQLW0jWg`5|S?PBsb2w^8PTnEUC6Uo3sb8$7eWRCcC47t$_ zgvKZ!eDHlZAZ06dNWY$krNl-PvB}HC}38BFAn0sguC}q`_)^8pcq~@6b8QD_*c)>CME#AuvgG}@4)yioZ&q?@) zvH@1)*)?wn8coCPs3PuZw6rJFU2X?hMXM_1 z#vMH$o`Gt1`l|gpm3uUCXMxDru|^+%pu2!Pej8}G1u>f0kb^e;5y3EzVsQA#+}*_O zIb@l$cH>MpEojGb$)>prUz@y?QSI90|1z9N=(W0eZu?)$8yoD68X~C8d?Lt2gH|Qu_w5OG!6`M2iYwjDT0@0=JEwp5#gfI; zK$N`V^@6z6@PwcMmbPW7*W}k~`i347n83(U{1A1$ujU{;S2RIDzdE-)>1<05m{SAI zTZvZHoVnTC{XE@^_0#%DJBCFl7jzRKvnUq_^*cV8IRkIJ+{ z>vx#JZ@&^haY3sNRcD7HTOM>!%QFh7cs5sBH;_vq#Y?Wsra5ooy--LOIg~o!RJvkf~?3+L^+yrRTlD)c3J`?Z+QXuVKSPUA@V` z$~MUw)+yH~r$oQzBAfldK~Hkz&48bf6XmKdJ4>dUM`wX0br?V$^7ua7F+;`Cq(r^l zFXNCTy!p+x#6Dw>I_*U$yxXt>Saf~z_#fhD(%fOk>roZGi}<%(jY+_y41bMx5$5o% z2QqE;viD(((f+u_d`g=p!`OGe8Il!Rs;W%7?p8rLVA}ZdBeFRQ@xPzX>eX6vszc^$ zgQx^@_1826^yxZ~XX{sUAo`I_g7?)A&buW-_S0*S+t>6zH~+AqVgv-P?;+e@GExed zpEpa)XJFAD+UO+o?0kl^!G0$Ul<(1hUM z3sr!ySh*nZ)rYPv4e5JiM5Q9()5-^niOO64s$mSgTMq5f&p51b~(93 zo16j4NDBtJTAHKb-ahI&&&$u^Q$LK5u`+BR@H+X;@>|eEqT45JFi*`8>zFa!WZ?M1%73(aZs=3x0yWsF!re7_80CxtPS!0i1Cb0&-21v9K=Yu$?Zg zxeG-7k2OpH(E6dqiffft@PX2(1fK;%tJdCsL*5_8xQ$!&KQjO@y1wu890J%3mX&3=U!ussB!RHb z5-trd^npp`fH+cr0Wh`)^>!|Wb}Lt$X2YK1}Um|pCF)ymjj-y3?T z{gnePVnAbcET?+4hCey^H?@eCMrc6C8i8{i_4k3|!aM32?0`_(^8=q*y$^~ zu5Q-I{<){(%}BBbQiP=N{iyO6z77dBEq$EWGZ>9YC|<*&nvmaodweXhAjhNk)P5M0 z$Y5*MGu`?3Gm;wU$cnW9uGz}e5vbT)@E-XP!EE_MM+mCRIbfXzzYa|f@%v-dHQoT^ z5qG)iLK? z@<+~+jYq6!tk9bV zv$!^|6?_0)(EikjV6Pn@fz5}5>IpRuFkC_cr0mGtQKhbkGS{RoZ{8SWlS+xir3w$Y z%P48RcncTZF3|Z8%^Jd;#*43kK*1tcuziXxZ?^GL!;897ykWE8x$EMh`xZI)Ceq{K z798=%U3fdSVgX-*R^O{Kg0VKy=YoLC)o!0gA-BbcJ}7pv8b%k6f^3g5wS=HBOv~Ec z+i(NX*)0FIS`Hu&zX>=|(@(|B!Rm2Z$!Gvvh=<(%^_ZXLg{-=d|Hz!c1M3U@XX z(;zZPzo*AT=E{$hE*D?Cyv8mScXjX2;$2b`**dMk-R;ay^*DDmIVC35#*5qq(|`i4 z&GhJ}knLdNI2^}+d|y=DCj1%2g2Q~Yd{)y*7pWiT4?L6%UPfLV9qY_ z>-e|DOK+7nVxi=TpE$pou(Kzl_q0C3^Rv(eFPo2z(jML z@=Dq8=)fWc9MQ^?m#TNqnPBzeb8DUf@qm*a1?Y5>qbEWXr<~nw*y@G#Jb5ZmRY3Q} zCxYnpqc)mEKihTfu1y0YJp1>zl^cXs-SK51t6RM42<;Sfj6z;nL*d^RS4VF?gcXe~ zE@jnY z+cJkUgxT{!!*N6gQqY+YTn{8=e}gGju!Y)Zq|&cXbIT8?E{wfyr)FQFJcQ)YZcdIHbp|?+2_3YXBBR z;`&bOy(+R;Z>^rYbMpM0O9Se6ca)w}3nm21t7m>eT|S$w!0INYX%oBo=?==)prIM& ztnslXt+mWFkpTTeJ+YV67AbJuf7%MZ7N#0CP8|bN3O(ur&sxluU;^N{h<+mT4&qhFiBH!o?D?mg5P{(DjBh}<+q*zT;!HM3vXzZnZjF;0Wkwp{m zKdjt85SK1-L+X?qjtWkPdOfxrGoDigtm?IzG?bhzeh$2awz z!cc85;)`&V6g77L(G#lX=K*>Xd_5E}pYRnLu7NQA$UZM~iupSzzetP(6*`DD-306}&89=q z#`mc_#nNpoz zMh05xCKw^6=hiB)JvS&JoABIRr{DEZ`n=$}Y5s-RXckW zGMBw^nFY}!R2w#s@Xcdp1)oL({ntsJvo04<5R1UuYsuPn`T8ww~#7EC(1~>o@wS+#uH09yrKQo7`AwIh=ABv7smP&iEP;*e3`& zVu6_AoIS|mXc12_oX#=C&>rEW=_b*^rPYI zJL-khUz!qfO4P@||0i8)15_m(qrPPY+RpzS%DNF)Xmi8k*ql*4;PW}83w?Vw&X2S# z;XudXtBGw7#eo3LMSEK{X|6tO+aclaSA!U|eTocXZ9UiKDRvS0H3h5rJldI4^9H1~ zV7ZM~xAGVBs%li8&SFuK-`Cdsw2BRD)$Sjb22x8S_KLLL#-!qxCLqQ<_O$vw8LeC` zi{?-MRXy0NL18(BZh$~*zkBr4flUlE*SA6PBMUV$;a8q_A%jQP6At3)-TKr}MRue1 zvK8fj@>xjlHotP#+#e&MUF>1YU43eHC@*)3P?GbolYP`T`-M&r|^n^M?N}tP>*jEx*Rv)^N&Glg(+3{>9`e z-f1xosagdmUkr9YMq>ZxrBPvr)63`E(Pg?E=u<&P?=bsmpq$4yiPwU73~|B?&l2kn zb3$ns#enSsiK$7oF?$`-nD8^t+7mo*y(2J6h1_;osNE=*mbaL&H`3vCr7D45WtB0# zc2D1v1i+>z2A)^Ol4XAqKScc{+Rij=E>us#b~?Yd>LJ({GrwB?6r3tJ&;}?HvI(7asPJR z2tD8K4A03fsoz#9<#RU6YQB99^VW~v&LaW3*Ww;pM5>e0Cm$_HOklh1a3QtOD@_`qyg7fa@J@yA{(GyVVbNHZ~Q$iK|UErNzQDHEK=P zKgtUwqi~BSGr1Tj3^$2{d~PiqI?ts{0;myt@-(3e3zmU#c&#lW*k8wOk{r-_z^p!7 ze(2fC$|}iO^%{f&&a!nqfvMB4tqcYii`D9VLm}} z(CP*D(TD;2uO8*WH+*LV>Y7{f8}d7p<~xGL6D7qMN74_k3!ga3z7ZokT*p5u(&Urg zDymNh6wfEqF!hrT!5rmo#stiTOZmqVpIP|c&YU$DMB1mPTFiM}-8p~yn z<7$~VCOx_Ya-J-JNFJ~XjtE}ws1ZNf30TIoZ(;vT@Vm;W#9L;@9vBT|(1;A7`l#(1o;q&K^~ zZ`SoY-0enjkY8saHF;G4n$T*rm^Ikutu~oxxBSIIATB2qUFOg< zs8p3Qd@qQ@dhxid?c4k|(McH#{U-9HuP~x5qF$lwKOa(AbRi&<(adDB zDn%e#E!PX?eY6Tof2{Bqb5G-LEEF~YR*w|@M>K+xb5-|3TC%S$V~u*wQZ3<0(6{zM zKBB3G+XLY}vn7gI0@&s;uv>eRSsa4i*Lq8(N_h(?#C{wA_)XQ3zo0c~ae{u;QS%@l z?swN-jHL+jT;uK~Z&)EeJX-%)S^J*;kk&laeS3HmxJva_Y?%X~X2JI~komcTeEhW- zyy4CRrg^_TDsdOcVD)9Z_01=l3}L}ynQcn3SwNNY_Kts2ZTJmSVj(LFfp?!`ko!a8 zB6Dg{+j#%-^kSmPt>kv~Wjbp+x%~WAUvQ7zY@uXbjMbq(5*#Y=?Ug)B*LU=Sh>e;Z z0HNCGve?*4AG&6RALjoPr7u_!noQ_)%ons}P>*n0+nK|J8-CEJZp$Lg^A5P##F8#? zMIOz6AQLABfd*bK0z{_Kj|sa(GaZQVi~qE-;r7(>6L+<}{Y ztcAF3A1(c0JD8;YxPljy(h6^9E7_tSgaOw68!qVkaIL@IXr&fI2U-Je_qIShyxZEs zVsYkYC!rtR*>79q@mL+3hOdA248lZePcR^1XqOXky{;&=J-8rzJ$U98oBcdj%Q{wd zjdBDN4Be%G?X*-unnAq|7vuTw4FJSPY`0q6=z$)~1*#Hx{Q2li$jE>qEc9A7cMZU9 zq~fVLGTgH~8SIjFD{mmzV*>f*u+4tGJ;!103jz>F9tShq(->4zmyK3hD7DdnhB+w% zr#pa$@@#PP~lFbi#JM((Vs8m5R z-!DhnXigx&H3CKQr7EX|9&7<(?D!`HB#d;K`yGUYhR;gwxK#TQr3`ApWzg^gR4&Dt zJVjeUWfT`|P!H(UsdVFLJRmYJ09;?-6}|TwBrho!RS#zHUijL`=aJGTz2I~45D2V! zZssnzCn4^U0aXS%P=uTay2eDO+15(t_o47wr8dWU6ifH*d_09XAtH#&{MHvc2<6yQ zafJMJX9M4sYtQdb3-FTI_ViZaLouGVgi0MSv7s3bEg{o?IzIKEIB2GJQ#D)WSkrxF zXdqzg%b!5s5066lYpK)g0+VQ-&?p7#*0(gU@}>CfVUs?e1MGLY#JLtbI&Ax(j?%aD z&)aZMjY|`}ji1*@eYJ|re_`VisV;L+s%l!E3`Whfz)p8zvwP3P(zfproG<`o2v{uy z{so8M^2|FrDW#(~7@|Jr}2%ohzs;Jz@V@pM2)!5~$Kw4y(;}{Gr+= zP_14UdtVKE{0=KSp>ScFYNA6nhnqKkEX6L-aicA&z<>e8Gt&qd%=25%V10a_OmW z8TEjWw3&nP2&`2ele9?I2kgD z%W1aR_sG+Nxp;kd0r1`#EH0ac%KcB52TrxJNpD91v#+(eU^5jo*L7d@VEfVFL-ky` z&8+<{h+cpDr+ISV>c{J8w(h}O*M6`Z?eIBm*M~7k<<5$zAEm1_(gYhS-viCw=eEpG z_Nzz2)6eYWy!OTd?b9sDg21K)@IC?L5<7-Opt+^0swx7+YGL97!?;#s*b}c(B<0qq zdfSC^rb`9kE-~T$SJiVDU*7-5j^DZbdD4Z@e>tzirS@^xY?^T(KQVf6ElRZKv2K6= zxi4W;e3=7?FTjsQrspCss5Y$D+nXpR^PacOCo_M;p^+Xf|0^c=eh#Z5(k3pVXG{TG z{)>a|S?=$Bw7L-hk?l>eQ05Po+bKVmK2nX0d*Udq6s|PE3OTMa4`Y{W#;0GN5jj=< zsQ>>Qrty|A6{$?RxCtyVC!nV_m+2HVLe7J6o(<2UP#k?{-Je#2W^ny1m4?3DfQ>}6 zV(+kjaxw~@dZwcM8Lh1!}-@J&(YCQruHl~5A})Yk!WT8-Hk1EZ_OQfdTbMs z(@i-a2p~}p8=+GD6ZgZzl{O)YXuG`3Spjb>Ld~XSiWc^YCHD1}xo_qQmV=w66&kM? zLhoQCsF)`*9rHdV#kba9FXoDhMajX&-MRCuf%d56-`({IVF|3!jRSIsQ*(O_jXAERZ!(9qieD+BIQhhs5TU+OolcsdV`2@9X90HNls2q=obH|4kNGS zpN*u*H~u4Egk1i483$af6|FYG3!!-{()-ctM8de(m{M-!SlRvl#OI)sFfK4VHvYyK zcNuI1;#l)C${8|A37j=3iNz{c&*)6lgwYT?Yom9ktMkAr?t-v^7@rj|A;pn zmnP&ViQ*I{?Y#drA3{rskreUIaW(G{g89+7xuHSH-NPgM-~PVD!mnRuxcoO6{!TPE zs1O{&U7&}ML_Rm?2W?mT2vv!T@1emY5z1JYn0nlLHN}h<3#xs;*1Jd~&P^zSN3njh z$;C53J_q$q*TVy@&_=#n1vJ0p5Lbbg*}%*Rf(rqYMm`oWt80*&Ssgd zbD+HyV23K%BH@V5F&RFfZmU5H(8GX$Ve@R6Uaf6qQqlyKao|lYtofj=IW}hRt}oGQMpu*j7WvzM zqQU;zN)_U9LRs@z{yYq9#y)E)uUyeL(iJZE8wDM`It6a3Xz*wM0-fNtV?f2|oAA;`$}DG|(<#x;?Qx*-b^jN- zCmVi=F`*$Hw)Z~aJmII#W#?ffIKClV%{WW4TTW~TzY#AabjrHnsP^`F@`rjeXjM=G z!Mgb{^k?{35^~WooTw7@jN;yo#t=zo!UbTDu>5wXjUqNQ58>k9ku+4{JrJhp*0Gpz z`>8`y& z1?RWLZ(G~rR=#7keOXf|E=ikmXh=v?V#J=ys_EQDHPuC@9DY7fi!PWM9QYN~KniYx{Y}^O?3Kddv zJrZYcPfq#Ni&@$OgE6ffddQzG`0v%ijGmY|mpnB}J{9Zd?@82Yl&awU=y)QH!TZ@j z-Wy2IRr8i7=JVYn;>fI^KV$oeEb_!9{laRSd83L!8?-!(r62E6yQXHzU}p#NKvPcO zRTheqK;R7svlR|Sy!dqIx)0`i>_!iv_i4?@vLh2ly1UXZKhBZHfAj3(Y|crtfD#7* zF2uzJ@DC+1v)~DScH+SZ!d1Hb{zl70x91gY)7v_MtG-)^t^N@mUx+yaLI2-*R+nzJ zUi^n(=X<;E4v%E)Xd6R3$S(|;vn5619cF71laNF#CyECL>>_YJt*!Zqz%67SB*5oz zsWacs&A=M*v9xq2uQSsal)U`2T!Qi|ZA$j%b zjxnyi-I?lU3U3DVdZrWZD3XPwb;eQ56W$r0z zl{sC^3TYt2@YQ2gibis9s!|hYF*4{iJ9SY}z?WCU!Io*MNw8AozUN~_^+1b-`|AWc&B-gjK`5N~NP>8OAF)mD9NybSi z3Hln`?yoAhN)X(vUPj?pK9!53Yh3PS4D|n$N-ViWXPtt3XQtd;DTgmt)rUm<&hLAe zzD}^7hS4eu*I}oYo3lC2w{%9#flm9Rl`yqRehaaZecETA29r@YSBwGZ8)SO{qr3Rt z0ew0l9uLD>VG9*C)f6CNf~jEiH|-m60yKh*t7Q)PZeg)=tK?GYuKkV0RqMqDn7ajU z7Y0vZy!asO5zflZDSS|mNE)HbMqRJZVFz%UsC34X2ldOikL)irt)L^pGYUx8uLDKw2GtCFEgIKriq&Q0kF|I#+MB7ZzUPO_XV_t)i3jLsCkOxjjYs|IwLI1<=Qxmx%SP-s9}xc-BzbR3 z9Wtj8Oxx#AI{)A{$!pMl``*)LQJa4JTj^wZPK7Dt--xQujw9Qv-coXXBFz(2R_^f6 zmarU8zmI-c^Z0kZNT?-6RKNgp^BnjaA$$Dli>!$i*`3sOi`N0?%Mo>qoqiA-23Z zeOXAB6}K{*i9!*U-qh5b#YZcb-76+XW!pGSte4=Hm{%G^-o2MP5o7M~2hA%%=ksbE zJu)JR_V+X~G3lLUk7hMFrTc^+c|&KWO0qxEn+t^-Fiox7>Uy`hWX+dErWplUn+nb)&4$gx3cI8x`S0HLEym0%k$xP zq?6y-u1_JTQaJbR?};-E(U3>Z$eYwP-zi!Q3lVq*4OmOj_Vn7DDV! zf-YFz%dMN%FijHlp((3vk<5YdUuFl+B~hkCpQhY-V1#|};r6Fe66Z^y+Mpvzw)_Kc zk?+iiIg-Qdbhx8?`+`elfiDGbHEg;J1WS6SC}&T?I2Fq=f`sLwqYesIa|y4H<~!2? z$5g37a7Q#^3)*6^lGHl%{aPuTF4r%5c-5hr2_KDE+KzC?{63O|*OhxW=FL{<&z~gu zt8@$rZn|&3PM_&+h>BX$J zD^=0l)pNcs=)Pcae<5y5m+7_W6F|Fyod$f8kQ^*-)guW<{Ib=SL%wkV=Tg-lS^bOE zUU&E;G269o35`15*7bc%%zKF+wH{86MX+#vicW?Z2SO$;5j$5?_d_^VZ_ZcZL*+q% z-v*tG>Q|!K?G?SZ26k_EZ;rr8XnB8Q`e%^4T?y9A(?b{WCne)|AKUu5TF~{LZ^U)$ zjB2Ll*t&$l&F zAZvz3$&f>t!rtQ>*gg|GDdpsR2Ukiqr_&c^UPaF|lWZ23v>7f@Jch73SdQ;}Pe@Sk zm41^Dw%Exo8}+rAiJiMR$nYn5Ldl0a-I(9LMt;5D9YUZ$x%DQARhH>H9uM-utRZbj zzCTNt;vZ6?z4ojsF`f5n_tTYd^Rumjz>8OT`~Tj_&mTsuwToiZTaR@`k{;oUxD|2j zZW_OmNB;^52{GCFK4?6~+-B_QBRg7EtnNP(dF<)Vj%T>jM|SrOd#8v=a_js9GqC~7dN_HVM?5PMrp5Z6=`5iE#5nlIg3jd@W>q6|7)->iiMZ=Wagz= z4|~q|(}^%3xxR$>p>U-amoIZ>A>jubGnJK<)Kzzrx%}|&-OKv0>>^ne{P&KX*F~x> z6_8>!2OAsp*ih}VMwfgPKLZ29p!)mwpY6PU2hbtkj9nU+YxzBFpCCX{*$2t~#`tsG zUN^t~(q&NQ*QvSHmp0j}8n)y=4A!lB!$og({^D}U^~yvyWi=BL&uKhuWAn=U_}m1@bn$M^a} z@3xE_gYwU(*-ZZyqoY-^qxPBRF$M6b7Uj>(V)yD7fEt%9d5BbfQSaGz(b|f+}{7!>_ngvr0 z*Uy^J*to+x?!yNMbM;P4B96btm~V)JUmwvJv;v1bbS{qF6xGB6yLh>MJLmN6tr`B6!*-9XdnqD)-85NEz& zSreh6u#K^|{#XLdTh$NqkVDOrw&D8t_}r9CCwmF=^ktZZR=7%X#>lNa)o=^`Je0F) zRRumrG>Xo-BLbCUdrxcH<3aN}g$t{Pjh{tai79VU!)vc7?+=g@+6BOhuhlxrU^~cL z0>UXVXGtjk8|`$2MCP)I7V}n+zQm~JYZ+J9*O-{_klq5j%dizSxB%7G(kOT8TSm^O zR~gR17<{i1LmH#W-o!2QY8*;by}&RQ_@+gxC!jAv{(Z_8#&Wuln2|BdNZ(#NUD26T z&3lwFK}bL_54|5?C!a3Qa(%s+Acy+?i_&fs@`PS_W^irwh=q8vm@EtM zxGfb`biDNEAC1K;L&d$Gwu0YJPu<14E`)SfY;5$U=VmxWt&Mk16xoxP_IGxg<>EHq ziG* z_n(elJkO1!u&^)%tg4bZ#@Wc`K9$1(w)C#SWQr`3#zQYkfJZ5R=^@pRv_F6TV76@N z737x*_~UC$m1vfBpNl(7_v7juAHEb3x$NH&$9VJz*CdrztHjAAyApZP6+Yx*F3r``N34ajz-Fy zQslepKPcxjB601z#CV2=hVn)>6pw`+vFU#|i;0N|@c*`fR$Y~Kny_o}JT;^pmqm41~${;KI|a~_1ol@x)r{%bY0QLEJ9#~r!Q4u~=Ng9qzwv5D<+ z_^|sNe2M|fvdm2f152kOFP=$_*`6PcXP9t{mbA_Wox(kugW4X{(n36 zy?2Rr{6_+f*SqK+gA~D-t#fR>X^k|swKX&zF#h$u9<-539j3AOMNk=>=q`#rz+DR5 z9L)>m#ys>^^m;BASt6ZqPDEdKFy^D4pKlZzw%|(I3G_WWA+>%Qm>ejU!K_JJn)CS+G#*%`)O#~Abq+m<^s(vZ zsl~1~{%yqxLl4mN&ix-6pRSiFji74G}|#(jmbPFdi&Kl*@S`2+oaQ{d-n-00Q;y3)B< zzdJC6!69-YM{zjyV6J+rFROQM=;}VqFL@$I_hAt95ic)qk2T+Lox27O{VCz*+?TY!V}rd@L~i z=`*{Dcl9^Se?QZEK4BWH*#RL6@1lZ&wMRk{;R5YMxkoMQ1B#d`U+Kle1y-ToiOB*M zMi=*!wz?JC)wKyM@DJ8=#QicDA*b(7%5cNC}6qev`wms@7Z;n)8d#!JQNJD;a}Ad+FhK&YyN-(hok^-`^j6w$LCeK=k$E}_BYL1f{dcK8(U?zP=CX%^h@9LXBYol4#ZcA%WIRQ-OkM+npfWA}y=ZB_l zE-pVCtE$HRQP&sI1>e4P`EO5hoB(DqG?>OMD0w<6LlrpuVB_B4sk1+fO^5v0oQOZ9 zk!wt-P_DIa!`01=Q_k;-CfDtL+nfVgW#&@z{?+x#h+l=y&n^hiGmZdapEaz+iN=85 zFS1K~FC#JWCDz^*iSXqRF5*24vlu7mVDz0Yk$1in+=)XqJxD?Fs|cS?2e#?y=o!>K z46g1}3alt8D=A?w6=}0uNL-uPNBvufp2(@4|L=<}F>j>~(ar-9&xwY%JUR)Hu{p_j zc5-qhC+9*cb|70+?PhChdve>HX$_z3Ei~7YNGHPpMyBUJz<89reY9~P72%OqOYhqF zAbmY@^oB_K&zrDCOJKZ=8gZBP1u}}b7uNy5Wb5tQx7?2!7}csNxJVd)MHDEz(G>9{ z+jz!#02CTNOy?gV7zQ7&B(%3wTF{SyIuC00_VJ;AH!2Z;JU!j@h#hgX;z=Q(>_$n!?f9&U((`s?A#UIk)OJ!U;IpEm#gTyd@`DnBMw9`a43 z|Hbm?5yLvJ&g**}Ttszr4EMz#Yp{cxrhi~}Hm!U5uvuCA}nd3~A1$)T%h?wm^ zxsqtsp<1u&Rt^aX32nKka`7E)f!##skHg5amX?-jo+|Y#RB9M_P132o_3rlWOZRWc zVt(n5upiQf{ni>$PhbFK(rO#XGloT}6JS=-nN@2ni%fBb-_I>BF5ciG!ad)$INyyP zmFt2|5g)=QfUc-x$2TYQ;&6T9=L>lUEkaU5)c!k*hNJ}6_+oSMX4_c z2?_oF%ql~_D-XDD;4%7-UQeh>rYkBa{Nd;4w^Z!Qm4sZ@_yC*S(jYB^rSlkL`>VZ> z7Az_ePqjF$KE>X*@FG{fPIDPKDv00K>WA2-6vDjJ>vz`#D3#;*(blr5=w@49+HcEDY56$6s4@Wm*>GE@`+;f$Ij=(M6ypR z`NLf{d7k**Qk&Njc6&{dciZv({K(XdURl3=@5(i_)@e2RRjLqFEjH4=xF2N1=Gd@> z1&9&wCos8M9|#NJ)B8QK+BfPz1af=WJJ2?{o5zC&07+1>@?v%%5`0P*lSW6?*jkO? zM9j-s1*W9Sm!uWt5zpF0d2Hv}Abw`PzP@v_ho#m?qx5hxMn-P=EBkuOC}`ONf4b5- zm63O63-Gd{Co;>KmwkYDPgNvSN*N?BF0MVuYV7fY%I8Z$dio5gA*yza1lF~)u9HX2 z=ctXrbt|o54Cm&xRDE9@Y;3wp_xh7k${8UQPn4FMvIH5B{l#7=Cv?YFCfht}CPqe7 zwagK}myY>bD>a8x@^sGYN?V1xWpzc$6zbR{tE~~TQ@$*2W{S}k*U-@L5IYY!x!E@r zSH&P3aS;mqOE{}~riE>pTxUj|qPi`a0ZfmY;PEv>Uw^-k>F-FgMB5l_g+qIg+4uky zkv+e_n(chU5l3j}3o^u-{c^!yU>QFiOfROvu_Fe`H!v~Vqr99%xSASy?Kp!m&}H&I z_VVfu;-Oo)z$3fRu?jieCON|plCB9U;psjtc&=imdA|%sIJYSqNuD#Kahh2--vBHj zmVmaKVM+LEU-Dn#9C4^Gv|;rJ4J_@;485o z?(2Iv2NGM4)4?2ZZC(;x@bH1VynL@4&DwIk92Ca*`e9%Ih%~)RRq<&XE-oUJu+ygA zs;k3_X_|{@YL~2UVH{Q( z5#iwrKIIk|uL13G#fq17%{`s?bEVBCRI@Y5iap9J=sfGDX#-dft@f^=fW_fPE-^vo z5$CKs$1iR|tV*2YlM@T7OV-(nAEo@Cof;41@Trer&=!H8oN?8k$Q?%#43T z!H3n9_$h;U_Z}vcPS^AXc5Qgbp6`UlT^hw6zuSK5{xuDzf>o!Zp^;s80>O(Re-|U6OxnNTa_DVmPcfpo09;?7nsU zhb`Bi+%Xi7IjaAoe8WNG^XJcd%f5)4?9bMg`V6K%D^c=mn`N(T3Hb2go93N|d)LWy zoLXsPaL{$nIghC7odld4UC%VSZ8(B}uxs6vc(U?QNNA{_4zIF)(L{FCr%xPeJ8yXY z6Tv?>qlQWFGwL>@1qE7swl`-&$zMB7)@&SC4XFDGqA$kI9SJRwC)sr;2LeNWzq>H( z6E`y-`pet7G9lqK2zAV8Gc9n zQ^+edKSu|N{V6#`iT1LaZUl;)ocyz4gunzoGn8X{OSIsQjA!AkMq?2by$WrEToL;& zhf{q3DTVUpPSd|C3`QN;1$ze5H*&9Ef1%ZdhEm-~w0Z zX2#*eTDABC{{VY*KXPF3DKWhGP~X4Gedi-La@gnlQYg2dkFtnfai&i2*&iaP&cmQ* z;fF~fhqa&^6_{~(u50Xr~ ze12#4@QafT(MhlKgt4}-U%zV9oOo!(&}9Ic?sL@lNI%If%z~z7{72idDDK`V(GUXG zIM>gBsaJm1z>lPDp9|Q8w+Dvw^kgdZV`Tw-2zPLDf_g(Sjg8^5Lz#j|nDhUy-bYP? z=mAH>m($+N{B}Jp*E)Y$3yZ1#%bYVVyMSo))47|Ie+$McG!5SUUlXY32x?{5{$oF7 z&e#LyJRZ{Lv){j~m%?@itE;Nk;~&0!q%sso+x0eP_kOASDcg?Op#a`0IKax{~YkP|K z+>FQ>qOF%Vp=CEWWDeX#ak(?Zu!k{+BHH;~F1*%^{P@}ya`{KjuV4SZoSd9Yi# z54hS}kc*|EjD5BHj?wgCZaVqGVkx^fDy2}$y$4GY%2aLX0SA2zn! zUA9OlEiLU(MucaPx0m6*Z>DTSZ7WESN-P0pszxBM;dt?Pa zcSL7=ndIKR)2ZA2&(4X^mw1%N@8A;4MIFPYFRsl36I3=tYE%`W-b#$Ne?>6$@jKQ+v{d$dhW zO(lVe%JyBN4b6iYbLGt4hPQFQu9NV`j~}fmIn2tp3b>B5-lwf)W3dq0-niC`1^z0*2)!jaqPY14h*!@=n*x3upKqLlu_bi4323zVWwVK)X%CXHeCMISIRSY;=HA2a-xG2L{8$M?S6|0gl7gQBU z<1lMY>%WkmcgI0!24A9hfYRt1k6^nC_8q&PN?<{@Yzsi#)ij%I z0*{RcmKGI7a02Y{lxdgPh=|@)5LuHoIxto7_VyNNqrl@E5NzkP8~fJ%M&H37sd^j) zPX7jV&5-L880W;4NKhXR@NfkMUYqV0z7klech(c^Etv-EP}X_ZerDZ$b+fahdD;VdW-)J z25ie+@^SE!1GeUf-rv7tUaS4Io@J9>C&NaJ5e$nunDf!Fj0)Umq%yZ*N;}9=b>yjH_sKrkY& zMumInBtS2sc0G19?mz>it=>zGF3tezV67}(UPLe9-=MFdF)21PWSDP%b3lAed^F~K z!%c?uLQ!U?Lz^f%&}^fK0@kMO+GOt_?&z(l`A*BncTP2s3sQI$;zb%z?>fs1w1!A$ z`8xvxgF3L)sdz;4zhW8DkdnT@g02*-d*I;WS~fH`K2vv9uFZ!3N>5LZ2YoHPQDXaD z2xo|iSE~_plgYepCW&ts!}@SrZqa3rOI zQV!!Rf#s{YpnYtW4Qd@&AI1i0`i!gI1t2h(0gSd}2{j?c2M^Aa==ZOb8+xIft?kEd z=dFCDZPTdf;g9$q5ekJHgP{+ucl~il!Xj@s(|RwX^4}|sq6@f#A>@$l8h!RgJRH7V z&8Fz(q|=CuP28%Mhw{qpzgyE@b%yU3B5Mvmn?K68F4s6vjh zUCP10(zyhL&#MADmJINe*CZIKdt8*_K7FKHpQ%*1_;h|G6YQ(mB@q?&hgIMJ|PGduZf8nZByg>vM3iScF03mS6DRl+XHv%gtjs=WTI*0YT9Ze;< zk7DTeZ76tWXWpZyPfLs;afRy}8%4_j4DZGn3dCh!zZPS7_;9m~`9B$<>APcWbTtbx zelFCo_gf%rf=6G~FEu|7b$K$M>a=llwspSG_-toGa1AN#@UB3qd;P}56hkYRPSWhf z!+k0CpWfx!?j0T?3lFe^7Hy4^ln7?MMub}2gR%n z=KhUm0g3PU#Kcf{cQ@ie&>H@C;L|UFWOkYL4R?Ssohl$-a!OD zO0b-=!WbSN-VZ-W_(oJYiaY%H$rHq<*x0rIQKYha#DQx&BZVlQjn!q|D|`9?PR`P& ziX$nEbEi{_Ya>X57fvQir*LN?$`%rUGM-9D3{qux9}yE1KY&UlpNNfJ7{@K8map?O zhc)Jfiqk-$*1Xj2S#g69uwQdDvAIeQjR`H4y9@iB3)|0+s(AWD1)}*YWaH-yMD1#fw;qEwGo)i#OGWc%h zMQ1v3-0d(jGHPFPGd}R+xYOXU(i5wgyhxGMUjyk&U>-vtZqlyksi*{v3Xz zo@~9Zl1R7B2w1PWyB*48LGaeC!b~oSCi%GGqr*dQTKZXMekcdnDWrKOAz2iq-+4bJ zc-)g(rN(|-e_1SZ_XB>MaB*)fDDk;juV1j+f_&FgSo9%d6b!&dHEXWSk3Os~UN~6sz5(jekdTR50-1>A`M`lQXajE4~V&CJp(ccau8P7=YM!F_uC%b`|%w9X(*Wl zh;z9CkBppc(Ir49^HWj1K_LO9*u}KFq^Sn=P>uw2)(nnCr-P!<2&JR z@jxc9fOhGv6z!GGbeXOc0^xVR9joR%yZX(Ro@33NlRoImikJVlv5Fd0*jr__Pp@`g z8bh@G3+^wZR_>!`GVnV_bsff$F*uy-8q26~fw?#Mqh#J~`FLi>q1%um+n>i>&kCeD{0hnc< z;^Vn1yV=vk!V*0G{`~pV2AGmEM;9A;^n0lhe73pN{;lsEYeFS~wq$J85|0WC!`7Zh zE-T2Fzsm(!=bJBK$FT&E^er=}Uw-8`!v%Ya9@ z6%7(s=(e1cTKRG5OiDUm5{JQ`E@C~vHK9$2W5*mR=Xs`?Z(nkB0NhuZF&A}S>sjj9 zp2r3gnAfe}h96@dItoh{v=vf#?*1K954jgo)~&8UBwA&dV9d585e*%e{dy^NHu3LI5Ed0@9$Y9>agR3+H6E*3P0Z zXgv1Eu(`Rpd-*T+BCNJw6nq3;L22V4nv4UWW=TDL%2>q-VC}?DZHz&zvO=mgCY64k zTKQJ8)2M8%T^uudeQdux`?%H~sHyu-*P(Rb)(50t0A-7W(H1ovvUr&m@A98aaQGD> zDzcP6nkSz3P5{SE8T9e9Hj~+Pz`{w6g&9@wm3=*FA?&nrhTtrLF`jTHQkB>8)BpGf zc|<-rpT(9jYwh@gs6!T+Kyw~3iH`R(&4$#qs*ecU{Pu+=4?^-NIVEwF&;WXVyUL{rmygC7Mme9kOIx{RWJshjD z7&O?)&+bP);^uZLF0eASL&0%%3K|ycY=$l6k=Oj@hg_0pD;g_R|( zl3#_0&-t#CaqHENyxH;M)q`cEcV66oTET%~{?zxzJe{?~WVH&HzJBD8Mp?71ER(dt zl}ts8gZ_@Iv-983O1Js}d0s=0-!vWxksYmSp4ihm0oQMFx|aRj9u5)zBfCX8kf zdRw~Vx18%klB=W6HSHYLhz?q}ep;@K#SFj+Nh1*^liBAr z{ZBs{-yT1_dJwI8Ehtz|Vlhw{vZS@OwuX?4e()^D7SAMEEZM0HzaxT_s|P$NFtX0_ zioD`me!l)Uie{;|k)Le_@XchOM#zY^SY`qBng+51g!rMEJiNSw6_fo3pxJ*91ae4E zEUj_nl2O(P$jQus_!s))_`U^2ApqT1EXi!l>>eOM2Dyho+}*Z=#@LsVeSFf=KlTCO z)JE-Q()1Dra&jJ-Ex(4Qri@(gf1je&lXlE*4gl-@1A<@gTBLNmz0YYq8l}Met_LAL z{&`;pM$EzLF>pbKbr^|k5!+Kur>Cbx3JgJS^=0PtpYu}k&0|F0x`NC14aZO}g6-&= zLQ20tkwDb-@+_JDRH1lUN^V96zVd-_#bL8P|g2<&x(UD2It(ee#D-Y zfQaT{kQXX|B9n8CXxnyMWkgl%<*mh`AI{4~d^Z22LO1MvtjxD}HS3#HMGT8G`~JgN zJ2Lk6WuZ{0Qy_L_D}l_q=k@g~+A@8+?-N@Y`M+AaqR5sM0OZ>OooGxs<0B^XUOt=g zRplr0Gkv$|yk=&8{@T#lJ3m3OkOPBmz0asqN{BkH#+ciY$A}3~cWdPF#KDl+xxAuW zphcQ+qXRkB$ z{wIA!#QSiG3YaLA%7LkPSO8KWUH^xxYEfP~^R%Vy-jH8eN%F({?_KIQw$*#1yvlvO z29K?p0&2=~4{8 z-Yk7d6T1U^ojHVF^P!&TF)y&uJAhZ|f*ZOV)U{Iv_OZbtj7{2Ce=IdGge zj-K|)UyNUBi%TT>{v4um4RL<|ejJ;MDvg}5E4|FY=u=RtW2v7BCl(&DyDi9mNGmD* z$;batlpKf6f{_buM0)31q?FL~#pR_$1>a`B$MVLIB3)cdjpgSo zi>8@e@vRAuY?=&IP>2!_Xjjtk{jRpR>21*)M=w9;YPbhHTqlrpS3?3g>7Wol+qEI~ z3Eusz@y>(Ww)ofG_IhhJ4ar5v#cjiOI@H&enh)^`cN&_$l^xHoJDqXUXChJabC5#1 zr5YZ2o1X_(4_m-9pkY%e2j*PNmZWDvWW8&9XTTA(<&6vW4-44?f*Smt!QXyCvn z#Uz~n_)JDG{11oOLob4u-rvE+0Y4mtug1MJ%ax<$uiH@ex)yz8l5<6~S^8F`Nv2_s159Ue(LALcN zpO%so5`hp$bYPL3YWhLMOV$h1j01!@fp8#eGk&5q87<7ib3E82@sR0OH_W9;8;)%) z8YQ(|+Ru&{XNwv>%`{HY_nREc#Zz@MyxhLzW^@uf*a5q3AMJN`r%1dSSc{z0rgsSm zcfNrbCg;7hhFi{_L!1-WoGxtfR_ROVU$d1?6V(S%#s_Mvu!fY;}9Rc&9 zf9xQIAcF9q7@9O)!y-&wgY*cr6%<7QyntnE;F&19t2@ z%Y+7HoB_&`C;igezNRzAel%PK`D(^yHiP&@$fXZ<0?`ddaDNG}l*PabaQd)(!ITEK z^q?LnTJP;d+i^opK@^yNZgx{(E<7Sa{%aDd?umLoSx~8qp|RJ_^fS8bYx|m{T&{H1 zgUwBEx`V5-j9#;TtW3b&S=-v$CRX$A7zLUFQ&pr7_jTQMPM_oNa{c-?T8FWJOj=h= zOB*0eaA+56z0xk&z2j6eR3WH~4XyYp@N7~1Fn;?U>3aKbaZatO zo$7V8agjkzLc*wV0Q}|W{{q4>JV1XP^@mF42HL~7)-+QzEe5>&DxQl1MLTWboL@(a_gF5NLTV3hv73O(CaBU}RSs zWh5t`B0vbs(Xh@wy*v1=90WLAyF9j&#e89TRKVM*7ZMfK*7q|kzM=tF5kDA8AF9)Q z@-2U#@!`XBv3=d_B5uVO@*4q$U@b(LHIx%nns7j*e{HSy5eP}VEB`UV6BUC&7da}* z0CTPJG)EZd==`k*6ZZZ~=~g*+lOibUiY3!CqDGFteZGnn;YOUEX0ce^7?oxNQhT09 z-{LeRvNVfFvaVIGm#$r|S0IEvn7jdMXeQk6K->*j&E_9No^CL+grTNE;4?d%1@*i* zP4j6>`;0lyUNe|%XMDZj%?>%QHR%jV2)Mb7{cJ(Pb0QO4{cS96L{47bD>WS>=J9{g zBGVhEcQXmByvV@I;|$6t8v_DNzqOv?w^NzVo-V^x9rT2bin>WQ+K>)TzXl@+W)=bE zcj9u#&nNu=PcV$+PlXC6XGcrEQBjGjT;ue_KamRBOZ~Pl$=HF+A zV8eJB{+a<=m;UzIqsmU8!a7E{JNW4}o0g|iDh5qn?CwsN&n?ZGR9sp&VGCjRF1t=f z;lQ5cTjG%x8VS`2Hz(?%Zjv zuC6ww7WK40xlOqiH!kXgfG|sIKA9SZvU|bsjcSU)`GbQmQkk4$%G4G8zCi~^& zyoX}J4$eLWmYBOvt8Z;P_IzL)wCtH-y<_(B?f9qsA^j|PsjCsN5dEW*Cp=Rc<-^eX z0E@q$Tu}v0TR@9yuEdtD(=T~QEski={!kw$D!2iIEO(pA#cY$BY<7){JYD9sy#OM8f^XJhB zk@>8oB)0{_af?%K4vrrMTBYLO-kH{@dxonhg5RQqS21FdGg1D5LcM+2(Z-Q!+WNko6^J89c3fzd6SGy^h>5IJogBw+I=2On z3h#p~VyoRiGRKYOaMlbSv&krU{6)SNy!U1cLPAuVK?id9+-cdKQ@iIz3_7GLrOuL` zOBr&+j1u$jhD4H?B1SSp{aSm+@o>o>a-E-ORxMJ}pL^Om_&-^uH*L~%c0S~p`Ln6# z5qYH-zT|nEuqOnwm;(Nj#qT)4+Xo@dpJS`cQ?3QK5Xdq&E$aUmBsVAg_Xh4;&Uy{26#(93pWW7q<^_ zSYlU32O90+iSdxf`MF6KUOctI0xn)KU?7~7SVeTA8e~*d_2S&z2l+r;i6cj4NQkXN?N?BKyUVA(T1WM?oRBIi3t_nnIV!XC^iduP!iCfWRir!=sWkZ)g!^ zceJ;bhE!HY@nTSzTSI!GemXzFZbBc7i3UY?Q#gc>&7;C57hvdD>e)IpVzID~!_H9F zL&Z4<+`Bbb^KV_@DWkY!=%ptj@1;4*54bCTZ=#gPl)R?Yz2^<77;Nr9yf6f>|Jbo1 z`wU3x&zKl;k@0Yb89|@#W|cMB-|V~>e*SE~VS84XGs;x9`}KBn`2#^38`t>mO`Ke? z(GqsQ^@>oD!Vs~q+qDV1rw&p5m015mn zm(q_A9%V~K5~aIrx#^bs$C)>sr=@jb2Tgq555etSXt+Zawws`;is_!2fs zSE4iyOOB8IGY{+Z$Q~fBbS(${c!8N9*0K6_Lnx9OC`il2jVLU?qi; zFf+JSxx++3MXM%NEzc8?R!kL_l3Q{P5`ZHP$DZN6Xv|9QTQTuch(IBl^U+dMQ+xZ4 zQpDo07#I^HgyK7jcPK7CT^O&Gf?9Z_Iww<*oLgMtBgC28?AsX7AZ8sh;kHLI-Zuz7 zztvi+CC9@QB8Aj^C?Tw|wsTr2C$*8}B< z+DYhLjsU>@bD8vpaemBmt+@9*pB5M$v;q-S#&E~L%j*da7RVj?5xDcaIyyQ*ch|mh zh=^25oEoLH)9^#{QJK6siSqK79|`R-SFS&M^Z@WeR}I6aV%Aku)9gX6zVbXN#>uyw%&_BpBK{6%WI zq!NP~&KQ zG7}RMOGkWN7>E^LMW&d$^GSUCg{1etSp@U@5mlLIKXd-v|GL)l~tYu-bU8zg!KxC#cSq)(d{|B4rW#Ky=tPKR|WjtbVyqhnLL?fV{% zi!wXe)#uFum6u@bernkM=}!}?04@47;3=F{ylTBJ3P~}1NHH4Oe68aASETh?=lU@a zf_(a+KnaK|&)GCM1XUNpDP6=I{S>ny->8=8-s!v}(;>F1a2d^nKD%=!w1o}da1L** zMU?!){8%8B*Rxme@A}c!Pb|Q8SroPYmN*mDfUlL))k)OJNN$;q{v?ez>eGGrApQHv z*X+!Ea^tOopdbc=f%eEuG9#}$yIAac zl*ZENxtkjmk~`VCevDvL3$92DV_`^Dc<&JXajaxRte?PX?nr^B-og1mF0*(eFzoFb4mgb@uITBT} z)%|&y%BKn_ z&GUpv+RaGP0OFal(>xmoI-mD}Vgy(2(a^x{rrHWws(- zavHt(PD_(41wmfF?`%&Sck&=dr16p&*SL-+x5_ch58K&0`GyNBHy?^g;?o9f*H0D_ zG{+sjvXYU)5|JUs<7aU--HDhpqAPyHGZAty*CZIdsw!GG#bOuCXcacT3J*D9sABvc z1+&*OWLlE8de{A2uP*R{%W{iveE)u%$beF zjGV85$ql&COzRoB!=or>9&uL&1`3h^+N8!Qu%p|2WoxSk_r*p z;tlDFeN!~bd;NO%ECyo`+5bUFp*~issX~58A}sV9%o=kyn?7V`XKSRy>vZdXCQxP1 zLq-LV`G@M&ef^D$Vb5=;cH6BXHLEEV;<8y}xH@dd4y{z5yfi9-F8;h0-kFR1cdb<= zhuAnt$h~K3!QcY&7vkmR?bA^iy~7T6({o9`1|F4qag9f<1ebf!uj2|PMhC;CR8&+j zf~Tc?@Q@oly*fLawcHX~U2j`CQ&GnJtSRfG z{r%%W?Gr#N-)IFUSbFGnVdkxCEv>C1Z0b%@2(y&zBt~1Vr)A2E%gt=pJkmU}U=r++ zT~iR{+19!xNY+dcpSQ4BwlIx(;;SrTJUsc1aNgjUnzdMjl}A{z3y9J?Mjfc5?DC~j zx~$9)lG@Y?l7D^vd|+6Za2-?R%I`fdc`~3^XLXTd4q=R^Hi$nNHl8POn=Yu%HHAY} zAN2bMM(dMDE@&ev-yc-6XL6f<=9FQ(AFUp~VdbMVfDxWJJ1^aw--95`W?96*W1j5c z{{Hd^c=4FQDA0EIroo}gMDpMlTtWZAz6NDvR8&;*eE11`{jR6&R2Q)nqD|)- z34|poIWch|`#+G6uJ1_rtbt+l-3{#LeZMlSrP;yF)3XG(Bv+@*-0ZB&Bl&!Nub>mU z^v#AdTX;b!2A^k50sSb8O&T;7^VvdOvA!#TOO2apm*3+HX(Ue{mGv#Keg_?08lX1I3C#!O5 z+%=GNPM7-JD9773N!tOAVuul<`19E1vsIP!>fG8qq{{m2_9mH*xFHfEsG*lew;9Hl z%~acx`TUx3g2;5o#G2SqCSL6?fKU`RDm<^nK$6tWlcR{c6HzUu`=52B(oF0q5$~Gz z^bO3Q10Bny#Hdn_yGMP z#c;3haM!i^6wg#veoD=S>=d=rDl+^kGPGT7IUtu>6>yb*?DJ?|kG}0M_q!Q;aIw%^ zqFvSaQw@vpaj`$%lJ3m-s}h{t4f_e`6C_8rNofB*UFhiOlJRjwe7TpFJ8vqUE#;Cl zPUpL}?BT7T!atVGB{>pKl8XEeXQb8g6D(2%e=RHp>pC^lZBrKWC9BQna{4@%@I(DL z)|Z}1`_3lpqUdLG*vpbljW5i!;m(XobohORtA(Z=Hr@rvR>j51#|^&?k3Gp0I?pwX zat`B;>dJJf`7%Cd`rTHrT?>~KFK6>uPMc?S(yp+&58u*&j{s*?GKFm=>w#efhd#t| z)D3@{^c%*tRd8#w3S{}mVvOVc(3)Pi+GXTo_WUm`A{2($gi)+mr9OAugccB%9iN|hH-V|tu=rXP6BBcXEOCD_SBtH@ z#{)GSD&0I0-|vQSoO@M83?+$#M%Hy*cP;}REr;a`KqCLDv}|nf54bvK#QKFjCV!UE zqUfaZ$&|#NlL0sB$%t zQq@}pJvPN*t_@B?3U^B!zK7c1={VxaHfO%b#^(GIV?R+-nxdv&7B9&n;r4dB_vXyx z#ZlM4hKTsIC8;m(<(46*W+?{Bj))XaWk`qJevfFej#WvwJ_DnI7{960@&!`AKRW`A zgL@4XkRIR2Gp?*VcKJefZT%!$Rj7-^g#P#d+Cq04nG$#7CCiYM$MS5n*J8wxMdILN zG7FlY79`TD##Nx^WfbI7CaLkK?1g-sCPV~&XBxi13}&8PVa405HmA>??fDW>wl6Rp zCAq>zOM5kUX^H@a{+O}+k8eNR2mclKLXG^RE6{}BZyS_OlX-iY;CTjA!x2a{ z69~p&6b9^qsbWtrFLK6bDVrDaY!6tl^h&a)8FW5C{aigX`+>HZZeYy zfpzvgi4abZ1H%HzTQAAojSLO>rn_|WWWiqR3>WLVB<)N`#gV*t7Ac(>Xtsz!@|>yv z`0->BgE`uUyyFUt6lqU2IjD=Tl9G0dz?4|5RkMd}fK!T-pPxU{JB0vf>A~l6P+?Q6 zTCQ1JxH3nbh9bK}w&|!fLzh&T(B6{=4=&|bj0cfU%AyX#EUc~N<{*OzK>emi+t|ag zAs7exK+Lam)MUruIW$n3@LjcRxuY2T1L0kUcgQ&UsMy#woZy%Y!N9@r_m8_A^lL^V zFC&gkPBsfd@64%9<9Q|8^B_-b37(Fu0#$;!!>wI?oZs-1<{;t^V7^F;^py=elnp#+ zI|~F4dT40Kik_BMta@jk^XeS^y7@9p9`r~z8zF-CzVNR8$O$2cpy4UiTCbEEVM;>y zkT+Yn3m4r2)L;YNg$}R~d0*G>c*d^%l&A)tTVC#){O(5&lcosp(y`Fi+B3%p9ycyQnF5#fGbCks$jY8XL#b^-t@vctDmTa5&s3#K#W*@;8E}r+X2$nZfqfR^GJd zs8Kor9x3&3e1AL%69|7+|BV)nx{d_c^fx&gl6eic{XW1fF7rGkWndmW>ZXRT|9CC! z;yA^}KYA9oK?#|(?$2O3?Bt4H*u#uED_L3nT!r}zKZt&BhF(aARs4Xgp3{)*(Q6TV z-QJ*a`v?i>ZqYPEhbF2x!#4 zcW>j=smEj&!K6Q0<$G)*s|9JRC zq&%dl%%LsszYPw4K<0@XNZIArw+@OS1~IU`4{II9hC&}YF$pWY2Ag1T)<>$j4+_10 z1qU6xo9`ayVhezMkqZ?+;22lEZhfAhCjReVTG-y-FZdbKva9)9W5P^e3bqTwEblJF zyKwtkj)d0>mf@KruqfUY^+}})?3-%$y5LtwxxWaB_0Sl>G8FOL_5Pj*gXe8t$`>FSzng_~&%{!jXOly^!Cf z24ee_cqu~E6Wk%`071NO1c8%8twYf?w;zy;ECLoHx4e5u49k|m7H#m~#QgWyg0zgM+RKpFSN`3e&Q+81)#a{G&LE2wRFn zsd*k9-SbDRXByW-E+salS3c)kYOl6^iKT1-V1pHy0~YEKIl|hH-5WXGLrzROGi7t5 z{MYUwD*{8@VDyg%EUv4kx6BNyqh9>&cg5o?<&!XvX>2ilA_T=)Ufld~uXqd0Vr1T<+A*N9_2R zHj}E;ED@q;_!V^FQXtjQ-hSw|ofDo9bH7-KLqBC%MLXxt9kM~^=Xd005Rd3-w{yF4 zaeAG78fyb>( z{-R}JVR<@>X)bd34!eQ*sy}0FyF%DpA>yr^l%!-v6L<)UEYjasO9iWDC2JX1#ovc) zzDw)dxd4!gR+WB&J?=KxcMPtgCuRRQCBDXny3mB&7BzBTv@I5c4(%JFA)neV0MzB9 zOTj;FNoNE=PZ<<`X)T^sZ@-RS4nC_;aK6<>@6MghUU<}4_Dh;7dN{-Uucd9MY0SdH z4twwf(e;2~m6ly}Uc&oy(s`_. +Easily take your existing ``LightningModule``, and use it with Ray SGD's ``TorchTrainer`` to take advantage of all of Ray SGD's distributed training features with minimal code changes. + +.. tip:: This LightningModule integration is currently under active development. If you encounter any bugs, please raise an issue on `Github `_! + +.. note:: Not all Pytorch Lightning features are supported. A full list of unsupported model hooks is listed down :ref:`below `. Please post any feature requests on `Github `_ and we will get to it shortly! + +.. contents:: + :local: + +Quick Start +----------- +Step 1: Define your ``LightningModule`` just like how you would with Pytorch Lightning. + +.. code-block:: python + + from pytorch_lightning.core.lightning import LightningModule + + class MyLightningModule(LightningModule): + ... + +Step 2: Use the ``TrainingOperator.from_ptl`` method to convert the ``LightningModule`` to a Ray SGD compatible ``LightningOperator``. + +.. code-block:: python + + from ray.util.sgd.torch import TrainingOperator + + MyLightningOperator = TrainingOperator.from_ptl(MyLightningModule) + +Step 3: Use the Operator with Ray SGD's ``TorchTrainer``, just like how you would normally. See :ref:`torch-guide` for a more full guide on ``TorchTrainer``. + +.. code-block:: python + + import ray + from ray.util.sgd.torch import TorchTrainer + + ray.init() + trainer = TorchTrainer(training_operator_cls=MyLightningOperator, num_workers=4, use_gpu=True) + train_stats = trainer.train() + +And that's it! For a more comprehensive guide, see the MNIST tutorial :ref:`below `. + +.. _ptl-mnist: + +MNIST Tutorial +-------------- +In this walkthrough we will go through how to train an MNIST classifier with Pytorch Lightning's ``LightningModule`` and Ray SGD. + +We will follow `this tutorial from the PyTorch Lightning documentation +`_ for specifying our MNIST LightningModule. + +Setup / Imports +~~~~~~~~~~~~~~~ +Let's start with some basic imports: + +.. literalinclude:: /../../python/ray/util/sgd/torch/examples/pytorch-lightning/mnist-ptl.py + :language: python + :start-after: __import_begin__ + :end-before: __import_end__ + +Most of these imports are needed for building our Pytorch model and training components. +Only a few additional imports are needed for Ray and Pytorch Lightning. + +MNIST LightningModule +~~~~~~~~~~~~~~~~~~~~~ +We now define our Pytorch Lightning ``LightningModule``: + +.. literalinclude:: /../../python/ray/util/sgd/torch/examples/pytorch-lightning/mnist-ptl.py + :language: python + :start-after: __ptl_begin__ + :end-before: __ptl_end__ + +This is the same code that would normally be used in Pytorch Lightning, and is taken directly from `this PTL guide `_. +The only difference here is that the ``__init__`` method can optionally take in a ``config`` argument, +as a way to pass in hyperparameters to your model, optimizer, or schedulers. The ``config`` will be passed in directly from +the TorchTrainer. Or if using Ray SGD in conjunction with Tune (:ref:`raysgd-tune`), it will come directly from the config in your +``tune.run`` call. + +Training with Ray SGD +~~~~~~~~~~~~~~~~~~~~~ +We now can define our training function using our LitMNIST module and Ray SGD. + +.. literalinclude:: /../../python/ray/util/sgd/torch/examples/pytorch-lightning/mnist-ptl.py + :language: python + :start-after: __train_begin__ + :end-before: __train_end__ + +With just a single ``from_ptl`` call, we can convert our LightningModule to a ``TrainingOperator`` class that's compatible +with Ray SGD. Now we can take full advantage of all of Ray SGD's distributed trainign features without having to rewrite our existing +LightningModule. + +The last thing to do is initialize Ray, and run our training function! + +.. code-block:: python + + # Use ray.init(address="auto") if running on a Ray cluster. + ray.init() + train_mnist(num_workers=32, use_gpu=True, num_epochs=5) + +.. _ptl-unsupported-features: + +Unsupported Features +-------------------- +This integration is currently under active development, so not all Pytorch Lightning features are supported. +Please post any feature requests on `Github +`_ and we will get to it shortly! + +A list of unsupported model hooks (as of v1.0.0) is as follows: +``test_dataloader``, ``on_test_batch_start``, ``on_test_epoch_start``, ``on_test_batch_end``, ``on_test_epoch_start``, +``get_progress_bar_dict``, ``on_fit_end``, ``on_pretrain_routine_end``, ``manual_backward``, ``tbtt_split_batch``. diff --git a/doc/source/raysgd/raysgd_pytorch.rst b/doc/source/raysgd/raysgd_pytorch.rst index 093a24a79..17e79896d 100644 --- a/doc/source/raysgd/raysgd_pytorch.rst +++ b/doc/source/raysgd/raysgd_pytorch.rst @@ -1,3 +1,5 @@ +.. _torch-guide: + Distributed PyTorch =================== @@ -467,7 +469,6 @@ After connecting, you can scale up the number of workers seamlessly across multi trainer.train() model = trainer.get_model() - Advanced: Fault Tolerance ------------------------- diff --git a/doc/source/raysgd/raysgd_ref.rst b/doc/source/raysgd/raysgd_ref.rst index 692571a36..0f72ab53f 100644 --- a/doc/source/raysgd/raysgd_ref.rst +++ b/doc/source/raysgd/raysgd_ref.rst @@ -1,10 +1,13 @@ -RaySGD API Documentation -======================== +RaySGD API Reference +==================== + +PyTorch +------- .. _ref-torch-trainer: TorchTrainer ------------- +~~~~~~~~~~~~ .. autoclass:: ray.util.sgd.torch.TorchTrainer :members: @@ -12,40 +15,66 @@ TorchTrainer .. _ref-torch-operator: PyTorch TrainingOperator ------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~ .. autoclass:: ray.util.sgd.torch.TrainingOperator :members: +.. _ref-creator-operator: + +CreatorOperator +~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.util.sgd.torch.training_operator.CreatorOperator + :members: + :exclude-members: setup + +.. _ref-lightning-operator: + +Pytorch Lightning LightningOperator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.util.sgd.torch.lightning_operator.LightningOperator + :members: + :exclude-members: setup, train_epoch, train_batch, validate, validate_batch, state_dict, load_state_dict + .. _BaseTorchTrainable-doc: BaseTorchTrainable ------------------- +~~~~~~~~~~~~~~~~~~ .. autoclass:: ray.util.sgd.torch.BaseTorchTrainable :members: :private-members: +Tensorflow +---------- + TFTrainer ---------- +~~~~~~~~~ .. autoclass:: ray.util.sgd.tf.TFTrainer :members: .. automethod:: __init__ +RaySGD Dataset +--------------- + Dataset -------- +~~~~~~~ .. autoclass:: ray.util.sgd.data.Dataset :members: .. automethod:: __init__ +RaySGD Utils +------------- .. _ref-utils: Utils ------ +~~~~~ .. autoclass:: ray.util.sgd.utils.AverageMeter :members: diff --git a/doc/source/raysgd/raysgd_tune.rst b/doc/source/raysgd/raysgd_tune.rst index 0826ad97b..cacaea0a2 100644 --- a/doc/source/raysgd/raysgd_tune.rst +++ b/doc/source/raysgd/raysgd_tune.rst @@ -1,3 +1,5 @@ +.. _raysgd-tune: + RaySGD Hyperparameter Tuning ============================ diff --git a/python/ray/util/sgd/torch/examples/pytorch-lightning/mnist-ptl.py b/python/ray/util/sgd/torch/examples/pytorch-lightning/mnist-ptl.py index da7731403..031975f19 100644 --- a/python/ray/util/sgd/torch/examples/pytorch-lightning/mnist-ptl.py +++ b/python/ray/util/sgd/torch/examples/pytorch-lightning/mnist-ptl.py @@ -1,18 +1,29 @@ import argparse +# __import_begin__ +import os + +# Pytorch imports import torch -from ray.util.sgd import TorchTrainer -from ray.util.sgd.torch import TrainingOperator -from torch.nn import functional as F -from pytorch_lightning.core.lightning import LightningModule from torch.optim import Adam from torch.utils.data import DataLoader, random_split -from torchvision.datasets import MNIST -import os +from torch.nn import functional as F from torchvision import transforms +from torchvision.datasets import MNIST + +# Ray imports +from ray.util.sgd import TorchTrainer +from ray.util.sgd.torch import TrainingOperator + +# PTL imports +from pytorch_lightning.core.lightning import LightningModule + +# __import_end__ +# __ptl_begin__ class LitMNIST(LightningModule): + # We take in an additional config parameter here. But this is not required. def __init__(self, config): super().__init__() @@ -77,6 +88,10 @@ class LitMNIST(LightningModule): return {"val_loss": loss.item(), "val_acc": num_correct / num_samples} +# __ptl_end__ + + +# __train_begin__ def train_mnist(num_workers=1, use_gpu=False, num_epochs=5): Operator = TrainingOperator.from_ptl(LitMNIST) trainer = TorchTrainer( @@ -101,6 +116,8 @@ def train_mnist(num_workers=1, use_gpu=False, num_epochs=5): print("success!") +# __train_end__ + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( diff --git a/python/ray/util/sgd/torch/ptl_operator.py b/python/ray/util/sgd/torch/lightning_operator.py similarity index 96% rename from python/ray/util/sgd/torch/ptl_operator.py rename to python/ray/util/sgd/torch/lightning_operator.py index 46d3a09d0..610ed92cd 100644 --- a/python/ray/util/sgd/torch/ptl_operator.py +++ b/python/ray/util/sgd/torch/lightning_operator.py @@ -27,6 +27,24 @@ logger = logging.getLogger(__name__) class LightningOperator(TrainingOperator, TrainerModelHooksMixin, TrainerOptimizersMixin): + """A subclass of TrainingOperator created from a PTL ``LightningModule``. + + This class is returned by `TrainingOperator.from_ptl` and it's training + state is defined by the Pytorch Lightning ``LightningModule`` that is + passed into `from_ptl`. Training and validation functionality have + already been implemented according to + Pytorch Lightning's Trainer. But if you need to modify training, + you should subclass this class and override the appropriate methods + before passing in the subclass to `TorchTrainer`. + + .. code-block:: python + + MyLightningOperator = TrainingOperator.from_ptl( + MyLightningModule) + trainer = TorchTrainer(training_operator_cls=MyLightningOperator, + ...) + """ + def _configure_amp(self, amp, models, optimizers, apex_args=None): assert len(models) == 1 model = models[0] @@ -356,11 +374,7 @@ class LightningOperator(TrainingOperator, TrainerModelHooksMixin, model.on_after_backward() with self.timers.record("apply"): - model.optimizer_step( - epoch=epoch_idx, - batch_idx=batch_idx, - optimizer=optimizer, - optimizer_idx=0) + optimizer.step() model.on_before_zero_grad(optimizer) diff --git a/python/ray/util/sgd/torch/training_operator.py b/python/ray/util/sgd/torch/training_operator.py index 595d7adf0..1c59c68b2 100644 --- a/python/ray/util/sgd/torch/training_operator.py +++ b/python/ray/util/sgd/torch/training_operator.py @@ -777,7 +777,14 @@ class TrainingOperator: lightning_module_cls, train_dataloader=None, val_dataloader=None): - """Creates a TrainingOperator from a Pytorch Lightning Module. + """Create a custom TrainingOperator class from a LightningModule. + + .. code-block:: python + + MyLightningOperator = TrainingOperator.from_ptl( + MyLightningModule) + trainer = TorchTrainer(training_operator_cls=MyLightningOperator, + ...) Args: lightning_module_cls: Your LightningModule class. An object of @@ -793,7 +800,7 @@ class TrainingOperator: A TrainingOperator class properly configured given the LightningModule. """ - from ray.util.sgd.torch.ptl_operator import LightningOperator + from ray.util.sgd.torch.lightning_operator import LightningOperator class CustomLightningOperator(LightningOperator): _lightning_module_cls = lightning_module_cls @@ -810,12 +817,20 @@ class TrainingOperator: loss_creator=None, scheduler_creator=None, serialize_data_creation=True): - """A utility method to create a custom TrainingOperator class from - creator functions. This is useful for backwards compatibility with + """Create a custom TrainingOperator class from creator functions. + + This method is useful for backwards compatibility with previous versions of Ray. To provide custom training and validation, you should subclass the class that is returned by this method instead of ``TrainingOperator``. + .. code-block:: python + + MyCreatorOperator = TrainingOperator.from_creators( + model_creator, optimizer_creator) + trainer = TorchTrainer(training_operator_cls=MyCreatorOperator, + ...) + Args: model_creator (dict -> Model(s)): Constructor function that takes in config and returns the model(s) to be optimized. These @@ -853,8 +868,8 @@ class TrainingOperator: system). Defaults to True. Returns: - A TrainingOperator class with a ``setup`` method that utilizes - the passed in creator functions. + A CreatorOperator class- a subclass of TrainingOperator with a + ``setup`` method that utilizes the passed in creator functions. """ if not (callable(model_creator) and callable(optimizer_creator)): @@ -929,8 +944,21 @@ class TrainingOperator: class CreatorOperator(TrainingOperator): - """A subclass of TrainingOperator specifically for defining training - state using creator functions. + """A subclass of TrainingOperator with training defined by creator funcs. + + This class allows for backwards compatibility with pre Ray 1.0 versions. + + This class is returned by `TrainingOperator.from_creators(...)`. If you + need to add custom functionality, you should subclass this class, + implement the appropriate methods and pass the subclass into + `TorchTrainer`. + + .. code-block:: python + + MyCreatorOperator = TrainingOperator.from_creators( + model_creator, optimizer_creator) + trainer = TorchTrainer(training_operator_cls=MyCreatorOperator, + ...) """ def _validate_loaders(self, loaders):