From 53489d2f85afdbe976875ae2d2567812fc265037 Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Sat, 10 Nov 2018 21:52:20 -0800 Subject: [PATCH] [sgd] Document and add simple MNIST example (#3236) --- README.rst | 1 + doc/source/conf.py | 2 + doc/source/distributed_sgd.rst | 56 ++++++++ doc/source/index.rst | 4 +- doc/source/redis-memory-management.rst | 2 +- doc/source/sgd.png | Bin 0 -> 27304 bytes .../using-ray-and-docker-on-a-cluster.md | 2 +- python/ray/experimental/sgd/__init__.py | 11 ++ python/ray/experimental/sgd/mnist_example.py | 134 ++++++++++++++++++ python/ray/experimental/sgd/sgd.py | 32 +++-- python/ray/experimental/sgd/sgd_worker.py | 55 ++++--- python/ray/rllib/models/catalog.py | 4 +- python/ray/tune/examples/__init__.py | 0 python/ray/tune/examples/tune_mnist_ray.py | 2 +- test/jenkins_tests/run_multi_node_tests.sh | 12 +- 15 files changed, 279 insertions(+), 38 deletions(-) create mode 100644 doc/source/distributed_sgd.rst create mode 100644 doc/source/sgd.png create mode 100755 python/ray/experimental/sgd/mnist_example.py create mode 100644 python/ray/tune/examples/__init__.py diff --git a/README.rst b/README.rst index 356ef60eb..3a8855b24 100644 --- a/README.rst +++ b/README.rst @@ -39,6 +39,7 @@ Ray comes with libraries that accelerate deep learning and reinforcement learnin - `Ray Tune`_: Hyperparameter Optimization Framework - `Ray RLlib`_: Scalable Reinforcement Learning +- `Distributed Training `__ .. _`Ray Tune`: http://ray.readthedocs.io/en/latest/tune.html .. _`Ray RLlib`: http://ray.readthedocs.io/en/latest/rllib.html diff --git a/doc/source/conf.py b/doc/source/conf.py index 2d212d23b..e362f7330 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -25,6 +25,8 @@ MOCK_MODULES = [ "scipy.signal", "tensorflow", "tensorflow.contrib", + "tensorflow.contrib.all_reduce", + "tensorflow.contrib.all_reduce.python", "tensorflow.contrib.layers", "tensorflow.contrib.slim", "tensorflow.contrib.rnn", diff --git a/doc/source/distributed_sgd.rst b/doc/source/distributed_sgd.rst new file mode 100644 index 000000000..5d1e48076 --- /dev/null +++ b/doc/source/distributed_sgd.rst @@ -0,0 +1,56 @@ +Distributed SGD (Experimental) +============================== + +Ray includes an implementation of synchronous distributed stochastic gradient descent (SGD), which is competitive in performance with implementations in Horovod and Distributed TensorFlow. + +Ray SGD is built on top of the Ray task and actor abstractions to provide seamless integration into existing Ray applications. + +Interface +--------- + +To use Ray SGD, define a `model class `__ with ``loss`` and ``optimizer`` attributes: + +.. autoclass:: ray.experimental.sgd.Model + +Then, pass a model creator function to the ``ray.experimental.sgd.DistributedSGD`` class. To drive the distributed training, ``sgd.step()`` can be called repeatedly: + +.. code-block:: python + + model_creator = lambda worker_idx, device_idx: YourModelClass() + + sgd = DistributedSGD( + model_creator, + num_workers=2, + devices_per_worker=4, + gpu=True, + strategy="ps") + + for i in range(NUM_ITERS): + sgd.step() + +Under the hood, Ray SGD will create *replicas* of your model onto each hardware device (GPU) allocated to workers (controlled by ``num_workers``). Multiple devices can be managed by each worker process (controlled by ``devices_per_worker``). Each model instance will be in a separate TF variable scope. The ``DistributedSGD`` class coordinates the distributed computation and application of gradients to improve the model. + +There are two distributed SGD strategies available for use: + - ``strategy="simple"``: Gradients are averaged centrally on the driver before being applied to each model replica. This is a reference implementation for debugging purposes. + - ``strategy="ps"``: Gradients are computed and averaged within each node. Gradients are then averaged across nodes through a number of parameter server actors. To pipeline the computation of gradients and transmission across the network, we use a custom TensorFlow op that can read and write to the Ray object store directly. + +Note that when ``num_workers=1``, only local allreduce will be used and the choice of distributed strategy is irrelevant. + +The full documentation for ``DistributedSGD`` is as follows: + +.. autoclass:: ray.experimental.sgd.DistributedSGD + +Examples +-------- + +For examples of end-to-end usage, check out the `ImageNet synthetic data test `__ and also the simple `MNIST training example `__, which includes examples of how access the model weights and monitor accuracy as training progresses. + +Performance +----------- + +When using the new Ray backend (which will be enabled by default in Ray 0.6+), we `expect `__ performance competitive with other synchronous SGD implementations on 25Gbps Ethernet. + +.. figure:: sgd.png + :width: 756px + + Images per second reached when distributing the training of a ResNet-101 TensorFlow model (from the official TF benchmark). All experiments were run on p3.16xl instances connected by 25Gbps Ethernet, and workers allocated 4 GPUs per node as done in the Horovod benchmark. diff --git a/doc/source/index.rst b/doc/source/index.rst index ca56625b2..e5ebdae45 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -42,6 +42,7 @@ Ray comes with libraries that accelerate deep learning and reinforcement learnin - `Tune`_: Scalable Hyperparameter Search - `RLlib`_: Scalable Reinforcement Learning +- `Distributed Training `__ .. _`Tune`: tune.html .. _`RLlib`: rllib.html @@ -90,8 +91,9 @@ Ray comes with libraries that accelerate deep learning and reinforcement learnin .. toctree:: :maxdepth: 1 - :caption: Pandas on Ray + :caption: Other Libraries + distributed_sgd.rst pandas_on_ray.rst .. toctree:: diff --git a/doc/source/redis-memory-management.rst b/doc/source/redis-memory-management.rst index 64d2035ed..91b207db5 100644 --- a/doc/source/redis-memory-management.rst +++ b/doc/source/redis-memory-management.rst @@ -1,4 +1,4 @@ -Redis Memory Management (EXPERIMENTAL) +Redis Memory Management (Experimental) ====================================== Ray stores metadata associated with tasks and objects in one or more Redis diff --git a/doc/source/sgd.png b/doc/source/sgd.png new file mode 100644 index 0000000000000000000000000000000000000000..aed38161cb159e0b3d3da2bd3f3645fa5b78c180 GIT binary patch literal 27304 zcma%@Wk4NUwyg`d-~mE#5AN>n?hZkNyIXJx9^BpCo#5`S2`<6i-_AMRef!FL@A3l* zs7+O^y=v8%-x!lH1vzm<_>b@a03b?Ah$sO7_%&$z5f&WuS&&-f1LzBcqmZNuEG+Em zFZrLKZ?T<3HJp@fO`Ti~9ZUc-8(V7=T1O)X6B8Rpb6ck~h%P<=AOs{u1XbM9f3KV9 zsC>Z!&Y5|?3l9QH{<(zNeg?!x@Snee-L6zvy1JHsF1O4%t^9g&N{!j*9~K_|i8j;; znSu8V#=;LQ*ifwJ>g^;X!)&7CE+xiuPtsq46g&_`2sS6te`yIr0G22aC5}SI2J{o6 zxm*K>fIq*&go?-cvrogPvuD_;Jh!N|eZq-uf2rxtb9GJX(xPI+?8_6XH-9n&26Keu9+H^28 zYg1Ogw`Y31H_tVxQOhHvDE*rOTv>+dhW%gXa+E4FiYT~TUX+xCROCR(f3keCxI8to z(ZJf#t`LLIGPq7ph$_RgCwEm(>fPb{`K`fE@)Sy{YVsaC9^>!h?g`9&-`&V6hXKSK zg*7pPg6I=90KbiU%aZ3fG{M^d_dTFH{gF`J{_i6qrOGsYhwS+V14CB`R-eYy5HICU zNlo+9iN^0GE+Nw+KaMz8EV{pJ{nPo&)~_iY(f&V!SFCB?GaO0CY3{4Qhmv6y-B6g2 za zCv*IHLs^lTDA=KhMxUI{nyw{%4vk(Hy9N<8Jjhw7C@GH~9vUDXZQX_~B>I=KUiqG( z1-#F_ae_d5d3~LJ|?9=g)tN)b>K}6gH zuG~%uo-6z?lJr|dsHgLqIsx%B`-V)CHy!Kod~EzQFsA6}`6WClVu^+hP&jxmv$Fyb)#swQ*a__2TLVf9Z*LvG6)Le+I7F%%#dHr(r)Z6pS zViQS(*g@%t)9ddL7>F(i4cW|7jR#1`RCo-)4L7s+5><#U4H;0u_$Y~LpU~lXH!(hr z2#k%5mGX}oFu+0z-8lRi6iLZX@Avli)#;NyN`}%N}sYB%9>h7+ow)P`~fq;?v4u$i<6cKqm zr_;gIK;@A9(Nc}kJ7}Ywq1Z^>*sQrt-#1y}*nw}@vr8Bck{R5tQRAla=^QmR)RzJ8 z@Od-{QHqO;=g>%u9E9iWe=kDoZRm1s>VMH%onwxu+6(B9c{L?#f#tqfSLZWNJshd$ zN2j>F&Pft7a2qEu(CCm3wfVK7Yvw59G?llb+PG&jk5w3=69FhVp4wLACO3~HWY2>6 ziRH0o7;jzfj*VU zSIFD8<4~{C!=QJ6d*w|VM|$_}Jv}amU3O`yRi2(Uc#5kWF;4X3evm_v^(}%B&DK3`m^p@5yyu1meeC}haeNU@#rZkqK1(gLXSHX#i00K3* zzkDBuL>xZ{4?@hu)+jYazMrFB9 zg0@}VgX065*x-}^%c2D;v=xmVyuTOlsxUJ48zHlmHsr>)i@#UPo>WJNfWB{d+>MF5 z-(qqf9v&D!d>g3jpR`=6Hk6X8l`#--V01e-&}mVLg7kl^_*PX`MvWE*2`<=yWVH<8 z1WO!SeELa6Q%5H~C@842(??25%4iYl&(nv=`|;YNKWQ!O`uaMw&*NLsQj@JdGhUu> zfs#9qo}64%t4AbyYR&E*3@W_Ira1!912GEhJvR=~oWG&N70Fts_iAr%UXA5f1xlur zvF(eaC}CJu{5N}ngj39ujs})Yy7hOPniM8gs*ckl@i=mi`{zSd%l>LIj>or!M{$;v zLy?|X_!*Ql49DiS1)5i8RA+zIS&1JKB82m13pU={+KnD$?40Zatrg?W*gRH!Lpe4l zEbn7Kz#;JK>+riSced$|`l`>2g*acT8TQas6TEgxRYP zmR?-AwOt8VSX~eBpz|D3s=;RR)R??|obwM{Y=WTgc#9i_`6O=xc_`?(_5zOfqtED+ zMP7Pl@rmMAA0c^1FZ=rOTb6x4b)2h?@U%J0$4E6-Tp{>?O@Y9v$!xtQoB7k%Uk{di z(E6Qjha(BTwlJqYXw9BrijzMaL+`3%Njlb7O4-dnYyOy>?@i)0PLb`ny`Dlb)f`W> zUQS9ELrdGyeBbElBS3r**>$_shr){njAgQ!e>FOL!NRmoJ#?Ro%*p;Up2amaEj`^7 zG#7Dk%V1{sHGORb!Op6x%8%}CB}XqK!&+>avuu!Sk1=}5C}_==ujh2vL)m$j6gSpI zIX26d*E8PgdfsNEO6WfpugZ}7zrOAmaZ@wl003HV4CMlEA3OpF9)dxaaY2#Z7dp|o zgBrI69H>rx9^BNj;=hhRjS34%W>N4acBh25F0GoNS8`5~g+l3BgrK;bUv{=t*%|Tj zph*xwuixzSFw(%i@50O1bj%)?86fVB;|RHVmgzMD(Q6p5Gbh+B6(&1`_T_{22d2`w z7`wB}!akDF9qraXf33!7XCd(Fs=P<}b4#)EeGK}!nim{dpor~WuNm85 z#ZzScuK9&A0ORd?ZNj}b-8@SQ$FW{#p_A(S5o!6PD?gmyOr!FTk_)62i?PPW#*=Sz z36X1iuP>6%??mBjJHZrWPrgxC@QVC*b>NV3A)>-~ve%TKwi)NWRvvvv_veay&t~B7 zXK<-#q@pIF{LT{~&WcK0@EIp(#WHRlCQ=-uk>u>cs61Xa-%)=A{9@uW^dPgXL1MuR zJe(^YEG8{pts6$ScJ0p^1A4Z9I%t%guf0e&-R88;tA`0)g2Dy-%2{N)uBzU!HJK#8 zoYX4LW_v*)LsUt7zi}^em!ILHU5%`CU&`Y4L3DjXD0E($t(EOga-v{*m z3JeOOJbiX1jvu~}v1!)#Ia{&6cw)2E3%0JWYNdudzW9dJ5p8^Z&wcyK5rD}mr5}Ui zbJ3R~akb#MNX61J+{x8Fwr}c#r^vDoktMYEq5pNGVS9_g=4O9bO3AVxb?ZjgE$saC4epyBCPyUCN1mY=yude z!TG)bt1fr@yQy`d%%_)6e#R@?Nk|n`1#$0?b&kGy&wdAoo ztUk!yTG8qrSCo#ncE5%3P9Wf$JW3l>uFuphzSNiYC9*Bo->!Rp@ien45QLRH9#3Nj zs=c3F;DeOR7=Ta_VqmGmlRZo&I!iVJi{9pEX^w}7hkln&g2u%D;sr_TFe$rJL?^FP z<3M4MxG921ablNm&vS9LF;K98xiUAyad*m6hy0Yl)#C zkq!wGD$2^rQD=e(1LODiKNot>t6Fl^#MpRwA6JgmUq>7r(YBO1Rjien@g&@xnGYmI zlED-D&SH7blClFmmtGXj$hrIGb=Nr++vZW6)E-b?&EK@S?_WD;j&)PZQ8e5s zR)o~K(sDd~!2f8D1_o$UA=|*22Hb{zS)Ugaw3iU;X;*)uc%*CnNPo-Ae!b>!nq|UU zKN5#(vxZ84jFBa?pPm}qZ1Lbb3;^99PMPKh)72I)){^re@YuvHqdKu22ZV)x_lVlE zIXkYPRxw}bUBkJS)~kX6f=A;f0pibkouNO#Z#FFN+SIVW2cbNEF)=nCvI~uhiu(Se zzrPytsW)kH3R}}+ zJ9%F`uS(Jjx$wKET&1wYyi0o`IeA9*@6@ddCT1I}u#ae>dM~3LxK~>e%jc{E-BGe?j0^uuL1-(8gTrG#V4^pO3B!W zc=_17x!qiD)wB3(uw$+d1ZyZ)p??K(#a?+o{Z)59Kg&{ecrQ*ol9i-`YR}9d>L`9W z@ZL(J#n;ZU+3HN%3(}6SFlSln*!-4nLF0Aw-@hwX1fOuZI|+(k5JCN41?QEN`gd2{ zQk1S*LVn6!H}r6VdcVE>W&dO@$0(&>S@yIp4=WywWUXP@DLYAwTK=br;0(?bzHk@rA^_8bKN^QG* zl~J%KNtVk34Di$U#dFQCz!e@0h?}UwJCA+YdcfRI72lQ;Yx>Ya-)FYIzduP5hcL{Q zD5uNKME09gPNgBk;qqmP_Do1$+$@Y`3|EJ{K`nM_fcT6qo`pxc z+z$iajRY}!e9S}-dY4}yRLdKaww}NBK=*uk(LLZX!dgb^sM9VrK&m5}BJEw~wBwJ_ z5ahRK{oG9_WhOviA&>uU9P)tSir`4ND%|q=2d&;Kb-^x# z)ME3|{#M4h!HxweO3k{gr0fEu{vHjei1X4GF1mYM!w|7@iyzo470U==YRTE541IN9 zdmpMzgMfu*}k4y@!!;aMyB+pE7-szZ;CyL(0-*tU+GL(zjdSl%tNE=$*?1u=9e56wC zG&5GUHUSLwu~fC*Qc9&rhMag5tIo#2j@aosd0kFN({CDg^0S4J!@tg04xlJIbjXnz z?AM6`Qb!{Ry%JPMO5fT@N>)9h?@lx}SShyp)?06>HmAmm+VmdidHgNR;tN;5M}T15 z*Kfm!U7wmh_^C90`O0uSA1RUTbmO7|17sgHvMTx?*e;`C%8T-&#?=PHqxvPJ=$nXw z`{-GK0mbH}3y)CcgBPj1rf$bNJP*57U7Pglh4wdgu*lze&5#&~=g(9~3udaRv~w@6 zW;K=C^#Z1sZV8n3m3&NMF}pyN^h8o6_}8@Z8xzb z%FbW_&I~*=D1#f59Q+21`RNe-*pq6WP4^p^hAlz=`!LZ9WxZhX#iw}^P%VthM9waP zH|S&S=jeLq^=5hJA4R9_#Y#XEe9cac?bj>kb(esgYTC-Nd_YlTwaq^Bj_~llmAPhj z-0itf+Vai^n&AUeXIdH~3Cydkz^67ft;PU6PoL#`&L?4JX@WA>)b@Sb zH#odfIk`mF^xX6c?)((uL&WE+VT|#xIv0M9*2}}`I3d}jWtg|+6Em9rrYgR{*_B6G zM;+$31_>|sG>@$CK3t!zTfXTbs3hAHO*;g@Z^mclqHdrId5zGJgb_5IWRV5umPTO{ zsWZf-nT{`{JuhUVD93VEkz`P{N7O(-cm0cj0CBS0SjOHL$w!a3zE?{{ul3k{y)4u& z>0$0lp??Df;Vkqgw<6`VXszLG4-lsEsfKoK zvzzI$&El;09DQ_ds~WHefWY7N&6E0_dkZxLE#(@9U`0k?zmv+Z=cvcOgni`2ta8cg zzKaMEcErH=3+9;S5Dm*J%KU7vl=1eAuBl5Z->9}94bor(fE~W{{iTW9Mc-MB8ZBOm z*Nxuyz^I_NY+vwm+&d5VRbVH0jVkSc`Y#;vH+J^_PCe`Kjq997yP{+8emJxBbJ5%x zcR{p;;Q4ToCM$tnU4D1T`bz2t>@-(9JPwoDhjSglf}#kt4KIl7)D>)XJyHG9)(kEI z3pcNv&wq1bo}tvWYX0Y%XBn#crxUrcr!-&dTV^a@rS(@V>Q5ebgkpt#gzT)*V+uuB zV*JSP8pmdvhJ9J67{?`WoXdxzK*c+j(sw(##mv~ib9#MTfeA>&{UKqdLIXQuF&{ms zKdOXSX|KySJsQ}c^(sUoE!!X7toVE#!}Z``A@AaN(w6Caik|)gwRviNH5YuQZWg)p zQ7v&J^e?tO!sb)#M)cp6}jYuj~fzR%+{|$KAiy2euGok%7frk2yvJ%7+dx~^(Mrc3LhhtEELy=076$M1YYeAn)>{%V;n`GQ7aW5ZP@WdcsndhCyg+eZ_h2dZn?QS?mn8zKIbdr<29l2K11Kt+U8Y>eUf>#JNx~M zxyr$BtKQ{}<0?G7^nV48g~kPi?-GeiR^O#`c@I59s|RKqPc2k4oR561rv6+2yC-wO;Gvbmr;q}4WxaaO# z2PwtSVE3nSg?!SSlhFUHdr>}}ak!5T4}*+~n5`;h5^Zhmu-*oXftO3q#i)qh5p@P) zh3_9yA1f-jxZ2zwUbtGD+!O9(L7e5VFGb!|%rRfqCJ-vN>Gh6EeB7oR;6^M7HdWU# zlGSujuE-ZW(kG*Y1=%~99Oz#*JWjR3hGlQ(!A$wsuZ}qo#vi(iVmQTW68IC~jmrvh zY+X56z0CPRD2@c=o~4)H2Rb$GtmnZ(5~oZ|86GhodFtmJHagB&Q)_xp za@M50jLdVadU;QGcXnq;s9V1d5cefopR1nOUt)-QLR$VA_wnqB;thYZzI?R@-K-H2 zW0XRoV&&%U!ot#^jOYaFe)x?JyKXCfxpvX}H-SAw)jbEA-onMjd||`chV*Lx=~epoeX}Ndt!jSb=q4QMgZ>;a z69o16WZ^p<{FUyJgdz30cs zZ*RrECMBkfwqQ5Rj8=@zxBQNRNf!9*dL^nur#RVorVt&amC$(^XQ{4LYH3j@TGkUH}49| z-?{6v7`();ce!gaOT8vK&gx~Z@^~)%5Lv|_`r6g#;rg3>Q*&Qvah#v~XhUU|oVgQ* z61wAs<&7-lH=B56vB^KP06v`F2g?JMFQ7odS~H2>0ySR>ttVbnhohv%bWvV>$Qb;$ z5|)cB&hEE~-G>FvB(WtK4>%ykYi#gUW7zRKUy{-p(ku;c!eP2HGnzW>8$C7veCX9J zMA}Kz-8R~FK~S6?^bUY7Z;#beV_3CuF^G9n=&h~#EPB!xy)%H1KjmYR{}9fOs*MNo zD^=hCzqISk(wcZz{)zHtx{BVuJaweVrqU&Iy%-P2<)5fsT~w=5E?+&)8lH5I+l1jj zi)_hIs+}G2t^E3ZkK6ol#BrZ3g$nOJ+><{`?Fw=V|J|kEah@4|8_HYzXr-B%a2RF_w0P4;uCdSC3rqXRTT3sv2XR6is80hj$lt(vzm~C0oF5G3+s}N~TZiMpcqU4K zz?-pqMwDXmoAyPl?i00c{PVQS7adZxoh6Ix|9kI(ikgvMn}o0R>xK9D-m}R1DGBbE{Y*U%ZHCm64Ic&m0A2Ae@q#OXsvp|2IfYRh zE1j%RpJ(}f%>w#%#FB5eHeG>lzr^1*c(R>hYLwZd6}#Xbgh0qrR*RGUWearCT)bxG z%cS1oVb(Ws#S=(;aRm zvR+ujfS+p>MHz9*NfoSWI&2r*I;?lGeYZk6jVX>#4|G_+(Zpb1jE${iP~Y!jg1tB0 zkzVPLQbtcp1%RPaEh)qj1M7sLNCZ!(r`z5n#bva;ZYnyYdH?*rY_Mtsi)0BH zf&3W>;CFew|H!c-`mmA@mgG8?oMX(Ti#5AzBX!>Ic9G!Y-z~SmvMYTsmV~PpUN~Pm zHx86RmqBcrn&^on^otqN4EXEuXq5v1z?0BzN;{+Gu2y1*=&hoi=R66mO1XX%90*u%(+lk$Th@UC!sGeP04Ij& z(oMGIp6fSR7hDf2NeL_77s0+?nW~K?p9eGb*?cBmr)uqyjxxy49#(yYLQ3muL0;(* zvBAD0U|w4Paf35guBs%n0$<;kVYue_MBf|cWoEIXE%IyM{5Oki+eJ0q5j)*MIghPP zyIP;336FES6Je*yDcOo&S=NGcbs~TS2W$U!s~Zv#P3Q<1N+GGSy9>Go8Tut3k}%IcA3X+e)ET-Po? z5r&Io?(x~U>zmiGmql>$7OYPleA{L+Y8tquI8Uojr1*Y{+7Mw7Hh`Zo&*L@v>x!s$ z)FyA&@$Q2zkZr?jur)(#Ik+mFP_Ll+C9pLQLQDQXJC0ZZ=vJ#ceP;@>X6|)lrTpIFQ|R zquXfMi$-weu~v5}#3uK`JnS|QTW}lavs~K)E;k6iM6Y>RqW)M1(z`|%7VPj|U2YsK z>WcQoa%=Z>qb?%AhsW_FwL6<5QdZ^tukndzyabPwZna?NnFL+uP=KFAeheT$TKr`$ z>bh1h&}%o8caJoy2m;thT?(ZlQCX|&EKAdmJWD<1q4B{GJYHkIMm;6s;)>p3-)V<@?%2 z_k)L#{Pi_St@#@;a?c$hlt5y@WB3`i2-Cr?HAD=K!?7c%N>r$TbkC}+#947xQs844 z#M4-ntr(E7k<-%KC4WaOD=ZENh>*QR8wYoiV-u1v#m@h{-W&aT!mc!Wi79v#Y5l|x z!2lCkON~b^HePs-Htr*Uix#ZH}P^+YzaBwmxV^7$9w~gS!KU$%rDQGCB zHe776i`45a1E57&x1_4vjs#)<;CAVu5->W@@uqtxtU8B!<(aVH`&bFBMEK1QTde{7D7uzb{EicXA4u2o!TVFjyxl%?$ z?`-dM+*SLrGj(2pIuEL8?<(0ts@*FFDr^s1{4fBh+DpzN*ePZ<_M zp0HNEd6v;gCN#g@cg81!Ch@#Xw_{v{1!j(8u9D&{`X_$o;0W@gv!xR~(}{Pndi$~9 zJNzM|NpPGv=#M4}if7&^N)M7XTAy0oALLRwmKvk1Kf*adPz&Ea{8Y{gpoW(mcu{*s zt)8hQ?IP8d;dW&EvQg-J8l)2q0c0B^ZBi;b%zCVqME;IscAa>ts_!El$TcRY6oY~$eL(sTQL zUUS|l_X#f0r$TeQ4_kc_A;Hq=Y%u#l3jk72l7o{U-}vl=d-_%sYMFWbkj!Uhfun*MfCCk#(8f*r(A{7&In~tyrAAW+$eyB9eJuZH zgbobQJhg{+xy#F~BBY;l86^=JHHoHR6o(!_;!nVTxscwvt1l;3V68sdG43s&OORwH zsJVI!&6v?^eMH|P6ArHOz`3L%Pb;=vU2o8Bi0lnJq8qjK&A2DJ)-;9|TpejC>j1;+ zRct`%vu(H38CFD@;U%hS|a;i^%3oY1YGMsd| zD}(2XLPIP*RUIU;Rjsy5L@Ud?YI=Km>`o=FH*sWgSK(sQc-KWTY(LJZ%D0d+FCZz( z(!QvZj12t(_seG3UR|uBVv#NE{D>Xugh!C-z}U0vV1j#1*oK$Hb!sb9J|j}62yw)`>`8N~9>+%#B?^W^^u zfb(_bn}y`>*LcP1cK76&k?h;lCfQu@2~mf@N$csl-y@_$1^kO}QWeN_}j~90R2*Ht^;NlIF z(+f0rvjhJ9_fhvhc#pWuab@M7^w1;r`NK0Fw~GV; zJyn`it>eBNxZujTBJgu6M|f!_;JwH~B(w8sYDss%y%U`u(^T@w%}>+0^}C0zkHruG z#i6VWW6Y;~53lipqNCQ&(VIcBUqfK7$Fvz2N5hlt?uHuUW4q5+IyZE!5UAY;LBMV@ zumo)884(Mfh!q3qR)EMKwuTgv7`HT-M=1>`r)96;dg4%ey(2y#u{3a1H8~c1x%TDv zF&(&$;x2M36v@=>V1()pfWh_>tbA(u9^sxud%bMqUv#@#nTjZmtI@7 z)lSe1P}Z_Kn95o_8)43H5oDgbQ>0>Dl3_Y(ZlG`DZ7ldCZsNfs}pCUeCroM2ojW{^n_v;11{}E2SMmNGhh+ZGYqDH|sj{yh?n`a8Gj>W$;Dh=#Fvm!lI zREw}*Xt+%o<*ZGBF18qMMuF@krF?y(t{Cid&@h}n1I0Oox!|UR8HF+*w z$Y$4?*<^_W{jZqxFJn2I<_q7*)5qi8xm1#)CVNv#Lq#navLsga4o2CZ>v z9yo{XjUaKeaF#Ky740tzJeJyj<=o!~4{m#d42%f|92^|@pi79OfK-x{yF`zA(_gzP z=DS>$S2NQcE6m!{rIjH9*~fEb%VIHJ_Th`=zCFxPpMxy$f~%H%4Z_-d5{zRZ#b>E*h}$;lZFd@GqmU8C=EJ!$6nrmZJX zSazJU?1IDNepp+R!J4{gk+-Z4ll-tYRz(uN=k-&02sdwuioP9p+2YH=DxTa3dojY;5Q{!E1wppvc*a$!kw(6CU@h*@}^Pd~Uj7c^^nY*k(x` zm4+mV$lH@^Ir(UcgA(;_0Ii%C{!deNkdOWBgnA0O!d9qgy@70Rh@Zr1pURg4hQ zTf$w;gj$u1Ai#8TAswP70uWK$Y4=4?V7c(L*fCyv91ziWn10_>Km-+1q_rbY$`9C8 zHeE6>sv-IwjMT5DGv(vo;pX++pu%gMnWc|VEM&lQ4sMPa08oZiSk_Z9%#0QkQw4qG zEAX@|*O|$mU}@EvBHEmt&&FLW$SPSBKxP{n(sUIfIvm5jJ>90IWG{u6f%Vv~>fm4$iG~E~iV(GaE_uEMiKdx);5$=~`tRTsDa;nW zgqDROActVl8}qgPh3&YL*wkQa@hi?YZ5@}UgJ8^VV_m9LL_dAwaJ?3@;ocn1tSC5A zRr$YDJ1KfxA{F1=VNkp_oCU`^@k}=%E>5YcqfA5<+CMidtb91qr|OqW_2u-oNM%HW zmsk->0^(Td2D6H_mlqAYn8={7)b>|=L;&~?Yl+5^l+fZ@qkZLMQWBXSA8Gw?ucR`D z14>B^TDAYTiXsB1UTprnkYAYhxhRae?p*W7=6H5N9c1Rz0!Q0)6wzt zd^<52W^LT(&KeOZ83c5(-Lu@71l%iHeUU|W=J}XQ9P%;W!!Elo*XAM9N}w*4oBSuk(5H>TsYNtLplVDFUImxbLmN0+_x>O z2b6LYh0NC}NA*+{5r~=a6O!(?iNV97z%Q8{_+MN%;LFVCM@yP!YLLt2} z3z(k_jGr{xn-1M{EXG00vr;~BQ;aq5`|?3`TO84tH4L~3f^y`-DryVGn<6lQrE%U= zZoe;3>6G6_Apw+~0&>;OA9{r)KNar1!ld5<7;(YUdLam#uv>}DH<>0cH?M`{Ts%b_b0XGF*>Vq-_TUu0b@R-x`T=_`jkxLprVb_*8^wvW_d zqB=7`(*s<<=HzA7uYl-{ciH#x>RuI3sudqf^6j)E6Gmj8bDNuG0~SAQ%eC3!6n*nu z%!=v6$$lFHm>8%HNo23&C-MT#3%zot@(=0QIho(-t>zca<@NW-`btKXFDKuRHTpR<0dc64Hl-%ie5B7?0J~vLPLIE856Q{mo5GIpQy*Q0E?Zgb+ zT-xEtc@#J)bG4Z0e{%YS$13OF+0PsVME{)MH1`;%x2xRnTOGPTTO@YN&7Gk)lzh*v zix~iXEfj(R@I%y1KD5V-?m+;mL1N%geUhLU{0#LSSklz|e8S@{o4N^eE>u4@W0l^( zdqA`!QWr<8;VwMh>1?vX^R)&0@s}l?{BzjzcOCriPQr{jW^(ngOZZQduOY`e9V|KL z>leKQKz#`9IYaSHTYu!5>86i~@%&cZ%iLr*Z~k@5cMsR(!f*8TRR7{nN<29r1el`P zp_A%v$0QG>t`0qAHLI0Vwl*!>!Ir-Kv}Ex(o40ya(RrE&C6I4vJ`D!1bmf*{k*i&8RSw-RbP2H#c ztx5!#g>!73XOxf3w>jqBXfk?G;w6?_tFWxPL|pf3Jx;h|LQ~;Rt3OAEcrm{BdIcGX z7RW;r6PWa9U9#2nHQ^UX)12Pt{^rpCrfy@~y+ zGAuPBO9!JV$7DRi`Cjp`H9Ch>csgv&E=P6}w~qvOoRJ7xDgar$oR}R6`Kve6pmyC> zA=!Tj@Lo-j06#c9EXdCvy1QW$7ExV29E@7G<=?t|OSJW2Ixs)0{T_JZuz%!RC*Zkr zh~*p2NbhO8e5SU#agU;W_+BL<_Ufq|*Y_^d*Fc5d`>kvlaABrcU7Y6=V*1d$TlEzq zI*5`&x2hxjwBe{PrIX*-nKr zn;IK~AYNPW9C{A)%;MtY@NkKq++U3uHxBlo_xy~3-9X!Jvtfgo~ zO|Rd>)2(+`2T03VA5~jY{?kMRnXAt2$~!_AVb;OdOqPteB3$H5e>NNgiaa8Ydt7SS zLZ4?^t=@&JDv9KmlqdoYM!9t(+9*(mu58wBq?ErFezV%9qYBJCY@dl{^W8W72yT`zCXIN$^%$Nqu{-`vKd{O<=Me!?B}06wWo;iW?j5UDe6) zIxBT8tNUQi3d#+S`gm=7EA8Wbp>NWw?Ynr$>_e`~q?v#=qPnLivhlmq;vZ4SoWAD@ z3=xt+XY+SiDu*oAVj(YOBtW;_oR=%D*EYyoAC_3y3EnVVo~@c~Y51m5xvyg9k2Whs zakUl%Ax|HgbkruJ)nK+Sqkcge(1+{t`;_r9`JrgSI@ulx06ry>Jj)ZLnCq2L2yfWK zE2}`hy*>$7l9IEjO;D&bX0^0ty#@YEOGEPg&^VomWE^i_1Zs4}KYKg6Ok7R^(TQI? zXTLE|#$OhEN_}|79Y8g2S(3Ow8zpnP&%MoiP~t!fDP?d(0UNqvm(x7H57+&6bVyl~ z@r<7=Ki>=!K7r-kDwk93Oo8-0oc#Z2RX>2)hm+i?2rCw!B#MV7N_&MH^SsI_7AOM7 zy+oc_MKo~8-mhcr#Xiw=P!FeW5-G>YZb{-s)jR&Z+#R60RaEZOU7rZRcOg(@;r#^)xcX@pR@*i z3@#pXIyl}|P)}yKJ3Ot(@lwEPH&6b8{-j0>3d`x8v&|%%v!vG5IT|hI+QT{O;qsr` z?j-$0H|qkmu(p}{d8r49z!F{0mWVqjpBoEq>Q|Z8A9-H?y6C)?c*_xfA>m{VVRze> zxmHFo4SB7xS@i@6KOisTH20vYzlnl)1jCJl4J4F4$#y5E0@@{HL0v(ontrv&H!B>o z+^l!YZawqS-Vvdt9;W16anKrnilQ6k=$*oZr<$R_4zJv}C@w&8qpo0TGQ9N5hM*od z|6jXE*@QaL3ZqxlieF~Zs)}iRzvO&Y7Gjf^Tpb(8sgR?MuYqc@KkLb37X4_7cT&M( zi3Ca(gHU2%YBl*RA|Z{NtJe<&3L_H?(~wH<4rq_ow)$CBn|0Ue2l}A-yycqxeb?E= zYCET^^Mmz*ibTDAN(vH<`L-YbXlN^}93D$VE9B?+`F7?xRZqZSBXAcbR{FcoEGa6z zrD<@Ito3}nK++V0meT(^oAcBMEX-hw(KWS*?C|&n{izB{#(`eJ!+T&dxxD&MD-kR! zgzWx;?DQF%_gQBH^Lnxgb@R9$gXK^25JjxFCWp~F)IraZ1Wc^|PaRC@lwAHn#; zMB*lV=Yw-wj1-XeUUr-6VH5+t5Xr}nAZY$8<3+P79sge_KWL0lEHG2$Mk7T?|Fp~o zu3dT;a9-Tu7XlhcN0!8FJ4wb^NItT!13OI0&)-#3 z=7%B#zgQ1yj=T0<+yX*dF}X}c|J7!6xD{(md+c&_&c@2qiJ}AQX`&$D*?tM#qZ{^7pnC6|E_3JEG$wT9C}}!%x@bdfbiquQ>wE0?0EQONhEeIi6<@gsIiXm9NqmFYp7>QfzJ6Q($vZ8+|IwF-Qihpl5VwKsS z$go{+o$rn(ypI1wMWt13@b&rvRAiE3KVBckyFG06M_6)(OOU=k>}S=^qY%yYvGTN_ z*48O2GhK1|Q}M@!Qi)k>mBku!oBb#|*$6-u0s9cmeRVc6f$<^~7$?ZVgH$zw%tVwl z4g-|JN9KqAJz>xhC2X39*+&?Mp8lByaHZ4m{&|_^G#s(+%$@WwCojFF+(7;8W+&1$ zPh@5A_gbo6iKK-$aHK>MqGUGXcyE-yR;Um z01+E%_lLeXX!!g643C`#C)zPVhlQ-38$(4-+wRX+CX<1s-RrS^8)~S8C4w3n8@oO~ zFaD=^ade1(atmAadtn5CieyudQkBmMK6vwuA8YH~tMj6w9kYD7k)7J3c{x}&($#j| z|Fe{l6qK9Ww*rK)akF9JLpRS~2Ng@J>#uVl_i^2|z4D1Bw=T<<+ROJviEvxzO)NPAH z^??w0g4R-HNnU?d;jG#G*OC)yf|CDtBCOn8_zz1f7X5h+`Rms&NN~f&N?kMv5n|M6 zbw+r3P%r$>4&axlRv;J%gM>?nV)6S=(=4dpI8iwPRp^Gq(jjltvmC)h@BBog(NY{P zL;Qx~vzdt?qI8JMBZYWTpiRa@EM8QYr<32B~G_=zKq+h51R40`JxE2Zk6J2xCYiX?7flu-I(`iy4PkE`_X_ zISnQwGb6kT4>Df8Ya{dm>VVpQf4kUvX>AnM{}CG2*9?I&l5?L?U>RJ>cg#3&J$A_) zNnQ9*t*iyN971*IBtAlgp*$Lba!MSr;#he1gg?o%Qg%H|<*0cJp*u$CKm5aa$@{@t zj{VPQ7f3R&5Jwj7{G9qTIfmsHcEtMRqQj-=tlcs8ioDA@@v;;vw zRN58P)srO3?X55pq%v5(DD_&bFkxU}oo#fqC4v^aw9HI8Jz}&lkTFo-(9n;NB2(CR zmo>QE;eHh?L)Y%Spo>XykIUCWB&E^zVx}bySl2oo~nB4 zwmeL==F8I(VzzkSX)1_ctmF3^esd2Jr>2%mwC&tF(_ww#_ucgNxx|}xMBvnv*G2?Y zJV$av7{-AwT`@aV#a^e$PW8u8vOtquG}UPQ^b4H;h0?`3h!&rYmNqbskcen<>2v}n zDF4}VXd2Di`vf^%Ho9{yK~^iv1~%%iiA{5^X077*`oAW}3^%;nh{W{%dwqRN;%S(q zq7S~f2ww{|UcExxEFhlpEh-69R1`nN61a^~kdmOex^qkX@k2d_jom+qpIo=5J_3m~ zGn^ieHA|_wv;{#B7cXbvzh58V_6UC(So{U_#oJr~FJ+Z~^E~){i3?B1 zACvg$sC7SaIyAw9A(m(LLm}q^7tV2$H91;1wzJ+LYjtP+KiV#=t5={cW#FzuS!(O} zD(;V;y!u#76qZO7{*Up}0LY71K)MPeA!eO^aVJ8vSek{!qFGI_RI(xlHm0H%o}wb> zTCedVoS5<-7gNc1`x6KHoI36|3xZmch2nyMoLtXf_hehQ3(T!#CmFp+A5Y+T2j5y( zB!A3fV!7mf{A}g#j19CK65PiA9~4^u7v&Y_XR{*p+2*-`;A*(^tLGtXs%nw>c^q`) z{=)VaiwFaol)B@fBP z-?8Fh?C&*;!zrivCq-1lZ$vbDHWJiw#rdSwQN@U$Y$b@+$8d=csH3S7C1$4omz(B; z;Oc196eVDO>dA3J0xSkEJ%A5x%W;Qot#Td&YR_mfgQ!caG#TaH&#+-uR_6Ma&+*G+ zU4&d<6UWyQUH@ve0*8 z?S7wMLwEK>;8Lhmg_|~1*7WcMnOA%vSOB45mV5!;k9;X4fW6>TaLm8eGJ|@5LhxJ~ z!-TJx#X@&{o$SwX4g?|G_d@R{3-ha$ym*Se^OiXnpb=);7W zVe^#5*SGKrCD%8h6l3Iw{7Up{lXhX@Y<+>-i(&B5Y$upXGui6{Pl@7XXkCi!0)4Ww zn(fV#4F|o8gO8ZP@G~=$pp@qF%0Esku#{;)E4e-hp`#MA<0=Q$M3$!sq#`{p6}}qk zEyIexw_F)`34IV5&wy7#>Y;|B4nj`Xoa6G!s+1lE*mHzyJK*QChs(&#X%$+h-EewK_ASt)->4vCNb-ba{HJDryKuTu}%VP)sXMi~rfo zEOy~l+`!q{nTrb%Sve~!E3;0FuAZLf&FKa&3ll<&8Rfh5{OvjO%_6bi_>2te>sBNy z&N~dQ48D8RjE_FNHb?a{2p^&{%a`Z~Sih({iv1bLg4)^Vb_2Y${mHjUh;BTO1OKup zd$-EDSR4W&b#j~tO7`*CU`<@x8vNc;Zawr>mgKs~V5vrmVRJ5wo#wlV}PEG@gqP`Lzzie`W9qYEHS=IRJK=eaHlbnT_bOorQR5?ma zL~?(pT#cA=CZQ6G3D+7V1-y#M$ zg^AyBdt$OfGv}v#WAz##7i@+&lbwy3Cx!V31x2={{WwObD47y33r%%dG7}cf$m;1Q!@0q+S;gUT zg%y15%8LB?3#2;DLvJzH5_s)xMvq@m%IV58HIytwr;b_hBr_gDlg0?uohk{(Gv|+$ zo~>kUh%FCCGHvpe7yiLEqLP;!cqeI4Wt|i87iZJ-=O3y#gstW5nF+nJ@;*}X*(?+~ zp(en~@q4U;4i(}GookcQ^OBTXwV_wFR1zW{E+M;--*W%$Ln?M0%IDYgws7;kb|?T$ zH&a5f0pp#T$NhBW{oG|l=NUYST?0jYJjI?5^Dn)4_S(~bLGQ^gU(w}6m>Ta)WWl%l z2<`-}rrYXO|Cx$LzU3a~K}yC6c*_#K{l=xt74@GN>sKh-Wm-ZZzug8mg-BhSoircn z!3SE8!}p`!YPU)vH@d19ze9Ug53ZE;gKf7}d5l;tg@w<}L6cuH?FvN*tM6<+NycT| z%sJoh#72#f)e{a)UyiSkhl$!D2D3AZ#Nzwfroaat^f%^sY7e&TZl?;$aO3fX7dY*W zkOvu38fu`qZ?RhUNhKe*X3fv~N}xY^9o+1F{MpmWRl!eY@7wd&WEW)zm^nwYauTrV zMGDS+7c)gaf5PJBA<^jhQQtN|bn>KfArv_#lKgi86o=wQA@`H>Rht;^?YckQVfi;P z?2bPKJ&||>*2A;=7HWA!sb=?c4s334^tGRj|8LK_i0%ws%3$n8e-H_O@8Wkgo$TT2 zQ@iy9Z0g_!kJqFYBAG1)K~#&Z3aPNm+7rg>qul$iOU$fch&YvRgny2zV+WsW<+OM- z?cENQ9t-k`QbEi1J;g>JS_LCKM=jKs6^rM=zy_D zA=q(i-c@l`E{65Sn7^R!O4rNdVxz5@ZhHBJ@O5_bwEs0ui!k(BEXOfrVX?t&WPU!W z&A5_>i>V=QV1<)}mrQ^~E3E1|rjN_D(~hj-ThR{p{XpQ!(h`&$tGc&PW&}K}ct38nxPZT9j^Mae?JD=^`#L!L_7Z+D<#fZ_XxEwR{g7cdk^UYFikf~%- zz9dqfdP-ypc5Clg`E6FAFUY@L0w^vnc5(91adzg}-9L*P*pQo@1#}b1CF=Ba&^Y3d z;z{;cJRc`)0RI;+cpEh|&yn$hfZEzx&V(wl#UOq5@ZezNv?mk4pO$qNvI4PY{-A8t zlGcI#n7(y3P;?OozFIjzl9<4yU1&vI{zifZFgG{X92(|`>O-*P2JCP#Fp!QnhmvC1 z^fcW|8#iei&EJVB?__E?OY}&mKdEvU^Jo*|55%&-5s1pG6}dZgH-*if$;#jK7_>l| z=;54y(cy%YmzNjaq^Dx6QN*T{fndG)$YY1OBzJK2cGXF;z?P|y>qZanOI_EOQ`anf zhjJN=t=HbBQ9La4!RVFFPPu>F-0vHeNK$A8*T5W^r;Aye4-Q<G`a?I4Z-6?D^d4ULWJudb)Z-s`Z`wVO$r?$#>`9e--iFBPI0;ogwJNv`*p*;O(8>;3V3 zV6Ix|_1y`|L#9)Efi^ucr=Jvw`p>=j}Eq*cWI$qT0TB=Qyzz=$@io z;3s*f#GZW~nycabktOd%cKhp47!R*SBsKvsep!(O#x_ValZd=Ly8V4XMBlX`2yI+F zlKLVU2Re|g=m;xH5OZ;dj!b3i1DwY~Nje``#ZEi#A2G*mEd_6n0`aG;gC>L5hx+6w z^`hlJpU;JAd-K+HR2!eOkD7`OdqYDx0$YIkgDNnI_P0&;Pkf|Wyjg*8&|?PhS5p#%SHIbCBT2uJNc z^bsSS_~lbo%(-&7W6)Fl_@?^~4B6gc!kO?@Lt!?YQD;yY2qlX+EgmjM(p5}cLu++D zy;$&P|KJqi>5_o~6j(ydWtZ)u%+R6ZWU$LW%j0C_HoNcp!#(j%g(Vr|1}fN5w~_ZD zIRn;peNDV0&tVwt}cDV;v!20f|91SWM@nVA;&O z`ozPeFBbJn`^gV_U@&JXZC({S*ZrefGIg9^_uim|A;6stFwH-d2ffD=W)rW$nd(QGlLNZKNGgtd^WIz$z5+X^X$f|kvD?O z>yFo(E?;31D`^)MIQO>Sy@nU1RPSWnbRLLZlT7`_tjeTdWiyq;QHf`Xh+beNS`P%a zlXiX+EPF60);s>GCSeOuKH>9?PNg{8*Ksxp%iKkga|wKdUk+e%dq&A?5qz@TbpU!X zcc?xX{Id^0+In{00Im+dt-<$|(jjZK#4&ieVZwRh+ZnvscPwtgyNT2M4928RVNAJP!vY zNlPD{PmBlp?L;!R-fbBogDNz$70skKx@EEutMoNFKPzV<5SQv9a$BuXojKzNE; z3WHcH{A`rt#(T_)ymTLA13_4Ovc^oh_Nilq=Wl464am5C-YzZ@(cWhYI7fjfXR&wj zsvX-5Tth@fo9bO%%l-f`zY#=(R}Fzq!$?=lFm9l9p#?X+?ayTPy`Q3s1V;1%k48qW zk*7oJss7#mK~t8FQSXKFJ-tKW6JSjgD|`vu$AV|p;^DCCDW(AU`&>2S@jBm3X!;@0 z&Bq!BAZ^WmJCdq}7NVu~B5jYVvtYzAzwTQtR!D;-ip9l7(Z1uQg40pS{_qnZNm;2e zqeK#$(Gy_-X)rR(Bzm1&zb6WI#B+*9eCqrg8b5qH3nhTD`*Z#hL%PiM5AddX+NTen z+fE2)hOq@(T)ni=Ul(V8E!qm)$l-t-1@B{pXf5_xZ{za`feh5t*a2>3<$_4={8G?j zLpAC(M~eBFTgH^b!Gk8n{ESa1al^FX5Ni`Ze0J&w6+8BecB}Jra!;B0v1k0eX$ybF z!GsWp;_0_aDbZuAt)Ajb`?t+g#7}jQ=eD zYCe1H`)b*mvWLzfsH&Vprn=jnEtpQYdpX0ybuQ#xZ)}L$74#bQdC#T(@Tdj1NjEH3 zu-ZV;qETt_V>$0EfO&?R1$owo=_n1%=<_`|r>kk2;?boIVkRX6iz1Ey~;U zTP)l|wW_1Z6s)yNWxI3l$$1xN=6;Jhk}z75%oIiJy>?yQ1V2q%H1C5qT3F`k(t*7X z8|WBTF_FHU8i>N)ljnrK?LwNV?ttpQ9c#{2l*t*W(aT98EkBBT6j@@u_LB8Pfpji6 z@Kbm*Z@F{aF}3FFJlGXUM^2-Q?w%G*1xO;gN50h|wd{8!czuXUIR|~HqF_Jc8*yaw zIHpy+w_Tm_=0MsO+!N6k^VSbiD+S%L>4-J4RjsMx8q7?Nj?NMxL?x(T|8#8seM62b z*N1xv(pu%2d#Z3q#T4I4)KT(4i{D$;XCFvBjPS?>YV`Wxm_6X@+dm+ysytCSlRw&9 zZ)OEM+CBSYg-DOYw7=|gP%;-n7*V^r-T#zN%?ZXEA@KyCP!6oeo$1Nd+ zWBN^%+ur_)h15_wwsL#O?NQ5drx*74ltH~q>8dJvZX(OzcW?aXsXt?XaE!mP+8OKX znEfs~eJev;J;qq~*Ov>_O>YeU>qA-jMIy2TG`_h+@^*+nybR6%7_g2Y@Bcp!7<2DK z?RyS4fZw(Ig=pD}1j-ZjlGTfA{P_H&1a5OW9grb?acnZSABSuIcs3Gx$JN+w)q|08 zkSJt%xfuX<(F945!G7Y6Jqyg>bKV4id6yNaSC{wwpSmzshlV4$2p-T%KaG5I9att+ zb1utHo9sn08npSugw^Pl@9^NxgE;-O#le*MYVh)}rLM+MlbVOa5ASB<`WtQ%*a&n* z1EMlNyX6N$?bEscGz!ofvBzyKCmj_cWj}1eH#P85yloS)QEa9&+$~*a%6QLmaE^{< z=c2IyG>ADji$FQ`W#@t&+fs|IWO&|P>PCz=ihSqB(rr80ccp?9OzZP03`-C~pqV;& z(#4i~`Yo2X`(8BcbsR@i=COd_k^7`TBstU3Wq1=d09*UR?Rr>#q33$IaC_D-&5hi6 z5R9L2*ux!c8nR40(7Yus_V>5kY%ouP0?Z`GEeuC8OCi*&kK@jTmi2|@Mi#_j0SVZ?%gvuoP;rC%1 zi{9XuNQq3BI*O#rwTh7f9y zTqqUM@jpoq(L?cn{cY)m5__=rJQ8g9Pt?Xb_E@GVl_k>gHC=sfv?^r5Ct{0KqE0`> zg+{!leVrb@#9su&D+|Y|-rvZAr?&)=OA{d?dKUt9JUcZ=y0D(x)$ouA`taKYys@9! zxa7-dvLdy{@1-`9B`_;if4%;-HYUAcJ46^mz(CMgPg!gF(61qO1Q%l7vfU#2xVos; z05NRxSZjl3oJMbRyqnO5hy7G%Ec<0P(F@oeqYm0sD~vHY+<0sC*SDg05g&pE+bkyF3SJSrlf zmUl)$S>bWCsJ1Pdfa{1(hb~dodoGuK5h0-j3(X5b+$&0Nq*`c&dm8v;tK*;ehdp1U z(m`WXkrhQN&yi5qdn_K6GB|?AO#Js70qG6l(oYLK;(U3qv6Bs+OYC;cSBE^A9#-Z< zH~vEve_pMV`7<`@G)XWxHA#RSlTGD&MXKj@4oxhb9qXo~o7ARxfDM_IhKcFGb96)X z-#pZc7!+ww9BwH*e(AHQKCyj+*e(jB+G0#%4SbDEn*l}0bupJx)){lbyQGwuanmhI z$T2Y+(jhF$lCDM`ANOh;l_aI5n^~AEG9+WF0cG@3Bt>7z=K~!Ze>=mn5Q@cvLA-=z zCkNyJwbkR$pxNq+bQfr5J`_WM!xKblAE=uai_kD4e@JT9fVV*J+Zu}w6o_pyFXU?C zsJ-Fgcz&`0ZXi@%8YikcGLm$BQybpmz5CEn#sXh#aQ&1XE{d-_lHkdgHI^NH9~gp* z1=KumVwGngX6}O$CE##AV|~V>E~-iRxOY9G)G4orV_1IFUWA^0Wp8wEy9O8KQ*=*M zm9y6R*gj+(Bwo;LhepR=!RpmQl8FoMEnfRT7yd%#!VON=Gz^OdDSnN^Lq6izv&hGSw+yQRXR)8$gUu5%AH6<(9-?ymnDP|RN-c<= zmyH^Ra?in&VHhk&Ox1!-qP)=W2*Owkya&ZwP+JdomJCI+$1lmemJ#v(kB&S#Fivh> z<+A5Kc^~>7R1@#$VOXu5edYTy&y5&Mu7#f!v!lZTIaCK(GV4Z4l7-;Ib*;TrIhu!uV_*vH2+J0Y2q!2Di`iq}=>V|7r1{}Y6!s{FhPag89QkL8lsG_{tp2W3G!Z5pARw5?s;4)h&#atXT@}UE z8UoG}5C|GKd+f+4?$v>!MNdziG#8Xg#b0dBUc6?JshVUtIekWt0#nyt(uqg`Zp4ba`h99|! z6_)2@h3DjTB<>T`H4);?l<3Rno^;QoOnwkz*TQhH{WrDlbzFtTKQ z9Bt2!wiHw=Jc+*wzfAdF?C_H0V9MYCcxU4-51tM6UmTtFu48Phdzc6%5b~iL-MzZO zOR{oF+^pW6?;v!4+!~66^Xljx*ExufSi=g!5T`6EMtVsm%(UhWiy zcx?CP=bY-iI6x+8I=mTK|Gqw9+Ty7wO(-h%6Nbzi6P?=<{n3^CF^11>J76VJTPI`d z_U!b>?^&r3<5z*r_t>O-6t#Z#JO76T??a7 zrT`+M?Du}YQjM!boRxYr4PdY!X^fLh0_VX^#kOze1PIBQEm@?#vSzIKmMN0wWZO7S zx!7fv-05?^fr#+ZbNc_9<7`LPe6B@OQP!}hRn{>glX}ems~A&?O~nxr3$Btrwn)h7 z3HQS~6uZe&P}@mJa&ppaSn_0rc;Gjtv57>#vier;Eb%mtZH}@<^m?!HaWJ00ii_W* z39m6|lJb7C+Pi|Og2u`haZ~;Sv~oB8e*moiB}99~HRFWz_cbdVAigyP$Vn?n6-yX@ F`yX!qI|l#& literal 0 HcmV?d00001 diff --git a/doc/source/using-ray-and-docker-on-a-cluster.md b/doc/source/using-ray-and-docker-on-a-cluster.md index 9ae39d178..4e7b7a52d 100644 --- a/doc/source/using-ray-and-docker-on-a-cluster.md +++ b/doc/source/using-ray-and-docker-on-a-cluster.md @@ -1,4 +1,4 @@ -# Using Ray and Docker on a Cluster (EXPERIMENTAL) +# Using Ray and Docker on a Cluster (Experimental) Packaging and deploying an application using Docker can provide certain advantages. It can make managing dependencies easier, help ensure that each cluster node receives a uniform configuration, and facilitate swapping hardware resources between applications. diff --git a/python/ray/experimental/sgd/__init__.py b/python/ray/experimental/sgd/__init__.py index e69de29bb..005b3fff0 100644 --- a/python/ray/experimental/sgd/__init__.py +++ b/python/ray/experimental/sgd/__init__.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from ray.experimental.sgd.sgd import DistributedSGD +from ray.experimental.sgd.model import Model + +__all__ = [ + "DistributedSGD", + "Model", +] diff --git a/python/ray/experimental/sgd/mnist_example.py b/python/ray/experimental/sgd/mnist_example.py new file mode 100755 index 000000000..8c2fff213 --- /dev/null +++ b/python/ray/experimental/sgd/mnist_example.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +"""Example of how to train a model with Ray SGD. + +We use a small model here, so no speedup for distributing the computation is +expected. This example shows: + - How to set up a simple input pipeline + - How to evaluate model accuracy during training + - How to get and set model weights + - How to train with ray.experimental.sgd.DistributedSGD +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import time + +from tensorflow.examples.tutorials.mnist import input_data +import tensorflow as tf + +import ray +from ray.tune import run_experiments +from ray.tune.examples.tune_mnist_ray import deepnn +from ray.experimental.sgd.model import Model +from ray.experimental.sgd.sgd import DistributedSGD +from ray.experimental.tfutils import TensorFlowVariables + +parser = argparse.ArgumentParser() +parser.add_argument("--redis-address", default=None, type=str) +parser.add_argument("--num-iters", default=10000, type=int) +parser.add_argument("--batch-size", default=50, type=int) +parser.add_argument("--num-workers", default=1, type=int) +parser.add_argument("--devices-per-worker", default=1, type=int) +parser.add_argument("--tune", action="store_true", help="Run in Ray Tune") +parser.add_argument( + "--strategy", default="ps", type=str, help="One of 'simple' or 'ps'") +parser.add_argument( + "--gpu", action="store_true", help="Use GPUs for optimization") + + +class MNISTModel(Model): + def __init__(self): + # Import data + error = None + for _ in range(10): + try: + self.mnist = input_data.read_data_sets( + "/tmp/tensorflow/mnist/input_data", one_hot=True) + error = None + break + except Exception as e: + error = e + time.sleep(5) + if error: + raise ValueError("Failed to import data", error) + + # Set seed and build layers + tf.set_random_seed(0) + self.x = tf.placeholder(tf.float32, [None, 784], name="x") + self.y_ = tf.placeholder(tf.float32, [None, 10], name="y_") + y_conv, self.keep_prob = deepnn(self.x) + + # Need to define loss and optimizer attributes + self.loss = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits( + labels=self.y_, logits=y_conv)) + self.optimizer = tf.train.AdamOptimizer(1e-4) + self.variables = TensorFlowVariables(self.loss, + tf.get_default_session()) + + # For evaluating test accuracy + correct_prediction = tf.equal( + tf.argmax(y_conv, 1), tf.argmax(self.y_, 1)) + self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + + def get_feed_dict(self): + batch = self.mnist.train.next_batch(50) + return { + self.x: batch[0], + self.y_: batch[1], + self.keep_prob: 0.5, + } + + def test_accuracy(self): + return self.accuracy.eval( + feed_dict={ + self.x: self.mnist.test.images, + self.y_: self.mnist.test.labels, + self.keep_prob: 1.0, + }) + + +def train_mnist(config, reporter): + args = config["args"] + sgd = DistributedSGD( + lambda w_i, d_i: MNISTModel(), + num_workers=args.num_workers, + devices_per_worker=args.devices_per_worker, + gpu=args.gpu, + strategy=args.strategy) + + # Important: synchronize the initial weights of all model replicas + w0 = sgd.for_model(lambda m: m.variables.get_flat()) + sgd.foreach_model(lambda m: m.variables.set_flat(w0)) + + for i in range(args.num_iters): + if i % 10 == 0: + start = time.time() + loss = sgd.step(fetch_stats=True)["loss"] + acc = sgd.foreach_model(lambda model: model.test_accuracy()) + print("Iter", i, "loss", loss, "accuracy", acc) + print("Time per iteration", time.time() - start) + assert len(set(acc)) == 1, ("Models out of sync", acc) + reporter(timesteps_total=i, mean_loss=loss, mean_accuracy=acc[0]) + else: + sgd.step() + + +if __name__ == "__main__": + args = parser.parse_args() + ray.init(redis_address=args.redis_address) + + if args.tune: + run_experiments({ + "mnist_sgd": { + "run": train_mnist, + "config": { + "args": args, + }, + }, + }) + else: + train_mnist({"args": args}, lambda **kw: None) diff --git a/python/ray/experimental/sgd/sgd.py b/python/ray/experimental/sgd/sgd.py index 2ad74710a..a66396068 100644 --- a/python/ray/experimental/sgd/sgd.py +++ b/python/ray/experimental/sgd/sgd.py @@ -91,7 +91,9 @@ class DistributedSGD(object): RemoteSGDWorker = ray.remote(**requests)(SGDWorker) self.workers = [] - logger.info("Creating SGD workers ({} total)".format(num_workers)) + logger.info( + "Creating SGD workers ({} total, {} devices per worker)".format( + num_workers, devices_per_worker)) for worker_index in range(num_workers): self.workers.append( RemoteSGDWorker.remote( @@ -143,7 +145,15 @@ class DistributedSGD(object): out = [] for r in results: out.extend(r) - return r + return out + + def for_model(self, fn): + """Apply the given function to a single model replica. + + Returns: + Result from applying the function. + """ + return ray.get(self.workers[0].for_model.remote(fn)) def step(self, fetch_stats=False): """Run a single SGD step. @@ -176,7 +186,7 @@ def _average_gradients(grads): def _simple_sgd_step(actors): if len(actors) == 1: - return ray.get(actors[0].compute_apply.remote()) + return {"loss": ray.get(actors[0].compute_apply.remote())} start = time.time() fetches = ray.get([a.compute_gradients.remote() for a in actors]) @@ -193,18 +203,18 @@ def _simple_sgd_step(actors): start = time.time() ray.get([a.apply_gradients.remote(avg_grad) for a in actors]) logger.debug("apply all grads time {}".format(time.time() - start)) - return np.mean(losses) + return {"loss": np.mean(losses)} def _distributed_sgd_step(actors, ps_list, fetch_stats, write_timeline): # Preallocate object ids that actors will write gradient shards to grad_shard_oids_list = [[np.random.bytes(20) for _ in ps_list] for _ in actors] - logger.info("Generated grad oids") + logger.debug("Generated grad oids") # Preallocate object ids that param servers will write new weights to accum_shard_ids = [np.random.bytes(20) for _ in ps_list] - logger.info("Generated accum oids") + logger.debug("Generated accum oids") # Kick off the fused compute grad / update weights tf run for each actor losses = [] @@ -214,7 +224,7 @@ def _distributed_sgd_step(actors, ps_list, fetch_stats, write_timeline): grad_shard_oids, accum_shard_ids, write_timeline=write_timeline)) - logger.info("Launched all ps_compute_applys on all actors") + logger.debug("Launched all ps_compute_applys on all actors") # Issue prefetch ops for j, (ps, weight_shard_oid) in list( @@ -224,7 +234,7 @@ def _distributed_sgd_step(actors, ps_list, fetch_stats, write_timeline): to_fetch.append(grad_shard_oids[j]) random.shuffle(to_fetch) ps.prefetch.remote(to_fetch) - logger.info("Launched all prefetch ops") + logger.debug("Launched all prefetch ops") # Aggregate the gradients produced by the actors. These operations # run concurrently with the actor methods above. @@ -233,11 +243,11 @@ def _distributed_sgd_step(actors, ps_list, fetch_stats, write_timeline): enumerate(zip(ps_list, accum_shard_ids)))[::-1]: ps.add_spinwait.remote([gs[j] for gs in grad_shard_oids_list]) ps_gets.append(ps.get.remote(weight_shard_oid)) - logger.info("Launched all aggregate ops") + logger.debug("Launched all aggregate ops") if write_timeline: timelines = [ps.get_timeline.remote() for ps in ps_list] - logger.info("launched timeline gets") + logger.debug("Launched timeline gets") timelines = ray.get(timelines) t0 = timelines[0] for t in timelines[1:]: @@ -247,6 +257,6 @@ def _distributed_sgd_step(actors, ps_list, fetch_stats, write_timeline): # Wait for at least the ps gets to finish ray.get(ps_gets) if fetch_stats: - return np.mean(ray.get(losses)) + return {"loss": np.mean(ray.get(losses))} else: return None diff --git a/python/ray/experimental/sgd/sgd_worker.py b/python/ray/experimental/sgd/sgd_worker.py index eb06b09d0..3dd0eefca 100644 --- a/python/ray/experimental/sgd/sgd_worker.py +++ b/python/ray/experimental/sgd/sgd_worker.py @@ -48,23 +48,24 @@ class SGDWorker(object): device_tmpl = "/gpu:%d" else: device_tmpl = "/cpu:%d" - for device_idx in range(num_devices): - device = device_tmpl % device_idx - with tf.device(device): - with tf.variable_scope("device_%d" % device_idx): - model = model_creator(worker_index, device_idx) - self.models.append(model) - model.grads = [ - t - for t in model.optimizer.compute_gradients(model.loss) - if t[0] is not None - ] - grad_ops.append(model.grads) + with self.sess.as_default(): + for device_idx in range(num_devices): + device = device_tmpl % device_idx + with tf.device(device): + with tf.variable_scope("device_%d" % device_idx): + model = model_creator(worker_index, device_idx) + self.models.append(model) + grads = [ + t for t in model.optimizer.compute_gradients( + model.loss) if t[0] is not None + ] + grad_ops.append(grads) if num_devices == 1: - assert not max_bytes, \ - "grad_shard_bytes > 0 ({}) requires num_devices > 1".format( - max_bytes) + if max_bytes: + raise ValueError( + "Implementation limitation: grad_shard_bytes > 0 " + "({}) currently requires > 1 device".format(max_bytes)) self.packed_grads_and_vars = grad_ops else: if max_bytes: @@ -182,15 +183,28 @@ class SGDWorker(object): tf.local_variables_initializer()) self.sess.run(init_op) + def _grad_feed_dict(self): + # Aggregate feed dicts for each model on this worker. + feed_dict = {} + for model in self.models: + feed_dict.update(model.get_feed_dict()) + return feed_dict + def foreach_model(self, fn): - return [fn(m) for m in self.models] + with self.sess.as_default(): + return [fn(m) for m in self.models] def foreach_worker(self, fn): - return fn(self) + with self.sess.as_default(): + return fn(self) + + def for_model(self, fn): + with self.sess.as_default(): + return fn(self.models[0]) def compute_gradients(self): start = time.time() - feed_dict = {} + feed_dict = self._grad_feed_dict() # Aggregate feed dicts for each model on this worker. for model in self.models: feed_dict.update(model.get_feed_dict()) @@ -219,6 +233,7 @@ class SGDWorker(object): fetches = run_timeline( self.sess, [self.models[0].loss, self.apply_op, self.nccl_control_out], + feed_dict=self._grad_feed_dict(), name="compute_apply") return fetches[0] @@ -227,7 +242,9 @@ class SGDWorker(object): agg_grad_shard_oids, tl_name="ps_compute_apply", write_timeline=False): - feed_dict = dict(zip(self.plasma_in_grads_oids, out_grad_shard_oids)) + feed_dict = self._grad_feed_dict() + feed_dict.update( + dict(zip(self.plasma_in_grads_oids, out_grad_shard_oids))) feed_dict.update( dict(zip(self.plasma_out_grads_oids, agg_grad_shard_oids))) fetch(agg_grad_shard_oids) diff --git a/python/ray/rllib/models/catalog.py b/python/ray/rllib/models/catalog.py index 4ffecf8b4..332158304 100644 --- a/python/ray/rllib/models/catalog.py +++ b/python/ray/rllib/models/catalog.py @@ -189,7 +189,7 @@ class ModelCatalog(object): seq_in (Tensor): Optional RNN sequence length tensor. Returns: - model (Model): Neural network model. + model (models.Model): Neural network model. """ assert isinstance(input_dict, dict) @@ -241,7 +241,7 @@ class ModelCatalog(object): options (dict): Optional args to pass to the model constructor. Returns: - model (Model): Neural network model. + model (models.Model): Neural network model. """ from ray.rllib.models.pytorch.fcnet import (FullyConnectedNetwork as PyTorchFCNet) diff --git a/python/ray/tune/examples/__init__.py b/python/ray/tune/examples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/ray/tune/examples/tune_mnist_ray.py b/python/ray/tune/examples/tune_mnist_ray.py index e806a1a68..e56ebd10f 100755 --- a/python/ray/tune/examples/tune_mnist_ray.py +++ b/python/ray/tune/examples/tune_mnist_ray.py @@ -42,7 +42,7 @@ import tensorflow as tf FLAGS = None status_reporter = None # used to report training status back to Ray -activation_fn = None # e.g. tf.nn.relu +activation_fn = tf.nn.relu # e.g. tf.nn.relu def deepnn(x): diff --git a/test/jenkins_tests/run_multi_node_tests.sh b/test/jenkins_tests/run_multi_node_tests.sh index 05fae9346..567e85140 100755 --- a/test/jenkins_tests/run_multi_node_tests.sh +++ b/test/jenkins_tests/run_multi_node_tests.sh @@ -326,14 +326,22 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ python /ray/python/ray/rllib/examples/custom_metrics_and_callbacks.py --num-iters=2 -docker run -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \ +docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ python /ray/python/ray/experimental/sgd/test_sgd.py --num-iters=2 \ --batch-size=1 --strategy=simple -docker run -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \ +docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ python /ray/python/ray/experimental/sgd/test_sgd.py --num-iters=2 \ --batch-size=1 --strategy=ps +docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ + python /ray/python/ray/experimental/sgd/mnist_example.py --num-iters=1 \ + --num-workers=1 --devices-per-worker=1 --strategy=ps + +docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ + python /ray/python/ray/experimental/sgd/mnist_example.py --num-iters=1 \ + --num-workers=1 --devices-per-worker=1 --strategy=ps --tune + docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ python /ray/python/ray/rllib/train.py \ --env PongDeterministic-v4 \