From dee3322ab0a9b8ebbd8a5747120781f64325346d Mon Sep 17 00:00:00 2001 From: krfricke Date: Sat, 8 Aug 2020 00:32:49 +0200 Subject: [PATCH] [tune] Ray Tune + Serve end-to-end integration example (#9908) Co-authored-by: Kai Fricke --- doc/BUILD | 9 + doc/source/images/serve.png | Bin 0 -> 27154 bytes doc/source/tune/_tutorials/overview.rst | 7 + .../tune-serve-integration-mnist.py | 687 ++++++++++++++++++ 4 files changed, 703 insertions(+) create mode 100644 doc/source/images/serve.png create mode 100644 doc/source/tune/_tutorials/tune-serve-integration-mnist.py diff --git a/doc/BUILD b/doc/BUILD index 3ad15c58f..3d249da1b 100644 --- a/doc/BUILD +++ b/doc/BUILD @@ -54,3 +54,12 @@ py_test( tags = ["exclusive", "example"], args = ["--smoke-test"] ) + +py_test( + name = "tune_serve_integration_mnist", + size = "medium", + main = "source/tune/_tutorials/tune-serve-integration-mnist.py", + srcs = ["source/tune/_tutorials/tune-serve-integration-mnist.py"], + tags = ["exclusive", "example"], + args = ["--smoke-test", "--from-scratch", "--day 0"] +) diff --git a/doc/source/images/serve.png b/doc/source/images/serve.png new file mode 100644 index 0000000000000000000000000000000000000000..ea0321c26169e99d83affffd8080b76b38308d17 GIT binary patch literal 27154 zcmeEu^;1=C*zVeNcL)e5O1DTzBO%@0-6h>@KsuBVkS-DFkgiQjNJw{gcf(oue&6}& z{0WB{hM6_bJbCAJUpL{3@*gnKNYMZQz?6~{Qw9JyO#pzjqacA_IGk%u!9S>ul3LE- zAKs^5$dPHBH29IkMO@QG#lhUg-N?xdaCdiSv9h;uHZgKEV{ve@NIl>u1%MZTl$eOB zN7~-Jr;qCRwb0R!Gc8$VJ=J>}6r#zA9SxgtYs3Iqs+9>m;@=2kM0SKaw`w*eLZcJ* zHLf-`T@7DQ>k#aArSTLU*L4VOh<5`cOlj&IAOl>^nTwR%zj|N4v@k#TW1D%o+tshZ zvYF-?CbiM4l>v`~`~QFbA2;AB^Oqa|zu!+rHTIoR8wsi^ARLJUiO8%~Y5}wePVN>` zBbk%x(X%^kHp5tKZB$fLCz|V(m?m%4`4>Y}BbR;~`CShIcZk;WBdQ!9&)Hht7K{VM z#l+TPNnB&+(XxpPm)--7`Fs?Bf9UUOlL%TC110JT7v%WcwAlsr;Kkf`fjH0OiR556z9J^x0a2IzD& z=AdB1@yx7e+X-unopI{mB9&tf+fhxZPw^sAnE>R7h6aGj^24?m;cvp|w!fK;{Pz0) zcA1dXXBMkGJ0Sref9Fkv+lTX%hH@$v@0T||*0?RNVU0elXS;W6&D;~}u*pG5>^sNs zr!YU*F*VoKsM~6E_jCL^2A<=4J=4WxOAe?MJw`4LKYfJaue{xbakX2Y9d)08@2OaL z{V8QI{ydL3rI+shQxEcAQWx?6ZU6Uhxa)HGeBh5*A15<-4>7yO8qc=8bC!?^1-laM zA=znu9kH^&9i!fuSFn@roxj#$1*ymDy#E9W$?HN_0CfrWI<#)_VmVIj^p0#U%0{wK zS!wacHTZ#4SW8(H#cy19zjbW=w-YHo6BSj?%KVMk|B^(!C#-o#I;$kZYk zhvU};1b=KjOp42Lx7>gq)MH8&(-DNvD^K>C7cOZk=ZwDd0pLbuUBNpwP}7PttYtRt z-IJ9o1q@xs1v4qm&9`;w>?I3w4FmO_l>{-+T86+BK!Y_RtV*p&>G>R1fqLd;VAuF} z7eCOx5Jj9tV`~U+`xV0`=R^WcipK`wzq^AkK&fU@{7sJPG7WbuXH%GQ6lfvhnU}+& zTNGG?w{;Rr$UK!P6CAXP_YxOZDQ&u+dq3xk0nPCD{+jdM>117qut{{QdH@;zR%MYM z&izjd$U5+E;*U8>y9~gp>@50Gi5hxeiU3Bhvx$r}E0T*VoZjF!LhE;PRnW@k{~kl% zeC~Q9gCRV@^ouuX(8Fv~q@_o3L0ocV)h0EkgpT=ZFejOP@B%X2k zH&^9t=_)GFOo$9P&Enu|404}zlACvBXMk)>KdtgB;Xzaut9ZZHF!+KtR50>7;1MI*rAG@E# zlOi<1^`Fw8E6NmNrn#!E_1ifPeaX!bwgH(~|pOHa$QSLG-qWIi+ zLOYS0CG9Tp2ju%V&HYy7qi~~L&kUsa;D2*La4LwNaaD^)S-4kD^j)jWW_iUMor*Fh z+3^mKLMt(zZiotCNH+N^k5EKbMBB3C%Kr_<(tZf(l$|ML4%d60zWOwg(n6}MMn3{@ z`|VeCz?Yt7(%dAG2Bj8R?}=yrZz7d|>AlFll#Qh;JBrYFB{XznY@j^Uniht#r|Y!F_9=nzznzDYRO1yN z^Tb4*9yLi(!r$ab>5yGjP9B@0KZ!uXq_xdB9NTzaNdf7>hdy@W5>BgpSPR96IL?2| z=zF{E%=!U*Byabh3*?+pm#y}SSR)n#YrB`HAO@crQ3eoQoV266)8e`{2UNsw@=eiFc< zHaSmAyBp5iM`0$}|Hz{i@TAP^@5oL5O25!)ih12Za(`=rm~M`06Z22bJmtn6kz^83 z6CpCbX>D--)^pD>AMa7)6^MB~>YDsk8R}jB&p4l?uW(eEUr3gT_Bu*%_$Cf5?6|&| zhW=ZCld!5;jR+$N&pB!2uR(Fqi^U{m$djka;iqMr)%pIVLcb?&IqKm)xJzl!@V`w7 z6#Xeqmp|r;8Q=sxB^eMuns`xzYWBaoa1NZ3=*aif`95b6ZdHAI1iH%ex5-TZb7Mv# z%Zwp%w0}VJcJwwGgF1|aT9e9Sd{?s)@xMF6&fr{zPgbL^t#MmI0qu>bs!vpr@nYnx-2yHp!IBYqogeL49Rs(0oENd*=RG zZ^`oR;%o|EV?B?Y+BVZfO{N>yj(kdA&(%+@-lVcb|6Gh`SzXZ;Dlm^ESHdDQbjF)8 zKs6qMIXN~Fv2R7E_37VjH6ssBo1jBCqe+Vpnxko)RmXy{k&eXUYjSRqL5cp)WC1|) zP~KO-0A;Lt@QgP_HY;b5!`7Nu_}~(}g@4u-Mflf?1`McHtZYTu5m(9?BX%<<$(JgO z)H1Jv?!_G|VXcsVCxU($N;zRcD8fipD8dN-eEK%)E&lmEARM5j!*t4%R{U?Wg3uXl ziY6Nw`t#d(dK9|jVzydbpc-H-^@scs zs#5YuPdYYb{P!Nsnwh^Av!UA|`y=WYmm7W5!vPIPxH%^(t5^|2T*y7Gn?{makx%DQ zVr3*&*eBQ;!ryW_`jMV;Ud8jZwzU}x75Ar*4>=T~z9 z)01BZYGk62ftlB7k zCPpCv!9KRrHzZIyv|nyBBa;)Lk(2UzMMKqR|Fxg$!%U4J=4mgA+PHJ7O`rYc8MBYOCC2BQN9Ot`|ordW6F*%4W%6d_hkM{Lbx7ln~h`dKg8ptq&15gPT{i zOP^@U+Cv?J>xamezk$smomaa`n2W5t1P|iDB2#DVfJcP2>}NUs8pxM^(?OFjsQTzf zNf*e}=0?t+XWi0{DINFclRjPwp!E~}->`}|J4Wlsp&voE{4d>;WiR(ZhX|m<^gC`= zsCLyVnudvWZl3HQ_O1;+u330J=8ny7*c~D)T*~E9CSXXA58uCg5Uu+8LC{<-ECHeM z;T$~w=k>o!!^wu2!PVX}U61!&e~*3J^Ij~0eS@puaCGpu zveJNpQ8Ksf7D-W{*-#^qCK@PXUgpxf;&Atf0}AN&m)Q~}qNzg}&n~EHbmI@WFPrcR z&?zBH7B~Zv?o9Ao2y1SPJB`-s)85UQ2Zmu`f30Rnnd#7KC(zF@h(oFn>(q*4WYh;U zXExeBq)^8`w4jWqe%Xr4$`!T4bX@%Ujz+V?@ZvN@86E>}%yu_3eYUI5^f^m%tAq-v_Z0KpK-Qk@p&CBa>ag{n|la8Z1 z@wW9$BFwTO=q1(3Up8uu$86z7z+b)V&EgO8&n3G2)g0sO7Ee(K;z(dwQJ)X)^KUDT z>`K>eCUVupB^YV0@x9=iF8x-~zS<-L68d&=*8I8;=uXK6$j5T5?> zNQN`5fMorMrqwp6TYz8*OdD5=vTASAjh<=tS8fTZbE56S&)nY&8cg0oVb7Ui7yxp6 z`sD={WP=mw-rvcnxTzDN-)ud5;vLlYGhlCUH;uR?SI|V0cwd2OlRDLa2#A%VLt_h{ zVYjsa(#sEzvl_mF21~J?_+0h;EVxSA)bYBIlal zlcsH|aBC2g;~fOe!;iuAGaQ+7o-p|;!k&h>BYLh|p8^llmd8x`&*EYUkU05LFEwlI zvFbF8SUzBHMe0k(?mO|lvyG(%vQcu$8k4~FxRaa2N5Zc*UuHI4&|h4N;PP6pcJ_c- zy4$OoJ~OiJ0zIFU!hujjUggHMTl@flAM8Ba^xVd;sHBsgRLbV_n5bjEQIcQpxEqIP zYClu-{Azp44;LjwKfap0{kUcRtIuj`S@%(@rw@L)K-?uq?sn!?#%finJ5#AJ z5ZARbSE3fT#-{V4#Y%1*U*C~~iz;V%#oXz9k==md=9kGCHQY4>nm!lqqS=)5#eo}H zhEm4;Mg1pK^@zS*ZF%BYzSz0qVIpDwxA|iB+DhvV?tC@m^sUX~pWokF=#YBhwmg#-1k>k{|ZPoB!o@E0WkQ0E9mDq_yRY!w! zE#Y{1(IHvb5;mt&aMK)WeOFS7oCp>6?XM_2@rJZ~5xblu3^uUBit!&hhlPdle-U#s zE9#s6!};AcyhUBSfI6$(4`quz{Nh}b_C>9Ys^6fELsJI=qm)2v_8n$8F^-+eR?ddu z^t^uO*Qh=+Zh~1889&rEiV6MoEBH1sf%w6$=HUJ zwcX@#zrBQab8DEqf90Qb7TMLSws zeXAsV6G`?uoDSdYz0=>*s$*-1jP%}LViE)*3;v`Fr z3!P##ddG#~YD3(=zJG2JdG}fP626=@Ewhz|ws{fx2#uhu@Ov3ti$#MpEx(O6ckI*k zL#nH-OH%Q)ke9*`8U)6+Sp_&}Tc#9>(G~WH_%RLRtSEzZ=btk*|Lp()fhbUkMb`H6 zcm96)5kOKyYae`?U=7f z-QZFIH%EwycN4#w86L2ed8*#=_@yR&^fa3|IZ|Z|`2j?6?ER%SCZE$3I96j_rJ^G3 zGm^<*o%8L?zy##k_u|-l3D=RhCw-i?mnM!%74KdLurRa#4xdiZg}|O3mWqcQmX#FU zO zU@jyn0#Cm51^PGL*x6)55}xgv%MV3M??*4tbnC6#I)5}rS@A?Ovovc%R&~bhWLS^i zP*CLr4A^=6EGRJ<5RiOZkCr&$O+vg*OQVU@fW|urUtVN%nj~oJf61-i%`vC0DkbZC zg#_fPe97UVesa_lq2gj^lmfkKD;yfKrN3XvmWL?n6sf~iX)u%z>9lSg*((G|2R*?C zS#=BL^}@x69;7(9N}u(6$9qHcl67M@lMOMKztNYB0#{&UT55?Q@6|U7{afHHc03LY z9FP+Xq`UP<#FMg!iAl-O)Z=x*(P*r3(rAj7cVeY*vs`@-&CZ*jDg3DO(kaO3v_f2c zIXnYNOw5ycdabacyLF(yE)4X3e5w>)7I}qf=2Lgs`8%krU?PdM73H>w0NZMy#7*D) z$Rn2UV}q+nb!9h z@rxz+Kg8>XXP8a}_Yz|AfvXsX`!*?wc5RGMKa7Lf9lKb?T)EZ9OUYEuim>0Dpq*kU zFme*_h(?{jD){XZ?*=zlpCc_>ly1=sMnqMBK8IBiSCn9=n3$*(SG{%VMe2jHB&c@ib;cklf5nA+{H}LbuZ0boom#=a43eXYAvWEk|0eGJb@w8XB@_sTphqN95y< zaE8cXXVyrd^J9jDIn6xDMvY5UOHwbP8JGkH;b`8H)cAO!q0RVc=|?8{p?57TE<(nVI7htUi- z#lAxQl{C{-jiT(zVv&EziDT)5yMZD()!K-@Zu_O`4>r<5vLBcP#KjKb4YkXY!Ce!8 zyH?KK{b+kklAhl$&M;wP9QUd4VBeF?%JHw=;pYQ7+f3#!tJ%pRF@rzLndpF#jG;)W z82N;k2!Z2c7w z>5FwzifH)}a3}qP3z|NPe$@HU+id70t`*~5j>O%ar{u1*NmK^ktIA+};h@1Otb7Z>^YyfACyG9)7P?nXupqk@n3oT1-s~=o$$Gk*k2s<4HyF^V0 zTie?)!E~?aTS%*peLU$-=1ncCyyA+)E_7Yd6~~`5HS&F%d_{^i_wG%YFx?!Y1+P!2 zprwDU<$X?v+>Xe4+DXIja4J!a3XOn>6{Cu?p6Jv546>LhWzJd=U%UO-MPWy!29uY4 zL~y?4xp`~FAc7QSap6s^QH9y)61j5LP1?F_hbOk(MpE2Hau0nbWc@j9${+!wzcM=U zLF^?Uud0(#QsCfO$UkPSJxarz#`vz?{Y^cUbkzHe1Q(Z5dhBM7!8`#_LCin}vG~TD%eNYNh7glP<>N2))9>aVEj@c_jEMgZS zeb3uj@4bTgxsW3i@6DA^*jj`W0e*2?^|!amMUIt|0j3PW`6muS1ck14VVQRN)M!P}}Glov?DZL~k6p(`=LZ|PGo5NSeP88Py&6jG0 zm3!5aA~FWS^WiGsVo~9s`NPAQk>n;pYw^Jwv{I;Zqzi=N9+V5r9PW_151ltwCg)-b zhu-{8p39xcr75n{VV5#&NA1LF)9xt)KYPuSg8_P&366Lj1^f{MV2ZZwAf0N07+omd zaWZV{Ubd9ZfUS*GEURT*vw+`d_^tMwUga+Q=3cA@;XLC``;WpUg^nX-F6ak>x`uaK zk440+in!MG>Ms}N%4gABmYbf^4o_TsC7Kksp6)ojq9s~}D+)Z1N1QGbG&%$hj7GNl zoZ=n?8ypa|mq2R}f>V5}ssl0S24j5;QB9Ipio_4h2*enNMSTnfx}#ueq*~|Ipf*K@ zb=Q$YGHX&19g9uz@nc`fv#?4zPv3re#d;MeE(OLS;&@?xO69G+wbkjJa^qY$voATo1px5_Rem7GOUzD^jA$>*UfOzWil;C5R_ks=CHc zqmgi!Qq=#j0?z<(spEeO%0G#0xtk;0cf4W+CX&eM8po@7WX3GQtzD&?i zU3B4(D5|>=9kG*epbT$teTu(fZgA4dBM`1W?nm<*)77)CWc^IHS%Psx8EPm9kXKfa zfJ><)8}369pHXW9-7bd&oM|Jy5Ihb!>6sZ?d7lRkO-P3cM< zc40b%uFEqZ+)eGWnLSew(&j2DP@+M3FR`r<7Zyur!Krq87_=fm9I}HqQV+go5?k9$ zfSdpJD6qh8Rm8A(%##*{Q{ZpI3ajh8zPI@y<8q7q2c4$G0qrITXRizv?dafXl!jIs z(OoRJo_&NVz@Z|lY&l#^K4T)N*Li|2b{|a=&=2us41U=l=3;N+It_2%C6<5+9t8#h zeuP_kA%8y8l`X#QABDaN&`yG5D{Lq=p2R+D`Z@FHk%Sx2a6hR5O8vDK_fnGjU za~>^B_dJEfYZux^syj~?Up7EohH!RUg;IFR#` z<^bBa8p7(*vp0z@e9JM>tB~t_C6lg=iY_xgU1ML<mLbypcjMzzuB<3#Xe0 z11zKQ_P@lCyN>ef3c_<{3ojA{-Yy@+Nq}v3fK{_5M>^_4yChAhy5#4Kf6%#9k7?K# z;K3Zs=3ND(SP>F;TU1q-Qps4=oX;L>Gm|DV7esAH&yXG|B1*p zoptSD@L{A3NM`0w-2xX})VmRM~V8RSGeEFV~ z_gSGURnBm>LQ;hi>86%_~9-UCKUF0)}3CBl3WGkH9Sg zjDdJ1f*T%I`f-Q?>F67B*;n=4$UD|wu3|M=hmT2+pXAGpajr6QGC>!k=O1 z?~5zVYpyDo_VJq_zGrrYK1;8I$Eot=F&yWAMB+9nD7Fz8Bf4z_2H}>Ro%h=4hW*Y` zzvrH;oh^w2rS3*)m%tqSX2RAkOBDAPdPlta1&*U}0Pot*1_5X5yaIDcvW~|Fjn>GQ z!nakQY?)wdKnjd(v;YIr%IvJlpEHYFzG7nA9n|kdCusNGUpoi zxC8aD*{>_(qCN))&O(ojaE3&2bKpu3PEQaC&3{RaA1HScnI`)q@+cl0F%-Ts2{bjD z>XeUHRX7`>*$UVX!!!;DTCB;9ZbivEn4s-a6q1uUG7afnlCsd z&KRHF7Z`s&;U&s(S;aF7WO&bvO=l_(r}r>I5=l`DmRc3~(fi<^5MCU9v=;%CpPc+u z%FoP)0!d*~war2eRRycIQ*{J^s4Uozx+=r|pZZPA3j-2S4v~k=hBbgaL<=j+tp1Tl zj3pb#P3fL8A~%UlY7~IYl#=(Pjso--zty;;(e9R1&cz z)t_6-DA2TJ_d(<^)Gh^h>ZUfmUAog9GLE*iRCJrJruf!nbXw5b?j`B!+p9h_zdKH( zDGMa+1pQqFA`J5@w|(98f?4Ob;yomdTT*&{+L5a)bp{i*?ig3SAfzLY=m>!>HsGk- zDQ87L4t5S9JCY2<=h^`*NRAF=7WjZJWOFgLxPOGZ9u9g))f3MsE&mjYRyg;@7ABW| z5q;Sp1nn(!dx?NEFNmGv`>=}B>4S*!sof5)DPP|BF;8Lr_=C0dJcGNa)Cz_=9_U(} zv#(47=ecGC5t^nl*Ybt+`g<9il7MjCtR^*+7FJHyv@aL&B49!UMj>QM>#-=cFNV6h8Y%6mg^-C2JD$W;A{cO?W6V+!~RB1?z)00@eU45)Nqy(Wtl&W zJWD%MRT&oQiRsOD#4~grQoo9px>Pl~VE^KeduczX}>AhP`G@>>?F z)&2se^3+yOSP9hxy`)dC=!-hrOI+`Vk~aig4W(U-&SYc6DAhh>JGZrP`R=FZH8rn@ zntz|ebZe}Qwj)6+#I{kDj)!fn`z0=JGWtQr)f;kU9Jl%hnJs^^$3IZT867wXv+?%B zTQ$IZ@%(a+d#iDv{+0MvYY!8~(hsT5_wNj6$jM|t8SCFmmub&wKoFKlqg(VV68F^e z(92?ZBQSrw_J&52-m*C3ih-aib}l?O$ktF!oUY0C3BIW{A?qvt&b)4yUgD3q<_@^Y z8;7;_NH@r9H9A&qo)^qjdM)H}rv^PlGG(&*B?dIHE1I1J#Xo2C+b(lIAqtEiw$liH z7#|I-#Drb0WRRo4e!cBCJ`>XY_;IK%!2-@TcDqY(5nZ4N&O47UBOq)^GE9&NC73de zWJZ_E1zQ?SD+oBpuQIFc?nwt?s(SDdH?-KBo)@4mk`#P?eLLG63u3hBApH9rCHT== z>o3B1N8EGkSFhFU8>}-G^e`Qh>2fH4@P+WRD~|Lw$35G$JWId!B806r3;Gccj!Vx* z1B&sTyTwoytL9p5jC-VdS}0|=fKGJF4}5xTo>}I0#uqowULfc3QHavs;CTPcDP1=% zI9W=K1R-C;@xV&+u(Xi1&qlW|c<>8Zk=`#8gCMxO`TY2bw%m`th6z#l^VmhuEw%N&xW(*h=$*? zguW=@tO{ptbXlR=^GvT?rqtKGWY(`Z?Q@`NUmlTYoV)=4>D}P z^loCLrU>sBr?Rn0$46QpmUJweV zEbZDsRZ3SX^@j!_jvUZd%;5*gGiH1UF!}Nro-N@!SV3}-@m`ZAynh+8ouOJr=g2>H zX$<1UQ_q;l4k|<5g`y@#lG;d5s129np>zaYPNe4)y!2#N4TbbCQyk9g)A zW~X#bk0dpuj<@C1Ry}+%zVVtvEgq#Eqk^}*#|M3% z@km9sP47KCdI|7hhtq08IZQag%z$VI-Vq+UI_$n-n{|u+>X}z}q?4HER(dR$rh323 z7<)HM1uH`P6;Ua4(8|xBs0_iSCHdeT7PAC^L)40G!aETQFQmc5e(s;57r4XD;92o@JH{# z(itqL&VuhIz#loSS#dV$r9w#6CgaHZUWp@D)8gpvbcG|=(4}7u&fvLlvoH2fy(&>H zV+lP|Zjs9l$tI@X!nVQ&2*sQ@#TDp#ttGrwwpCI+9f|w~b^T%ULYw%D8e;4vn_?b? z{XT+(&Gk)!{4Mn}lm??Q80>8)u$@9*!C-4OLoHKwf=R=r+v&l7hLDRR(R- zvX3Nb0vbY*yQh{4LVX{>V(j-CJMPBv*n4Uq`+e6*sCA!qJc%KQfiy0U1$vXfN}haK zmHXAR3RA(f^>b7CaCksWi3~;jf`8{vI5AkGPv7IYj!2zrrpl2Ue;^yH`HXa=+t4BP z1+H=|g2<<(KBbqCde7OBE9cTY_fd^sl*H2cR?$BC_KHl>&tL$Q0^X_5j|YDm&|r1? z{yowAp2maesqVFQdg;mV_Zf(|<%0wUnQjz~eXi*xk7Z)=ajx6V1~=som5RgEo@rCmgVZuLz_6^VKR=7P zrK6PLH2f>%Qo>c-vna`HR7syYQGVFPNPHdKxBQ29*|5fDuLqB=dLD7{^ zS&;3@REi?5Tn`Aaehhcy3A4nYUl@40WGg-NnoMntSnoT&!{+y_u~ltY z-OfD=fEDVT?cB7vd`FZX<~(w_If@3*ocHrC>qd^Kz@bTpF1bK1b85Z%{OOsvA%OM@urx{ zH66;NcbOPwzO|~%K^$isTG;i~XvtL`b}4;)1#BYkXdebmodeyp!+Kdx&Rvy>{+xrX z{WW&1R&CVnZ#|ysJl&-Jb{0&$N-xv^igq@AvEsqhd9+|x<>wmEw_sq%GMf@Q+Mk{skd2z0V1F2=w@x#g`G<_al zblP~gP&&&!;6)(4kY3e3O^g3 z1Uw00i@pq>g_KopQj(5UGBAqs1zw(hBk zVLooI%IVAe?MIx`HU_z}4CAWpkgCMmKi}JROuRl&Sm7KBDssm;BjcM!K;QCze3>-v ztSe6UMk6}_p~0n~#5sj1L10-+(H?(9Vks^{F;d~>B6&lQsJNd1$mv7Y(AYoynY3)i zL8Ga)Hj?reJ9dp-hw8J~vZ{_jXiI9VKe~ng-$aCN(troD-3~xs?oG`b9uDJKUqV>G zQZYVBa+T?+yk6KnLjVgq$_BTEiqp7^N%f;QVSjsLVZ^e!&wN!@?hykI$NbwTMq@(Z z=F)4`#qIL}>>r(N?nry=<)b71BzKLD3^3YNL%{#Dw;KhtODg7rXP&dzStuNPi;B7` zs~tun^=);E9~E!I@^^z{QItD;0ixgF@VX zgzWnHbQO$;dMC@^p#CrHzmX5#hrLpH7JnlARFH!UXQgR;g$;svS_b<*;^QKEq}B4F z*V0lDrp2%N-3fm{=36Wo=-Bq_EQr!>)PnC*FP%}KuU0bVu{QJc8CFdi6!bRM#AC3@G03du=O1qjt0V^NH%|T1XJx3aY&S&r&j)H( z1l4&^)e8?q9;0u<2V@m9o*-=H?~APHL3m3!8E_fza@n^Y@u98HxGuHWbyv*zrz1ed zQT;aDS*vpu5loP8i-Yvu;;TBuV?Nkp2GU7MErigk=lh~wm7a^7hw6K0CI{dd>UZ)ihHzdJ7a>0)OI2E$=7b(@p$ zQ?CxDH6T8M2h-$|zRuoFSAPy(0m*}^iW%j`%J5b1Ytbr7k9jj(Ynv{J zc?N?fbC|}eq;6m2b{8HS$h_NqqKSyyFjrv9vd=$F!dQ=w3os!b8O~Livf3`Npe=cv zE$;rhze%%mEG^yMUnMYb{$LHcdwHNm)wA1*m6bQy({##_WI5>)hVd@8c73+$0=S!TX_Z-vlG$tuNA&9JanQ+pMxr^B;S} z_awjDUE+h;aEtZc`CF%6Ld>uXY7d9pM9$1Hn)3vmKz_gt9@HY_WpPvw_4_CCYL0pJ z2*FMAWQA$USBt~*7P!YZR*a-FYhJ}^|2hRuQVRiNH#nAj8~zxYS6VRbcsJu4&jI)+ zb~;^|8jC|ZieZAt=s~HJng1nkhHTzFt32fw%)L!lrs=>ZUfen-rvlo7CZApo$j{BT z-Lj$C7i){m^mdHr1KR;4*~j!PWzK(qGz2sHrgrtPIyYrGyFK#>#i+O_T~$NZ7V}wZ zQEq}gHP6+Ml_wO@i&D+}ADxtiw85>2SKQ(cizsRPJr|5 z4awKQ`42HjUoKIJl2F1y5`d-tSBou$8jN|Qt%JR{(?f};Pn=d<5=C|yHm9%eqaW_o zjoK{BOnQ<2vF|ZZUO&K{Hc^Gt5{udGt$|1SM?sB6?Y-?Z%9DCkv0tK(9EbTob5x)D zI>dsM)B%Qp-S1F1}=3r;)YcuPv@j7b0u_XHJ);DdO~dkFlj`mt&?>bPHJX+{}ECX zCMg7L`WgX2cFUc}{mM}zkf$6=@PV`%KbfZJ-RGxQ!#ygLdL?7h<4rDU{&!oVWNhy= z8T;u(N((AWW#;>@5Alk%+^c1Q@>VGw_B;L!_u)>q zGh;}U8Wu(kD^Wnv1tm^zWJ()g56VYuIcR72+|kPvVO`~RauvbAe*Js{@oAcb%O~Dr zIN;A=WrG?BZjK(L`d`93eo``#p*-bE*=v8IGNV8v?xqXK9X-S?N7=zasH!H5?DmBf z$RG>5zkHZZ$&K!*wIviX1rO5%x&Xng>r*VKKj0zIdyEawMt#nPd~N}ZKr)gbS59cF zesEcYFfxD_bLJ4kP<^fn6VR7^8A3jd3frr*aDaHQ988M$*G6^Glc%s3+ z?K9*m#Jb4%9)O`PeE2FDa8r8`*az3K@tGU>a$yU!)F3TZX`>QQ1$f8?Q(eMEGSz`g z$Q5~7XWJMRuqgQ!h7i>ab2#dO)7)skqJ-CkEY$)J)j@7L5M6A;_hlBWkgFzic8oO) zptehZ5B(m$m;rbk%6$g>fb*m>!1mk*gL@lsy1P0!(M$qWRdpttYvt9u%B#nuccSB$ zzU;U4@MkKdSX+izlBXNwNDg2Ht_YDH zeZ(PmAXX#-#GKHTI)jcATOLl9q)#1{q&B)Yy1F_g{xJ7n@5z!+$42J0Px_@7NYU@> zWyQ#0y5t0QAsv0a@Ks=1P4a{2nnOnqWyyc}b$%|Al{4cFtA-c5H@D)78y}PfTR&2F z<@VRTy{ma(=2IBrpH=7TaEP3E!FehJTj3m5R{u$FnZSR|4t>uX+rR0BE0j_Ij_;uU z*dzXLpejqUy9&{_jbfJ{d=O~2B|{^>O!;M304?ZCw~ky74&0cqua4$NEAvq;s1uSr zs9SLe`tBbh0{p6jQh;Q8IQ6{4Qw6#@#I2U+g8^~bd!&)u5nvAdIDQ_9b$G%1rYyaj z=`OgSWHQ{y8HK66^`-!$5SDPdcxm7#vW`X;@|wC&yU&2oV1u z(w@-7w4f=feb2+8b4+k=QG^BWSRhl;$%-Aayoa94#O@_FiwSK>6Hbx=;m)oM#I=fg zOjg35iN#Rom9a21zqY>^V4sXpA{`hJRl;^M@FIH;(4r9bV#|vw60I@5a1qYuu;0u2 zRMuDhIxDFjrH&O~P4N4W{kS*I?0B*Nde@i7J64(G)$AOc#YL%mjtfm%{MX~u{zKph z|7^b1l?CeX<5>s6nb5}Lhh)8u_aw$o!5;{&58|WSK%SK((>v;k)1A1Pl#l(ncg`q! zqK$diQ3ReKG&GM8MnC+m%5nnmi!t7A4*qF$cbgq$8YyEq1B5RS8}8Z3;6^eIT_CF2 z0Dwq|#9`%4qw6qtwlWrIttg?{9(mTlll)pHbz6TcMo%p4z;4*zh-fr-b4mV>q5`_QO z3s97?QYqg9Ic153PqjPi-7Q^+h`6!9IyT6)WNZ#+aA#|i&VAx@Rx6d17mX*v4|7ni z$KnB_YNie2(KaCRXi}&t==O=LxZtG60V98zd-!C;6nK1oAnYW}m^Nf_xGR4_nzzr( zGr9Ks@9uQHq+lB_#J8vuer_&foFapJdldu_wHQ2}=euSDkhRJ6b7qysoQgx!71^PI zVIAZvUS18@I|38=BV&tSdj>z>VRIF*ex<&tF7xLFW_{{m@#nw`E^{g|PJu==D?RcCh!ygqcXih?C@0 zz&)yX}II z;-u4OR&zTXi&4AdFX>ISq5mq~(E*BUJ5Tfy_4|~j^D|&b9^Mf#PM3DjWOc;f?IDQ# zS1bNKdQ>;9Gz|?<5anHiK%pQvdSr=__@GT4?#)3MDkg4#F;=as1;$su9tO&u=)9hE zcWkP=OJjaAVgytcB`KSm>4_m=Jvpcza~T)RUA2$9^!KJElfyGFE1>#Ha|_qjVEy(J z!{S)zqy#X8adcPf~CFF+2ZC}#wXY8BkFu$wm zMhnp|M+>hY62fae7gKTm>Yv(wU|!w_S0y>jfIWBD9|z+6BdVg(=?J&DQwn9YtY1UE zT}foB^zBm7*0?U25$XmHBv;I^w;>cvsQtr>J2GY5c|6V8ky@0^Nz-2M$^nz@^abzC zuiE=~+v+}>I9_P)OWwU%K5#5tNyX#&f$VJqbykjb>bwyTbuS_sRO@=A=7bs+!d-&g zQjm=0>#+dF-`w+~UlZ^Trg&PY2p&A&nm2=J3_jh9nWUwYZo_G1_uPKpxwV7Xe-Nd2 zZ-!97*7x7--;fbwLRW%MZjP@1j6bYz$IvPihdMv^d55$h+sK{vK4mdD{%sRrhzX?y z_z{2P)azUz97#l&%%Lr9vUJ9Um1RA?(ZhFZ5<}q$Ma>ATdbDnIh9px|3sK157cD4H z&#_3}pZW`({P?|B4ziaQxxBCQ@pIT0;h_{9E}tRfuOrMFKTn}~B?=

ZdJ>WrQmS5`*L9WoDLN6;C z?~s(C52&I!ZgLg=IWz!D_Txa)^!H}@_hYx1&fb5|@zYdR4nnrTuGi{DIO6w?Mj%Uo;(aVF{dDz!h7Q@ysie zryeAS9BJ@O)}*0ruB_&=pD`6+!(s*e{jh#)jJp2b=R@&MMf{X&{2SU1CQjrguW)Rtm06fM{^?Qn+WyK;+R^iSIA!k{gh8?o|W8 z)Sh`b%(;zkHMUk_4)%7clK8eB3Qr92STtlX_O{>_eC9_T7Xqew=QV}}vZqu6>M*36 z=iE;shTf0AI_=jCVilJy*Hy+v%gl|SmSm0vdoFWJ)PG-E-3ImQ+f{)98iFtaOcY?- zElosiVM)tQR297?P?a6q^kLuK)^ZnP&|qfrWe@%c*2x~487)sPJOz&Vh)@ zaTRg@Qktx6u<=!rCcTv0`1kB@*|iy`ZIi&#DX8y==Mn9vl;1M%XGNaa+{9q_-DXi0 zf9&4vH4)*QE>5)m?eF&9!#WZ&;7m8tnHg#W67X2jk9+~zm-Q-ntIV0Dg#``*`4wI( zMWsk7pX%~Yg`Q02qL(aO{d<_()sAK_z&Eu2j+qdtK5&K(JK8 ziG#xYNxH5KXPaB?3!oK~X_hQq;Y7aow?n&8sAjrFPq}+Yn^&v; zIHAXUtPzM2305ng`Wo{230ksCUR7!t2~4ZhaZ+-hw_q>X18M+GyVfA7<%KC+$fMYz zV#qW? z$#>3{CSYSGKzR!=eX8Ovu4Uh9P3!%$|EZhwQ7FsHHcQZnCQTF%IiD||hfDP^c;|B} zv8)&UeA^lWji%I7twO-q?Mz?lu2cUlw2oFX`YzNxlohSEx6e%Qw`3s(N4T zGK-#$`qqzD5oMG?RcniU*}P;hg=O1^#RH79`-ewhP)dwSSS%IFm+htDdB1^=c!3Rh z9=lOs3tV>LG+Mj~uXf-&qt|jGwLM^x z($v+lvHLos5y_eJ=e6c9mTZOr1YKzxM!WHe$=VXR{hf(IyXqU!m-vpTCF&KgIl{7h z|4(;U{TAgD?H3SGdg(^GQwa${StJDMlunUG7LZ&-fu&37T5{=J=@5`m>26sX1d;Ch z!uQ^P;_mb8Pdo3Kcjxq(GoMi?8PT2N!c1X)sPhSCoA!Qpk%`QVmBigvg!55CLu*z{ zAobqC&(F!$yVC2@>!1NYNm;)hn2k2-y|16NHvZ(O8pcoTzr=or$obcC^;mT+ zS(91BOE`YsUOt$l#FU4QuTIh#jqk%kmqrxfZ>SCM%Nx1`dJNMP2(*M%RHT^ z$&ti>tC`AXFD-j+$xq2TL%B4LMeP(Xs4)aU*32+9N>d)9B|*HahTH*oj@;%x@crdj zjzhTzbX?3JdiUM*0~3EzgFntjgbQr6YA23A7tI&c;h*tn+?G@WsSt)zJVO)fh?ici z#MAUZQwe}$FY9N@zX}^&i?ifCaF!w&!!~8HXYMuP#OlPb*#A1@bh2wIgEcTl8e_&iTWmPD?NBH7rDmLCc8~?V( zY8;b}@tF+UCz9hB`lw}g?!kr+#d|g5=A32eZMTKMMz;ygGSt5W8Zm=DjnCfxhp*VG zicEqEr7Wx7>7TuPXS&yPu#k z+SHUX(I3hGU+kjoudKrZZDLaT540M8|FGTGf+4(0+lBB5T)=J)JpfW(^dF zp)r~lV;m|l|MI|Mpf|qEqz=siSmtYtbP;-K$UP5u~&1Z zrp5`VNIMVF1eu@TuebZ^lkR1iZ)MadE}hTUiNpKg^HCh_r^K`g-oF)?F}|h}Lc(^b zCB%;h1YX~KGC!bV=+#!@i{4ED$rc$`{>~|QLM9&Yx61KNakyjzqyJ^K{Ef>trkQ{3 zsH>WOPMwEu6CqrbCI(dh0FJD)3ilwE zQw9akD4rURu0O1b8RcUuXxo+l7o{f*lVqc`Jvn3}{ncT9C<_|sRxH#lf3e(~8hD6q zdnGn$B|&iD)K1u##H)+nfao8_wy{mUeuoWmD;(OOCc{?g z*WZ%61dCWPG4U!a_x`=r$-K(Cb~b2`|iLXJ1&5RB`ce z{j}ICZmPGgFmrKKn2nZCMQBD{ecU2gA((N`oTkV*jKJP$(LuX&0eNkCwo`pAQ2C3N zHQdv5apvjrW5BE+{mC#$d%jMM5oIew=QE zvT%@!7UuDsixR6LvB45Ez%i0?rGX8HtlsRpqJ4Ky(?`Gjlqj!Znes!*U5EViA7mR? zA~Rf8nq|5;`WP9;Lj4{K9>sGq80S9aQfaBwe8&F_0hVYk`%CPggp>V6M*(N`^~uHaWBM|NPFegK znL|k_6%{d~X!%69be5fnV!3_(QTLz#tMT-?*JBGwuGu>_8!#rUk)YkpHgC7}8?~GK z^?j?WB<0U?1A8|PG#8wtQVChu*J^Q{lemQ`IpDc6Y#Pu-;KDhbEG1EC6AY zC%IGTCk!0FC_rvw$7EWD77A*8vpkIw-M@j=p#0$SM0hWmBXBw~w(T%l^|3w(6qjw0 zudlPNFJ8*f8>I+$f1b2G2WLy=1!a z?cn}qnm%3`zy?Y$sV57gi%=!P3+2L7lTj#}p1~l_x~aVl4d3qlLD8|6 zr90o&eLO4&p%J~x{F2jp-YedMsE~K9`jo^)z7Zde&EI2GFQ)mTvSaI0z2%rWa!Fy< zuzEP*QS=nDotLAICDS_Elg2gbAbLwDDQ_C+I*)mV7?eWEu8DG9g9Iubg6SwqSNXzw}_e^wT zhc`X#IngZ?h{lW^S2ql?i!npviQwCZ*hqt-HLm`8vwP|%N{yV(7aIsy zyG}NXFFK9NhcZ6ZqoA-wALe~Kjt>2yzttdJ6YEY>2%Jjxa8x3|{)Gy*=8ucwuo1BS zxXNZvmMXeyB&R0+kR9Q#(S5)mAKqGP{oO6YVeQo}&kkQJ$%nSIXex6fO*WsTUy%@0 zRmWC_rZ!20SM3Z@e3{Pp#ycwAG`qR{q_B|7{2Gh~{!z^^_jpSMx6*pVI{6|xiJ|ew z(bhC5{w(+q&&+?N@er4dmfEq>#|x%8{_(;4rdRFYX&bFVmMfXBuIBJJTpI5D*^`({ z5-&vF%f2yTNyaF^-XNB2eQu!%19ZI zPyCcQlzaepRh#bQ=vKQP(PfduUE|%tA0L-heNDc30JlLneghb3s~t!uzBX~FPm64P z;ZSnVRvSM3c5|+3PkcOa;_ckK4|tS8qahHIX9i1|lz54+h|MGg;(ty7HY*tPHi241 zWY^Mg9Pvh3iy1BwMyR(JnTlCa$Yqxq`%=p7=Tooobmz*aB$nhvE_Ue5~L2aPe zu{6Blhkwh%7tqgTM^^-oM6ti$xD-CyEoKlMF688@<^+Bh_x6)8_?pn@5Gsh9Vu%3~o@seRB$!*Q2yZu0IR7e-U`j zLD~J&!%sPO1TVJ!$xXl9tUnE$K?RV>_4<8GHy9fpjvajYh|BA5TCMm8H4VS!L6I1G z!le`pr9^*7~*#!o=JEl zjoZFcUcclHsF-}O*}q3kn_*f#fNkZX&n=K+NOzQaDJHqz`ll0vdcMP5dSQ~UhZ#Xn z`A>anSz`Aw+oNCQ=VZX6AR{@%*sW~}OBKZqzex>HfB9fKM{!nND)PzTm@igQQ~D2sQM zZy70gI6xFfB};F^F?lae7vy%MVMukB*KGjdwg?`N_-?3JhiPD?_vb8rTxj}*F6hZ* z5xIB2Cp`hLOn9Pii6BRaVyQ`r>45}(zX%p>@?>i* z-Vif+76kD}EE`$fxF*$YY-%bm=(Hi>w&KiBg+u}|gA3*@VJ9M@k+|;B(vUtDgvgJ| z!xpeY)3v220hmyQHRAiP;C^($5hnefy`WzhFdfPqn2t4M#~Y_TQ1z`zhd6&vAD-QW z0S)*hSa7ZcgPK7%$90PzV8k=WTcwg;UdNcSmV5#G!4v#z@hf#i?|r(`)*>9Wm)Igj zfkplYthmoswnJ8D7o8VVyTH^Ch%f#brz4!z^b^~0uU2mq1$3A;qAKjX3#Wwmx7?-~ zsM%SoBs@HPX>IchtIeyNYjW5yixxKHtrOKhTO88l!mecwCgusVfD48JFmcojgl~ zPrqov`gZ=Z0=d4y9OXG?sz{28rc%F-5KT%e@SjhFUxfEsKqzWTe8_GZs=j;2`jk&k zbMn$gvt^{#l$kgb%SLeaCj@fl)D;^K^f2(jnX|b5c{3rM4PL+DB^%D1GN-}AebL5= zm}C=PS@Py*N6xXwMVM#faHik>EQA=IiNqT85z-vUQJb} z0)=%4i*OshM5ifpcBJLAA63in8!XsvWArYqGc3Ky(Diy@KW?27|atRnzt_LjHd9ElKc|9szB zfz)dqB&X9lqoeH}lMgh?`2QPMw39n}mR!NVu=;)V;KzF6<-Bb<@z_N1!CcoDWNbZC z?16>%ih&W=1TwgL$Xkd$xzeKDRD7_iLnY27VC5nCas?>fUrUH1{Kg(HaSjh$7sqe7 ziZ2$)^Ur%7tN!716}&B4MF2$tK+JHrny)m0etv~()^>Hj>rV)(`h^l12d28r`$ff7 zOWEGz)Dq21)h(oHTNJ~L5tEI)Dv77AMzJ!cRlgnRV%YqtIiH<55xqgLX_oU=x<`gLrBb;37>^$_w}Xj`%sBqrEUr9VN@{ z;_%Jgawgx2Qgs(H(Um?fDvB`WLJ)WSHT8IVl7uD>afG8xZ=U4U@V3%^$c@B>UrMY9HZXslEV*3~*P{3i+LE}v2 zh_V*KDJk&u0o5vrs=_&?UMoVlP{ePuUMaO)5n`=&SNT)S1e!ZGsVQ1Sb_8!RB9KKf z0SY>SQEXZ51Fx!w-CW(!rSaCDeTDw!!Jlng8+-dmq=VQ3vJ)!t+tXmq`f(}C&HYQYOX&jW`7jD{nLYz1!1j=lRJ**e_wegZnfy$`E_O`ugetBbkK4D zmTQa6Oq{yct8!m2yd*Fw<4;8zUt`v_+z)1xs&Xq63GI6TmJfNJ9c4GRhbfQ<%sw{e z6|Cy0&hyLniAc*`)bAjUeI3Ez`<9}2^^zCBXvq+q@*{lsl|$G%q6huN#*COzEiijE zcEG@o1~z!enShywZBRON4pd53uZ_2{biK-XZymah5Pti#(w_$>#79;pMy*!CHijo+ zL|)8pD4hY44Xqw@OaHNU6a<2m&qCcG4|}gn@Vh@ty6Sx_#U3@48*=79Xq10?;2Q7$m{y?jPS)RC!G(5}v zVT_`Z8ac}%8r=waJauPo@-|38>uJ%NOZJ1&o@VIgT*=~HWBt6`@FZ>_(eC>0l3=@p%-?C8%9Uz+HIGzx^g%{DtY>)R%ML$);bBO28+3I*_CG|M*@SRF)R zoqR1HG5`IzsdZ`z_%+vO^2Liw%VRmH_AbDIZ?-_&}L z=!W~nc5rhMkV~qWB^Au>;T!kHW@2$uNn7943Ux@LHV>6(;0qH&>;su<)o$ID{>rwM zg~w*%Ww$(7iRLklv@VpPIFJ|3*Pl`r^#)$``TE>fe{V&Lp7q~b_AoyJL=Q_#^oD)35<$+w|~Es^g@qVZHJW7@=OY#Aap^9ub42Eh*d6=jhBhmfialG#su?#z#Ap ziQiGpMeC_EOlxS zOX%P28z+w4{ZPl-_TxRE3e|?W@_LPx+_(8taj_{Q(D@|$YZwOt4fZjb%>{6Q*++%( z&XgrI{$WK_xf_-53cUhCK`|gx@*#w=p&p)YTs_d7OD+M*QhX`Q`#|2P3c7jBz&f;d zy2Yw!6#02{>aORtq>RD~B5VuasNp6s6ZKQ-iy38n3**m_I`*Fo#*g#n>W6@JTJ0Wb z3!uqwQJufu7Fv9B0BG}~-?2`a-a8*}-BBH_BS;q~&5qYP$l|>|eI{!bL8KFkmIrsA z=oV}AYyIp|rX*sGb!Dbw41j^w&UR!5jiUYV8pQ_+WGe5lm-Tk`2gr46eko!Pe!N6e z8b&9AX;1v<z$$IG<3Ms2WpOqn zFsbb%szGEK3_w5MGoM-hvagYDt5aJ27;w#@t4cH!2ObM;$46$sjRL;6cLbEmrD;}e zv426Y{6`nGyzfm&z;vyp=v+e32GNPyw~Gd5r*^7FhOc*C zyOlVF_Tex>13q~}=h1ERLjprzBRKd1mwJkFjCq;gPo$JX~-s{c~kttQC1 z#Q0L&(uenjgQK1=3YBzfqH0#~{(Ih7|2rK8p6OS5b_eKptAZ?FWoYNj^AZd9dvv$0 z(C^-g$lM2|m45HFELl^X!&ov2)_Py)wBkDZ-W}s^gu`-}rhN909#A(+A6_?Gfrd;I z#pB<}q^UMEiWU=hK;XaMm?C<-aq;gKb?|{uSKg~2G2n+eUt1e@*gSPZ5aC@b{^<9< zS}`K^uk8FRN%(q%h@1(;eB&o&8bSvVBX4K_!HD$6B!q9aCBhk5L-s4_T=zQM!!s>6)eP7*5ls`FUyZw!l`M&TjV~ov(<36#(v51PNWj;p^$6rRazv!%!7QQcRu- zn4A75QF7#gkCED+2mP6+q^CVI9hcf0@N7qeVQ6V*aW!XPw+YZaIh}2cBIPoyh6VTZ zNU0hDpUUUuLVSVceX`N&B**Qy5_Vq>y;FU}(l}Fj`I4f;HkWhTT{mc>4irgLuQ%-T z!_A%O_N@`l0_oWnNVME;B`rrVuz3pXM}H;wA+=mSYW{zkK32BjZwH3`;i$t?Eh}|t zopR5MbY*D3lB)tN2XvnJfd=s{d%_y;j9S>7vrz@lSD?VVs*#q0lPXBvaIbwjWL4Ph z2-uf*r?{Jyk>37-5XKO)YBfgZTh#bWM(hzGIZ!rK0(o%?uXKE$w%K-C?X$Tf>+>cZ zs#C%hIjPSrR99zp00qW=+zRO>0lfr83ek_`yA2NVz38{U8+-r!$Pf&BHY zQ^W$%(g(H>h*U(lbM0q4p*z0R5(6z`WIFNF8O?IcfxY=0B`Cd&@be89=F=AmmYS0*?J@AWe?IZ*$hUp%RYy1olJ0B8JQBPoIY+(*&v_p07= zLJ?bB2~yykOjWq~VGrh2zz7!hAbEZL3Ose>7o}xBaB;x70vKqaUt$YINo}Ri7(Ci_ zp|m8Fyj!Ys4InQv`4gu|AtLkXw-n^a_ogoA(5tR4{%1yIt>i(FIPuAMJYa7Jzpw44 ziAA!c&+It+)0<$@TMn{djT6Q(m;E8d@lfGGV|y#Z9O$s})eEI&um9T=IvV(dplmN1 zD`#0G!d8j&s^`4(EZ+xAn>X7H=HBD9b85APUkXA3X;XDcKZF(V&oK@(xLo`_{)TEm zxD&#Lxy_W^w2KN3+eVZ4Wba&<%C}k|>*yWMsTAkY(jv`vIDf;3Gc0u^E2XOlab4>VhV?pZ3>k-$o=B`_{;=%?}Tym=S%@G*s zF2Da}fF>{+v17e*s$WBzj5{i*7{kTgiI2tF{X2k?AvwKkDQcb`9e zAb6P)gq;$dfb+ju`j#qg-(64=cHXB?1vDNJV@WcP8fvi*<L!1g3bg6GuGJkl)cSPTMggBeq@QvCYNm?HO8pR+d9&_NWG=z1#&YQnE zKCQz#YCO9eHw@x2tF+LG>v_#Tw25~n4U3~RU0cMI&fnsDV$(TQDDuA7qxCMLNMr_* zYI~=M%)Ads#JRnMdo52i*j$M;X&R@)L8Dn{Ytm75Ze0X0=#~J{k=*_BYFE>O${Cd;R_+;NfH}YPTG%s>lJz?f>@=!vFM!+);(! V@-=TOaGw)MS>Y9={JHsu{{e&N97zBG literal 0 HcmV?d00001 diff --git a/doc/source/tune/_tutorials/overview.rst b/doc/source/tune/_tutorials/overview.rst index 326e8c403..b0a641a39 100644 --- a/doc/source/tune/_tutorials/overview.rst +++ b/doc/source/tune/_tutorials/overview.rst @@ -48,6 +48,12 @@ Take a look at any of the below tutorials to get started with Tune. :figure: /images/pytorch_lightning_small.png :description: :doc:`Tuning PyTorch Lightning modules ` +.. customgalleryitem:: + :tooltip: Model selection and serving with Ray Tune and Ray Serve + :figure: /images/serve.png + :description: :doc:`Model selection and serving with Ray Tune and Ray Serve ` + + .. customgalleryitem:: :tooltip: Tuning XGBoost parameters. :figure: /images/xgboost_logo.png @@ -73,6 +79,7 @@ Take a look at any of the below tutorials to get started with Tune. tune-sklearn.rst tune-pytorch-cifar.rst tune-pytorch-lightning.rst + tune-serve-integration-mnist.rst tune-xgboost.rst tune-wandb.rst diff --git a/doc/source/tune/_tutorials/tune-serve-integration-mnist.py b/doc/source/tune/_tutorials/tune-serve-integration-mnist.py new file mode 100644 index 000000000..1679d14ac --- /dev/null +++ b/doc/source/tune/_tutorials/tune-serve-integration-mnist.py @@ -0,0 +1,687 @@ +# flake8: noqa +""" +Model selection and serving with Ray Tune and Ray Serve +======================================================= +This tutorial will show you an end-to-end example how to train a +model using Ray Tune on incrementally arriving data and deploy +the model using Ray Serve. + +A machine learning workflow can be quite simple: You decide on +the objective you're trying to solve, collect and annotate the +data, and build a model to hopefully solve your problem. But +usually the work is not over yet. First, you would likely continue +to do some hyperparameter optimization to obtain the best possible +model (called *model selection*). Second, your trained model +somehow has to be moved to production - in other words, users +or services should be enabled to use your model to actually make +predictions. This part is called *model serving*. + +Fortunately, Ray includes two libraries that help you with these +two steps: Ray Tune and Ray Serve. And even more, they compliment +each other nicely. Most notably, both are able to scale up your +workloads easily - so both your model training and serving benefit +from additional resources and can adapt to your environment. If you +need to train on more data or have more hyperparameters to tune, +Ray Tune can leverage your whole cluster for training. If you have +many users doing inference on your served models, Ray Serve can +automatically distribute the inference backends to multiple nodes. + +This tutorial will show you an end-to-end example how to train a MNIST +image classifier on incrementally arriving data and automatically +serve an updated model on a HTTP endpoint. + +By the end of this tutorial you will be able to + +1. Do hyperparameter optimization on a simple MNIST classifier +2. Continue to train this classifier from an existing model with + newly arriving data +3. Automatically create and serve data backends with Ray Serve + +Roadmap and desired functionality +--------------------------------- +The general idea of this example is that we simulate newly arriving +data each day. So at day 0 we might have some initial data available +already, but at each day, new data arrives. + +Our approach here is that we offer two ways to train: From scratch and +from an existing model. Maybe you would like to train and select models +from scratch each week with all data available until then, e.g. each +Sunday, like this: + +.. code-block:: bash + + # Train with all data available at day 0 + python tune-serve-integration-mnist.py --from_scratch --day 0 + +During the other days you might want to improve your model, but +not train everything from scratch, saving some cluster resources. + +.. code-block:: bash + + # Train with data arriving between day 0 and day 1 + python tune-serve-integration-mnist.py --from_existing --day 1 + # Train with incremental data on the other days, too + python tune-serve-integration-mnist.py --from_existing --day 2 + python tune-serve-integration-mnist.py --from_existing --day 3 + python tune-serve-integration-mnist.py --from_existing --day 4 + python tune-serve-integration-mnist.py --from_existing --day 5 + python tune-serve-integration-mnist.py --from_existing --day 6 + # Retrain from scratch every 7th day: + python tune-serve-integration-mnist.py --from_scratch --day 7 + +This example will support both modes. After each model selection run, +we will tell Ray Serve to serve an updated model. We also include a +small utility to query our served model to see if it works as it should. + +.. code-block:: + + python tune-serve-integration-mnist.py --query 6 + Querying model with example #6. Label = 1, Response = 1, Correct = True + +Imports +------- +Let's start with our dependencies. Most of these should be familiar +if you worked with PyTorch before. The most notable import for Ray +is the ``from ray import tune, serve`` import statement - which +includes almost all the things we need from the Ray side. +""" +import argparse +import json +import os +import shutil +import sys +from functools import partial +from math import ceil + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from ray import tune, serve +from ray.tune import CLIReporter +from ray.tune.schedulers import ASHAScheduler + +from torch.utils.data import random_split, Subset +from torchvision.datasets import MNIST +from torchvision.transforms import transforms + + +####################################################################### +# Data interface +# -------------- +# Let's start with a simulated data interface. This class acts as the +# interface between your training code and your database. We simulate +# that new data arrives each day with a ``day`` parameter. So, calling +# ``get_data(day=3)`` would return all data we received until day 3. +# We also implement an incremental data method, so calling +# ``get_incremental_data(day=3)`` would return all data collected +# between day 2 and day 3. +class MNISTDataInterface(object): + """Data interface. Simulates that new data arrives every day.""" + + def __init__(self, data_dir, max_days=10): + self.data_dir = data_dir + self.max_days = max_days + + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307, ), (0.3081, )) + ]) + self.dataset = MNIST( + self.data_dir, train=True, download=True, transform=transform) + + def _get_day_slice(self, day=0): + if day < 0: + return 0 + n = len(self.dataset) + # Start with 30% of the data, get more data each day + return min(n, ceil(n * (0.3 + 0.7 * day / self.max_days))) + + def get_data(self, day=0): + """Get complete normalized train and validation data to date.""" + end = self._get_day_slice(day) + + available_data = Subset(self.dataset, list(range(end))) + train_n = int(0.8 * end) # 80% train data, 20% validation data + + return random_split(available_data, [train_n, end - train_n]) + + def get_incremental_data(self, day=0): + """Get next normalized train and validation data day slice.""" + start = self._get_day_slice(day - 1) + end = self._get_day_slice(day) + + available_data = Subset(self.dataset, list(range(start, end))) + train_n = int( + 0.8 * (end - start)) # 80% train data, 20% validation data + + return random_split(available_data, [train_n, end - start - train_n]) + + +####################################################################### +# PyTorch neural network classifier +# --------------------------------- +# Next, we will introduce our PyTorch neural network model and the +# train and test function. These are adapted directly from +# our :doc:`PyTorch MNIST example `. +# We only introduced an additional neural network layer with a configurable +# layer size. This is not strictly needed for learning good performance on +# MNIST, but it is useful to demonstrate scenarios where your hyperparameter +# search space affects the model complexity. +class ConvNet(nn.Module): + def __init__(self, layer_size=192): + super(ConvNet, self).__init__() + self.layer_size = layer_size + self.conv1 = nn.Conv2d(1, 3, kernel_size=3) + self.fc = nn.Linear(192, self.layer_size) + self.out = nn.Linear(self.layer_size, 10) + + def forward(self, x): + x = F.relu(F.max_pool2d(self.conv1(x), 3)) + x = x.view(-1, 192) + x = self.fc(x) + x = self.out(x) + return F.log_softmax(x, dim=1) + + +def train(model, optimizer, train_loader, device=None): + device = device or torch.device("cpu") + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + loss.backward() + optimizer.step() + + +def test(model, data_loader, device=None): + device = device or torch.device("cpu") + model.eval() + correct = 0 + total = 0 + with torch.no_grad(): + for batch_idx, (data, target) in enumerate(data_loader): + data, target = data.to(device), target.to(device) + outputs = model(data) + _, predicted = torch.max(outputs.data, 1) + total += target.size(0) + correct += (predicted == target).sum().item() + + return correct / total + + +####################################################################### +# Tune trainable for model selection +# ---------------------------------- +# We'll now define our Tune trainable function. This function takes +# a ``config`` parameter containing the hyperparameters we should train +# the model on, and will start a full training run. This means it +# will take care of creating the model and optimizer and repeatedly +# call the ``train`` function to train the model. Also, this function +# will report the training progress back to Tune. +def train_mnist(config, + start_model=None, + checkpoint_dir=None, + num_epochs=10, + use_gpus=False, + data_fn=None, + day=0): + # Create model + use_cuda = use_gpus and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + model = ConvNet(layer_size=config["layer_size"]).to(device) + + # Create optimizer + optimizer = optim.SGD( + model.parameters(), lr=config["lr"], momentum=config["momentum"]) + + # Load checkpoint, or load start model if no checkpoint has been + # passed and a start model is specified + load_dir = None + if checkpoint_dir: + load_dir = checkpoint_dir + elif start_model: + load_dir = start_model + + if load_dir: + model_state, optimizer_state = torch.load( + os.path.join(load_dir, "checkpoint")) + model.load_state_dict(model_state) + optimizer.load_state_dict(optimizer_state) + + # Get full training datasets + train_dataset, validation_dataset = data_fn(day=day) + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=config["batch_size"], shuffle=True) + + validation_loader = torch.utils.data.DataLoader( + validation_dataset, batch_size=config["batch_size"], shuffle=True) + + for i in range(num_epochs): + train(model, optimizer, train_loader, device) + acc = test(model, validation_loader, device) + if i == num_epochs - 1: + with tune.checkpoint_dir(step=i) as checkpoint_dir: + torch.save((model.state_dict(), optimizer.state_dict()), + os.path.join(checkpoint_dir, "checkpoint")) + tune.report(mean_accuracy=acc, done=True) + else: + tune.report(mean_accuracy=acc) + + +####################################################################### +# Configuring the search space and starting Ray Tune +# -------------------------------------------------- +# We would like to support two modes of training the model: Training +# a model from scratch, and continuing to train a model from an +# existing one. +# +# This is our function to train a number of models with different +# hyperparameters from scratch, i.e. from all data that is available +# until the given day. Our search space can thus also contain parameters +# that affect the model complexity (such as the layer size), since it +# does not have to be compatible to an existing model. +def tune_from_scratch(num_samples=10, num_epochs=10, gpus_per_trial=0., day=0): + data_interface = MNISTDataInterface("/tmp/mnist_data", max_days=10) + num_examples = data_interface._get_day_slice(day) + + config = { + "batch_size": tune.choice([16, 32, 64]), + "layer_size": tune.choice([32, 64, 128, 192]), + "lr": tune.loguniform(1e-4, 1e-1), + "momentum": tune.uniform(0.1, 0.9), + } + + scheduler = ASHAScheduler( + metric="mean_accuracy", + mode="max", + max_t=num_epochs, + grace_period=1, + reduction_factor=2) + + reporter = CLIReporter( + parameter_columns=["layer_size", "lr", "momentum", "batch_size"], + metric_columns=["mean_accuracy", "training_iteration"]) + + analysis = tune.run( + partial( + train_mnist, + start_model=None, + data_fn=data_interface.get_data, + num_epochs=num_epochs, + use_gpus=True if gpus_per_trial > 0 else False, + day=day), + resources_per_trial={ + "cpu": 1, + "gpu": gpus_per_trial + }, + config=config, + num_samples=num_samples, + scheduler=scheduler, + progress_reporter=reporter, + checkpoint_at_end=True, + verbose=0, + name="tune_serve_mnist_fromscratch") + + best_trial = analysis.get_best_trial("mean_accuracy", "max", "last") + best_accuracy = best_trial.metric_analysis["mean_accuracy"]["last"] + best_trial_config = best_trial.config + best_checkpoint = best_trial.checkpoint.value + + return best_accuracy, best_trial_config, best_checkpoint, num_examples + + +####################################################################### +# To continue training from an existing model, we can use this function +# instead. It takes a starting model (a checkpoint) as a parameter and +# the old config. +# +# Note that this time the search space does _not_ contain the +# layer size parameter. Since we continue to train an existing model, +# we cannot change the layer size mid training, so we just continue +# to use the existing one. +def tune_from_existing(start_model, + start_config, + num_samples=10, + num_epochs=10, + gpus_per_trial=0., + day=0): + data_interface = MNISTDataInterface("/tmp/mnist_data", max_days=10) + num_examples = data_interface._get_day_slice(day) - \ + data_interface._get_day_slice(day - 1) + + config = start_config.copy() + config.update({ + "batch_size": tune.choice([16, 32, 64]), + "lr": tune.loguniform(1e-4, 1e-1), + "momentum": tune.uniform(0.1, 0.9), + }) + + scheduler = ASHAScheduler( + metric="mean_accuracy", + mode="max", + max_t=num_epochs, + grace_period=1, + reduction_factor=2) + + reporter = CLIReporter( + parameter_columns=["lr", "momentum", "batch_size"], + metric_columns=["mean_accuracy", "training_iteration"]) + + analysis = tune.run( + partial( + train_mnist, + start_model=start_model, + data_fn=data_interface.get_incremental_data, + num_epochs=num_epochs, + use_gpus=True if gpus_per_trial > 0 else False, + day=day), + resources_per_trial={ + "cpu": 1, + "gpu": gpus_per_trial + }, + config=config, + num_samples=num_samples, + scheduler=scheduler, + progress_reporter=reporter, + checkpoint_at_end=True, + verbose=0, + name="tune_serve_mnist_fromsexisting") + + best_trial = analysis.get_best_trial("mean_accuracy", "max", "last") + best_accuracy = best_trial.metric_analysis["mean_accuracy"]["last"] + best_trial_config = best_trial.config + best_checkpoint = best_trial.checkpoint.value + + return best_accuracy, best_trial_config, best_checkpoint, num_examples + + +####################################################################### +# Serving tuned models with Ray Serve +# ----------------------------------- +# Let's now turn to the model serving part with Ray Serve. Serve +# distinguishes between _backends_ and _endpoints_. Broadly speaking, a +# backend handles incoming requests and replies with a result. For +# instance, our MNIST backend takes an image as input and outputs the +# digit it recognized from it. An endpoint on the other hand forwards +# incoming HTTP requests to one or more different backends, according +# to a routing policy. +# +# First, we will define our backend. This backend loads our PyTorch +# MNIST model from a checkpoint, takes an image as an input and +# outputs our digit prediction according to our trained model: +class MNISTBackend: + def __init__(self, checkpoint_dir, config, metrics, use_gpu=False): + self.checkpoint_dir = checkpoint_dir + self.config = config + self.metrics = metrics + + use_cuda = use_gpu and torch.cuda.is_available() + self.device = torch.device("cuda" if use_cuda else "cpu") + model = ConvNet(layer_size=self.config["layer_size"]).to(self.device) + + model_state, optimizer_state = torch.load( + os.path.join(self.checkpoint_dir, "checkpoint"), + map_location=self.device) + model.load_state_dict(model_state) + + self.model = model + + def __call__(self, flask_request): + images = torch.tensor(flask_request.json["images"]) + images = images.to(self.device) + outputs = self.model(images) + predicted = torch.max(outputs.data, 1)[1] + return {"result": predicted.numpy().tolist()} + + +####################################################################### +# We would like to have a fixed location where we store the currently +# active model. We call this directory ``model_dir``. Everytime we +# would like to update our model, we copy the checkpoint of the new +# model to this directory. We then create a new backend pointing to +# that directory, route all the traffic on our model endpoint to this +# backend, and then delete the old backends to free up some memory. +def serve_new_model(model_dir, checkpoint, config, metrics, day, gpu=False): + print("Serving checkpoint: {}".format(checkpoint)) + + checkpoint_path = _move_checkpoint_to_model_dir(model_dir, checkpoint, + config, metrics) + + serve.init() + backend_name = "mnist:day_{}".format(day) + + serve.create_backend(backend_name, MNISTBackend, checkpoint_path, config, + metrics, gpu) + + if "mnist" not in serve.list_endpoints(): + # First time we serve a model - create endpoint + serve.create_endpoint( + "mnist", backend=backend_name, route="/mnist", methods=["POST"]) + else: + # The endpoint already exists, route all traffic to the new model + # Here you could also implement an incremental rollout, where only + # a part of the traffic is sent to the new backend and the + # rest is sent to the existing backends. + serve.set_traffic("mnist", {backend_name: 1.0}) + + # Delete previous existing backends + for existing_backend in serve.list_backends(): + if existing_backend.startswith("mnist:day") and \ + existing_backend != backend_name: + serve.delete_backend(existing_backend) + + return True + + +def _move_checkpoint_to_model_dir(model_dir, checkpoint, config, metrics): + """Move backend checkpoint to a central `model_dir` on the head node. + If you would like to run Serve on multiple nodes, you might want to + move the checkpoint to a shared storage, like Amazon S3, instead.""" + os.makedirs(model_dir, 0o755, exist_ok=True) + + checkpoint_path = os.path.join(model_dir, "checkpoint") + meta_path = os.path.join(model_dir, "meta.json") + + if os.path.exists(checkpoint_path): + shutil.rmtree(checkpoint_path) + + shutil.copytree(checkpoint, checkpoint_path) + + with open(meta_path, "wt") as fp: + json.dump(dict(config=config, metrics=metrics), fp) + + return checkpoint_path + + +####################################################################### +# Since we would like to continue training from the current existing +# model, we introduce an utility function that fetches the currently +# served checkpoint as well as the hyperparameter config and achieved +# accuracy. +def get_current_model(model_dir): + checkpoint_path = os.path.join(model_dir, "checkpoint") + meta_path = os.path.join(model_dir, "meta.json") + + if not os.path.exists(checkpoint_path) or \ + not os.path.exists(meta_path): + return None, None, None + + with open(meta_path, "rt") as fp: + meta = json.load(fp) + + return checkpoint_path, meta["config"], meta["metrics"] + + +####################################################################### +# Putting everything together +# --------------------------- +# Now we only need to glue this code together. This is the main +# entrypoint of the script, and we will define three methods: +# +# 1. Train new model from scratch with all data +# 2. Continue training from existing model with new data only +# 3. Query the model with test data +# +# Internally, this will just call the ``tune_from_scratch`` and +# ``tune_from_existing()`` functions. +# Both training functions will then call ``serve_new_model()`` to serve +# the newly trained or updated model. + +# The query function will send a HTTP request to Serve with some +# test data obtained from the MNIST dataset. +if __name__ == "__main__": + """ + This script offers training a new model from scratch with all + available data, or continuing to train an existing model + with newly available data. + + For instance, we might get new data every day. Every Sunday, we + would like to train a new model from scratch. + + Naturally, we would like to use hyperparameter optimization to + find the best model for out data. + + First, we might train a model with all data available at this day: + + .. code-block:: bash + + python tune-serve-integration-mnist.py --from_scratch --day 0 + + On the coming days, we want to continue to train this model with + newly available data: + + .. code-block:: bash + + python tune-serve-integration-mnist.py --from_existing --day 1 + python tune-serve-integration-mnist.py --from_existing --day 2 + python tune-serve-integration-mnist.py --from_existing --day 3 + python tune-serve-integration-mnist.py --from_existing --day 4 + python tune-serve-integration-mnist.py --from_existing --day 5 + python tune-serve-integration-mnist.py --from_existing --day 6 + # Retrain from scratch every 7th day: + python tune-serve-integration-mnist.py --from_scratch --day 7 + + We can also use this script to query our served model + with some test data: + + .. code-block:: bash + + python tune-serve-integration-mnist.py --query 6 + Querying model with example #6. Label = 1, Response = 1, Correct = T + python tune-serve-integration-mnist.py --query 28 + Querying model with example #28. Label = 2, Response = 7, Correct = F + + """ + parser = argparse.ArgumentParser(description="MNIST Tune/Serve example") + parser.add_argument("--model_dir", type=str, default="~/mnist_tune_serve") + + parser.add_argument( + "--from_scratch", + action="store_true", + help="Train and select best model from scratch", + default=False) + + parser.add_argument( + "--from_existing", + action="store_true", + help="Train and select best model from existing model", + default=False) + + parser.add_argument( + "--day", + help="Indicate the day to simulate the amount of data available to us", + type=int, + default=0) + + parser.add_argument( + "--query", help="Query endpoint with example", type=int, default=-1) + + parser.add_argument( + "--smoke-test", + action="store_true", + help="Finish quickly for testing", + default=False) + + args = parser.parse_args() + + model_dir = os.path.expanduser(args.model_dir) + + if args.query >= 0: + import requests + + dataset = MNISTDataInterface("/tmp/mnist_data", max_days=0).dataset + data = dataset[args.query] + label = data[1] + + # Query our model + response = requests.post( + "http://localhost:8000/mnist", + json={"images": [data[0].numpy().tolist()]}) + + try: + pred = response.json()["result"][0] + except: # noqa: E722 + pred = -1 + + print("Querying model with example #{}. " + "Label = {}, Response = {}, Correct = {}".format( + args.query, label, pred, label == pred)) + sys.exit(0) + + gpus_per_trial = 0.5 if not args.smoke_test else 0. + serve_gpu = True if gpus_per_trial > 0 else False + num_samples = 8 if not args.smoke_test else 1 + num_epochs = 10 if not args.smoke_test else 1 + + if args.from_scratch: # train everyday from scratch + print("Start training job from scratch on day {}.".format(args.day)) + acc, config, best_checkpoint, num_examples = tune_from_scratch( + num_samples, num_epochs, gpus_per_trial, day=args.day) + print("Trained day {} from scratch on {} samples. " + "Best accuracy: {:.4f}. Best config: {}".format( + args.day, num_examples, acc, config)) + serve_new_model(model_dir, best_checkpoint, config, acc, args.day, + serve_gpu) + + if args.from_existing: + old_checkpoint, old_config, old_acc = get_current_model(model_dir) + if not old_checkpoint or not old_config or not old_acc: + print("No existing model found. Train one with --from_scratch " + "first.") + sys.exit(1) + acc, config, best_checkpoint, num_examples = tune_from_existing( + old_checkpoint, + old_config, + num_samples, + num_epochs, + gpus_per_trial, + day=args.day) + print("Trained day {} from existing on {} samples. " + "Best accuracy: {:.4f}. Best config: {}".format( + args.day, num_examples, acc, config)) + serve_new_model(model_dir, best_checkpoint, config, acc, args.day, + serve_gpu) + +####################################################################### +# That's it! We now have an end-to-end workflow to train and update a +# model every day with newly arrived data. Every week we might retrain +# the whole model. At every point in time we make sure to serve the +# model that achieved the best validation set accuracy. +# +# There are some ways we might extend this example. For instance, right +# now we only serve the latest trained model. We could also choose to +# route only a certain percentage of users to the new model, maybe to +# see if the new model really does it's job right. These kind of +# deployments are called :ref:`canary deployments `. +# These kind of deployments would also require us to keep more than one +# model in our ``model_dir`` - which should be quite easy: We could just +# create subdirectories for each training day. +# +# Still, this example should show you how easy it is to integrate the +# Ray libraries Ray Tune and Ray Serve in your workflow. While both tools +# also work independently of each other, they complement each other +# nicely and support a large number of use cases.