From 369cb833feb3de8dac344a50f4d1648ac24135c5 Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Sat, 3 Nov 2018 18:48:32 -0700 Subject: [PATCH] [rllib] Implement custom metrics (#3144) --- doc/source/custom_metric.png | Bin 0 -> 26534 bytes doc/source/rllib-env.rst | 2 +- doc/source/rllib-training.rst | 77 +++++++++++++++++- doc/source/rllib.rst | 1 + python/ray/rllib/agents/agent.py | 15 +++- python/ray/rllib/evaluation/episode.py | 22 +++-- python/ray/rllib/evaluation/metrics.py | 7 ++ .../ray/rllib/evaluation/policy_evaluator.py | 15 +++- python/ray/rllib/evaluation/sampler.py | 42 ++++++++-- .../examples/custom_metrics_and_callbacks.py | 66 +++++++++++++++ .../ray/rllib/test/test_policy_evaluator.py | 21 +++++ test/jenkins_tests/run_multi_node_tests.sh | 3 + 12 files changed, 248 insertions(+), 23 deletions(-) create mode 100644 doc/source/custom_metric.png create mode 100644 python/ray/rllib/examples/custom_metrics_and_callbacks.py diff --git a/doc/source/custom_metric.png b/doc/source/custom_metric.png new file mode 100644 index 0000000000000000000000000000000000000000..3f448613711a3b0ab1e67e8b5485bb3f1eeccc6a GIT binary patch literal 26534 zcmbTdWmp_t7p7Y{!GgQHLvVM3BxrDl;1JwB1bK0Hhu{|6Ed+OWcXtRr#ru6TbIrNt zoFC^`S65e8@7l8VTKDsGh@!j{G6FsV0077`(h|x509ge95IFFV;2BP=Fih|T%27;4 z1s)!LX+vQhe2eEKsp0g+*3`+>(7^;Sv$6eV!sPhX!NkPI(cISQ45~v207wBD2~icd z^y4;n9fG;0^VeZ>0-1hv+~`iO{!frRh%U#NsXeHB1w0KhcFVQVl{75@XKiPm!2?JP5s2Vqh_MCK#;d5l`~@HGW|emD1sPk8pGk};|mXM zS`z?Av3~FK_Z)9z*nKpGeJ|a8ci1sAm^=XRCx!PXbtLv^M)Z3-lK$++2l4MYpa|pb z1Ux#VGbFSqDzqq|NFxHipi)8~Lce|B4`j>#=R5Gz@7_Luh7idB-mZ*BO;V&WUrw>0 ztJo}LLu@yV~Zu~ z_#q4JzA?I{BGS*;hza~e<*b%ku4c+9`~-z9xY9_D15S|bK4(5Yx)#h(Z%GeT9NW$A##)Znv(QjRp^7y&mmfs8^?ig_*DU&geY&U`{|y@WdfV&GwBfH3 zk7aBQds*&+k}&SxIG$c(b}nFcQ;}=U4*2n!ZMDPNWJ`bDmV}@CW&1S}59mx*YRsFI zX&fZ~+L+9n^Ql@D{5~#(NSvOc8d~#U5D{K#csajV2$r-pp|JGz@|ThIzH&hS*IjIS!IEk3~!8_rU(=dZBNGe_*C)%5Xy0zgLs<+SA^P^%chB65w3P_z^Gq5B-_S2RU z*q)0Ur58Ct`W=_U(I4Sk>%3j?!B)!TM|u8fXRnhqOYxQ%lz5jR4Osxs`#p;wWW)r? zUAr(BuH(U=>++e=cbQWHIPeJWRJl2(iD(CAal>G1 zNJpVyc5{5(vqka-Ry`Uk@^d`adYxEp%bX(9)O9ualEZFQ8*1M!2jn)uUrR5~?T2@V zvnG^+-zaG)_|asiZX*6ke5V4E!q~ceSB!`t9EP9%5dV;V(18QMckGG`Z4Nr@)~+!$ zk0z{Pg5ZbADx<|MG3CF)_3pkCkA?@5UA_hXknP`SAg?Le7@WizZO`ws-(|+Utg580?rmVj6_EDn zXT-)+Ccmj|^~uo0&m>M%G1%#1(H%ppX6iG)2s>vJJOk#%FDph?#j9V;1CEJu$iemu z2e#0c5NM_6gd&@$csDXCKw%t01G~9`3C_qfaI$QGNSsn_VRc2tzax&V$~VxM!gbsa zd(|sTqu$0zH2H~Mi6Xg5f?ENVc%Uh5;|46}dME*vgv6zqf!0HnHoz3ihd)fT$j$f0pKaKw1 zs{WaTJyW4j8PB!7;z{BLsC_Ba{dqWPUadaa&WSmTFGC7A<%)t0F7Apqde4+Vm3< z6x=ZSvC{jlpI@gxH!h0dICXy1y4~&^kDy`ltQI#zK7ig*ox|lJIK;Px%KaVXQqlZJ zK!tv9qgC;H(;ztolJ4F(@_U-}Y`IZvQ$sZ`SL%E(fU&d2!o8T#H;5_b6)}X;vE=So00$A*6SLRcKYH8#?PV`zw zxX;q{IEEEjBc+_NBhK zM~OGH=GBk@GJ7(~S0lg;_@zG6(xTra^GOL)QfV;_rD^3>4F~Xh-aqZd%0FTRfX;^5 zTFbJdvL6FicWOi{iM*?OpQ#&K1sYy#tV~}DzIjGn-fM?Sr4nA0t44_JK4AT5VQM;J z3+yd{;NLnQ2oHgX>*2Wmku@G~S;QuXlw4NOHOUs<3FTK>Rj~BD$G-(v;koix>VZGTx{PvcEINsdU6h8EziESSO`u~kwZ&&>^f_X7lj*2@%hk*C zYpKD=7~vrn6u|^%klh$jA-Cq`6;lM9mCkL+b9qYsQoS+PX=QcGVN|WwTzL)PH}IR| zR4)s!8H1b10a3h--m5_;BEsfsT#>c;IZ~eoHoU+v5B;1une+>FgO_jB(KtiIeT!?Q zc-jcwFy$N($LVl~v+x&>X?a-FpW?}vpXNkoUQ$zkbO#6Kd0J6_U<`*CgA)0uX(B#^ zDiz8EjC&Z{TAe074RA`D_SDmRep0>n_;U2Z#K6?VFO?=4H4#_ffrUQ?mCQ{{>{G6) zO*Xxm)!uBFU%JrbZ?V$hquXRP)zv#L)q9LM)TGG8sxg)NL6&t> z-FRk)ubIyG({pp%4KupON(U-NWZqW8z=CY}op&afQK$+Y*Y-Z&=bf%Awjg|^6PiCP zTk4>~DRHULPS^DMgM#7m^x?W&hC8Ur6C?CI;Man8!0FKFibmnj_@3(9_oklRR#ZI-{1(Aok`4bnpPpX1l#9*MlKB7=^NRPc0-v?eul%l2iU_z?HU$MY z=6~C!Refuf;!^nQpgO47r{;BS&4To;_90o8yO>f^K6=VM8@QiHxh9@g*hjeLQ&_T) z;~s@NqPFrW7d!r z>N!V`($}R34RBy=A2?82;?Iq$n3Dsd*m0Ls{J6N|& z4@>A*9HyI{rEH$-nQ#uyAX3z*3o(+*Qg%-%`0t3zi+P(pzg&c^Y}A_$uEdV{x(p|@ z&Wx!FhbGJRNk3OqL+^E7ed|Ox(v~R!P=_wGlrC)XV*1)d%vi0&@;oMDPIlK7{=C&=Zo~YRNTRE>rv(J!dOAm*h9@6yjvyJ^hSL*C+ zZ^;iS{I2HTzrpoq`FDt$y2tJj)JF>>-EuDj+ z#jqkL2HA$4`7BLgMJmrQc6M&ni*V;aNgl2CM`^i{i8k#}5?vA-5Atd0b$`6p^gGH?QLu2%gtXQcHEMjn zRB&GR3Y)gz%OAx^aeZ6#uB`(5a=$CpCU0gieCa~&KL031T)Up=+m5^E$jjxA6%6c4 zvGA2NptClYr{?j;*yo<3GQdE$Vf1hZ=_z~rYAUa|B96Zf1M`vplV z$%9m~mJa_=h?wq(n8rjX?e3#~SriVtfYPPZ7@@(`G2X_;_uwsmltT6mTm5bB5lsm| z{mpqy#|nisU+Kv7W5^*K55^=|T#ai_^Vn{NAVR!eX|G&0?H5~9P_-eKYd4M*GSd%8 zNBN1&=3k-)N_>Fv`!Y#FHaz?$Z2S4iwwxIciVCOjMFORJ&uS?xf8=B`WYvrDLlGOh zz{C$N=6w~p$Hx&^qcIVMV0nea&QGAq-rE|R`#m8FrhAS&C`rjQm}UQ|?w_yfbF{y$ zYB=x$O(bq(|ULg*f_^ldpr4^L$<{G;WL;PgEas zu{86GnVs^vHL&@six-G!lzj+iyCo>)jzQa<+#~pnZp7c}Y!`Aq ztE+v*cImxX}nw*;af$vE8MjEY)5YoA2uvPZyCYgCXaU@NJ= zC>nl5Zs>RaW3vWs1QogD*iwEU!ruoGzxES?RKicE5#OpL(t4?O$>Hb4I_1f;DN$?b zjF1s=M3siw4bC1sUX5vT$y59!)=asl<4zG79(`8jTB7!Hvx)6JAA^1e7)tb}PJ2t3 z)bJ3vTmHFdw-rui_}UU0szC_h_E^LcHKj*Q`7wH`q3XH6pi*c&x#p&S6_*gR0I}D& z&gffsPckP(rF|??!dMyYN?UPlEzeQQioq}~qi{0`mj8#&c46i!9!OT~Qs?F&zdy9q zqyK>=Y+suKs$$XJU`UAJe|ZTs@Lzx;_FrBC5BwLP1pJqmpaTB|DDS@iKjDj*{911f zkT$~gWF{*+=#&87xvA49f)vf4uge|j?)vz~Lm{1~%Ip5Jh_dx+l>bvAW7B3R1Gxfu z=w|VkLN<2xyutnx7f*rAKt!C62_<=X-3F|<5ft6Ia>E00G)YX_1&Xxf3cu~sM@+S= zjb&8uZmzFU#bV>)@?@QNhg0I?G0W8m+|kRfSOoAO5jfl+{t$v*BP2%i-B`qh1Y`bJ#e}A>7U{n?!78XXI zKpTn@OtBb_8)S$}y{O_(iW|}U_wU~#<(u1EEk^8ow$Rz%)z+k_|J1u-N}`G7%FV2; z*;Z*d@#${M{2J{JI20(MWJl3_xcO5wW0_iDf-_WY?ch+|We}}iHndZ%!H7iRNcjhB zP*@Lo0=cKCgwpZRQCypmd*&XGyh{~v6kLG;xoo-GqnFS^PvpYgMCQM!>;Gfu{{L{Z z|1vj7zY7#p=#-mu9((duA|%+QrKP;kWUP0DeRPbHMDW1S7hCco^GI1awxHb#I_bD(O|(G^B0uL~da^g0Ow>fXh+ieD;LXXwWTHWCe z-tn#53Fr?oLlSgp3~)b|&I}Uc*wdfl$W47Aa|x9lvQMX!sSca5ti#2*h~0dLs+PW^ zWDYK-Xiag(Lueu*-82%$k+GJO)6KK#IbY&|iPz_~!It)_s`E(C_Ldzu@CxAkE-I>A z&-shZC{s>{INVxuwz2yL)&BZVt}KrHrZ=MbkItW5`E!YTc6ax=*PTjR;p_1qCMG-I zMGQ9c_C{}>wh#6)Un?_gUoS?EZ2R3G4qo8^;!4+(MNZDXRm5j3EZau#o@i9cM@aG+ zUT;f1CMkL99U+R@*RNlcE?@)2>{o7hL#{9mk2;e7noCI~Eu_~Ph#~;nyLwAr%`iYK zv3|NJNt5w{p`^nyT9GrXx9F4C8%Z4ojO1y|ZO^3T>Sz<7)aWfN5Xb6WKYLwY4khtr zR0IW4i%F~QNHsN;LUe9jaB*__9;LnaI!Q!Uh!@_vs>1k`NE9Q&&p_n*c|!&IjnMW> z50~?=yBq#hRtYk6%gp*~uA{Sj_qw&l?eo~em!VcV)BaSt2SLj8&Cq7_@bUB&n{0bR zD$`>lP`_N$aRoY7+^uc(yngL$kH_Y3Cj7jYVzxY3v+#X01lmEp3PS16yca5bdg@!ZF)tju`Up?}jBJm~WptUkNi zYI-taB~~cdc)z3|bYU&pwfK%Qc>nkq$@cO2rM+D!Z3O#-3T$$X?rl^nz8{Z#fUCAn z!)RLnTmWroTz~)8MZoNDFbWnH2JhXQd+P^;;>=uy?A@=ft~NF{7O@?koKOXX>k&J4 zsmSKDl?1A4>=}A^dX}qMsmL(nk$9b7_zJ!n+1jq{B(ZotNeO>e(1!*d@7nHL)`%`1 zp{(XA?#F^V3RUR&_?k1lHWvyCrs(JrtG{Whmo-{8BSw<4CxA0$atg=Uc>(IXi23$6 z8=I!R)fQ@@hbVu+E%D+ZB{RwdRZ4He(gS6W2Rqy zTUht^+{X)#%T=k1)zz1-Yl67tt3f)HnWXLwL%xesxsU$|-rv)?pW*r-#sJM9M$oXoI2Hfi= zXtJL0h8*-N&+63QuQcwKZpmaKi1k9}oQy4ZF%zp(l1onuyyU&=RBh6y7TC7)R+gVI z0bfsmaONQx<3Kj|69J`pzNWf@3D8`F=FKjy&sNJl5N;4l#M%C$)Q8J_wp? z3$lmsW&l9Ft#4yNfy;?pU}cBP{(@N+yj<4HQxneO5`UjSYFdj&chC}-&C}iQ-=43J z9r`c6V}kiA^jx;9=4^Pj9p!G_8Fz(0leukIc`~<){cvQ`6n@f5Pto})s_ucZc|*@ofmTU07LtN(4-b(3fiCL z1}FfG76Hf5!B?w~!8AM%zAq>3uUEEr$ZL5smRTN$6Vj9lFib+lQqi$^mU6KVH!c+e z7|_wYL2&-VXeF;N182Ukx0OE6h`_qLUN@2xRnWwFD{Gai1ANCC$tt$w2UK{ZH44O7 zN+nFrDh>T;J90q8B+Gb@JeA+62`_`YNbS*>&KNhCeU$bA3&CCNCjC+xzywecXmJ%$ zEFU4?GsbV^e~SHC+^FxhB7YPC-Os11+>1v6kuR@Kyexy{G+CmrYTKAbLoH)
OIi^#0vZal?6|F*GP$$T|7=bR1Le3Tp{5SF)xasNtFH~NPfOEIVyyaT#P2nuD$al=7A9BGxQXB%wm3d6GIjmO z$P@aYJJY~Bid$Y1tKg$C%z;DGjYq~YbsS3YlzBSM9LB{>*BCTrzF(I+WSgFE4_0Cr z4s$3unZ)g5&y_$otEMJfTD-eJ=IQCF$xz||Lw#lC*xprse*OT%tW85yJ5(M6^a>$J*8K=$`L$ntlKlB+$5%m`3dT zbg*>Rs9HJK{xJC6*2mF%-oaO)pZzg^bMPHyV10s9<7W2kehj>Cl(y>O6cfo*X zjq0dpYJ9xi%|2I$Wi5m7KAXmvjq}>CNf4_eaa3B7CYq$-=RO)`Xf7KPg1R3gH@y|n z_pL|+%frrxbQ?W3c8(Glcs2;t1YdsF3Nvt31m6)pU!AE}w3Za@kIE6e>SFfzJ9DqO zctQZyYcJvkdK@c*Bs*S zp%h{CkC}-=0bb7S2!INvF@Ag<%B#H@>9mXl@ODikls;nYIFZ|lbgbnzhi{yWr3D~q(y;RO_GMl2*`Fn{0yGyYp%faJ&56^~I_pw`C_hr8JOsj}`J z$qg(})G~z&{LPYI(2W6g-@ISsfS+E7sf6?~SZu_{{mijIu=%WlM#_@40jv`sPOs?{ z$xn~Fbl|eFfl1`7<01y&M`h1*NM40lKT0PhDjFNK`b#9vL=mEMWS}INdF}&89lhQ= zXcU8=|LVwGopt{t`9pKg>;?B#I|08K<#Te$NV`%gZ?aBa&haI|1d<#2yc7ccC|lkx zMB2pM)vw50aRLD7@e_L|Lhzfz;v!olQnoxv;n~6+_eKL7yoIucb`B2?Vz|RZNdH}X zf{e%_a4_wK!4ZmJ5}a+Jb-T_jUpQsJiWEgdhsc~%+R%_xu#@hSNNGWTnnVgOU--qP zO0RNm!>*rs_Q1)3uy7_4>C`!fhcFVXHs12(Mv47|Y4|1bxKORQpB|;+^D}=086Bg! zphbgF19D)5x(rzV6Ebr&1~9^~Bsl>vfrlYO&6!sW&{_DzkHv|K&Cnvxij|Lry6y=Y z2L9s(fHtK66XcJt5U3%FS|E13p8*CT0$8jz!|%J@t0HjAu)S^kql?tm zLXddO9|qC*y~wd04>qF)scq{!B}`rBq~p7=J@<|^mKKeuzC1-*W1P|%ORy`CjS^(3 zdL_fjaPiF%+N=EBt5Uqz3SWCHZgWg{bp1$cp!FzE%BFi|Thiu<4g2|1gj^^%$lwp5 z_Z*-8UYv7DDw&O@rvyZUaqHpXL3;EUE5S6kQ7P|HI=P`kQrfkyfR5ruWBtNSO#fpZ zecFo6+|kWMyGuH+WAE9WRQ?O1zuoCFZ{$|)VE=|4s#tSN3s{rh9&08IVa@a@DPd}B zYg5tj3CGsdjhaM}M+Gd*{JBNQm6u_fMP43sr`I_N{KH&?k=|FLTKap5D=nhn0Eu(5 zEwLd=w3wQjnzAOC<$)#(HD+U`#>Nr}F)KDS0Y88Klt_>*onZ#+-oh}WL`h7^1X<^s z8Fi_I3C4=6qF7Qihg|(*tUs9Ln=1t2Q@YsDA@I{0g&0`K@4!NIov$~k;7j^t=v(gL zl0*WqqC@%*JK+n80{sf1(;k3m0nzlk0z4R)cb4?3cwSyjKB}z<@K3EF<*qO%gb&;j z(J6BL3=o}%(9`dt@t_W&rxap?BJ(8J#h=ZaPJ1O1xJ6!0NS?JLJq1P6CnTfs!davN zQWPjcQg*oMcSeDMS{^xxFNOu+rC7iWg)~c+A(4*?Vl;#f1o>#yE-F&!*x72!VX9Zs z5Yg&=l<)Mn#;X^CijEu4_X+L-dcgkv{^FB?h0p3YR~+BM_Lr*zhBImWhaW(+VP^tt ze>jA*|2%nMPKRqsy>T%LuxfrKeA#rz7@PeC!o5Wh?ZEo`rAedE_^go;l`)OOb&@xS+G* zlBcD*IlUT>8`+4D&*$B1Ek_k%!QBh<^C~K>qB)8>uHN#SPXP}PECr>V@N83}8Kl@U zljE*T)5}Q}a?|7Or0zAkJ}f__t-)FqKz%naVUk7S0Rz6#k5C!xUeW=Zf;kJx;d6gZqVY&pLVyPAq6$b{RSXq!p&bIKie(lp8&p7^tIE?MGd-R zn6pJ_lQ0F#+t9Sv4Z03xVK`wGPl45kv39qM9^H$Jf<6-8hop|Dn^ev@WLNbb*6?Gr zAJkI)l~oXem}Kk-;Jk>+hM?6?8jQTarYJ*4jXx%QwcgOt0PddtSJ~FaLEzR?R zh=_=B=;KZlgIHAn>cSSei;Ih8ofgO=oLyWFZ`h)I{7Ic$TuNpv-FvDTLAD{K#C9j! zury+yejLQs*4AcWVL{=N3)ZX}H(*AxoHjbm2T5`ZEg;G^M+`V3>p_NqGtqk!{?wTU zh^{Ow;%1yKV@1B1qT=g`N<7Ns>#BF$T+)4coHQn${hA)HGE`iUvsWROBdwr6J!A47 z8j?N%0Z2{LK5e?IKr3d%{^Z7wNy5d;tGVi5tU@0s_CcqtO2OYoCSOJGvmWCRH8pi_ zPtSl-FPRRry0UWY@rP&+4-b&JnasTdBc*YNzQC9xAx(=$<=iLu7~L`u>S(IDRDIoZ zes?KCjhvT_&89LdU?T2&-V|qjkHw?Pn)Yr4K^hvx6&@2tKSW)K*U~{+vYYc_Fd{6a zu>HDic^IppWG}wJZAy;&Q+KqXhslM~K z+ex74<{nPtW@$aSx_9EsIKF~L2=p&F*f6R~C+RFyq-CaN$&kmv!J+ZA5a%Rqkw>L3 zKek6J-1I3{DJe!p1cb5_T&jFJvT|t`-ay}k&&Y+g`seQP=#nqidL=Ljs|S~^BUSn4 zju9_@&;ZRt50_fFx^zoTU&e=apLRB@%t;2>u?%AtLkJc+<|KaZy~O2oZAuv}bvt#C z?09=e!XPRWyKg_{=SN|Boe*%8yV+fR1m@eHH&Rm<>$tJnUlw!Bvb@Ds%_e`|7cu~V zdWumlD?BH-Ea8c;FyV2;ae7b!3L zO|RtHNB2B~`E@scQPaZLk`2!)vW?s0bq-DLQUqAP zae3L-(c%mp#y8ziWP6mJm|WBN36V|j=?_MJ;}W~S`CFp~#3(KaW-n2P zWjp|3FzLwGbwz-PbHbGz24-jzvlLw=pp##Bp3*2R%JUf$hNXgwu!HI)(XZ0)jF%WfIFNwe73F8o1LphMGH|h=m?=;H`?<%<>nt~ z%M9|jlIx=lVfGhAO;{LIp3^^3vQ=X5Ss*&gYR^S>AwSI7(cJ0@T}&7#KHWRAJgo=6 zhwz)*3S-}=NFsdvX;EC}E^8Yt;2L|fOQnKniqR?^$qxl=-%is$A>+nRmq23yqr^R^ zU~MPzGfog5-Re*Jzby`lQd4PdFh1BEV%CNPh@%>ci|@79yw+$UjCXWY>$tu=74LYCSn)Xx#PQ**+Mbsx%#NL&&OpQ! zTd(7Kx%e{9g2GdJfOT&58>w4pcqV{BWc3UM5{6+jVSq%zF z?;T7PQV8o~_C!WTO6#QFL1Qk6c$dxcOEzZ|4viMs9-TFc(x}v=Vzgeay1BGpLdr)p zH#Z}}itFp^Z}&j?MMk33X%Qtb;AZZD8bj^sR}hxHBNNJEDN!*S%S48TRN^%WYaI>V zx&V2x0xRyyl{4j;Kbbo5f%ig zlMkiO#201d-%H-!-WL}arZv2apT-ZIl=9IfCk~uG#{NM`Nkt`j z;}-=!Sl)#Bw54-x@y9uL5lC*hGHB1SP;Z}oWK_VxjP2R^Xq*>qwn@qVsP4RiUvbFd z#n$8UNoMB0Oe!|W$wUmE%=j;{Cj9+B$ey7Ilro8oKSi1Krt{GlJXeCCN3-#hHI1Z# ztwMP-3EU}laftefW)ehA#g8;x*GD;nt8x1X#$Z)Q9CRUBb>jT3u_9}HiZEbBoK_MK# z6PahxoRibllL~>!#Ke?}f+r*g?&r5RHx_jm&WAFfoY7ldcB}+3md+~{OMEJ zaSZIUPsfO9n9Jyyza8P+++3f1daKcj4foiUO@p>x6{rodvpfD71y>wk@OT^S+v?p{ zT1l)`OUrrO_Ov(Fc78mTnJz4BIGhLARXpl|dEKw*ku>g?>%tbBj&hA0#qAy42KRB6 zhhz&gY#IDiRa6nwp)F&FSJV9C9rB?aQ}0 z1hmSveB#(u_V)Gz0|V7?9ZID$VK^*_g0Se4O6ofHfnZjEiO@5nuFsVB8H95}%*@OT zR@+t^lA9N9U@Zhnchr^h;0(7J>`zF7H-_1-g^~x2w*|px0^3 zIt9TSbIMpVLIeaPdRav=5lQ8`Yuzx8R%vqtu?>th>phxGdXWMgbw96p4C}9wZLuEA z{E>iWQCramr_9jNH^`x|vhDLGQ#QdH*{ndIlOZ6~azOY&b0%cx$XPy&!r%jLg+oz8 zlLh{M(0fmIF-QD55R)UHUafF|{uPTl;fAYtv7^F7BE0VlW0azOmT*Q9y{rKz2ImN2)=t&j3+0GW9a9;q?=mBu+J?_fR z#XdR!>BL>W)%VUWmyH}g79T0x@3zWVo0p%!je5pr6JD7m-L&QJ6*f#0B-(xWWtj0D zUHu$O3f_*OMYS9{AmS$yC|YmqW$}*r9+_6?ZiU*bNu?GNAkk7nKl)fu zuy$6Oa(v{I?=kwJE{?|B^mc-L;}3h;!=9NqeG^vd4vP(W4}TFqR3IBzZ)E>#`uN#l zikBb0sjwDQqlYzqos7~W$5*#ycl2AkPk%uc#L%e{$|sldLX*O^!IsxKb!W*6zY0e z5*Bz><3C+$Vv&wqga#b6Lp@^Dc70rz@u0H}43O_SPhQrS!{0sfTER;LKs8H&R5WM+ zhps{R5h|pb9 zDT#H@DkkqPnGMiFV!m@dEJ(4HL;v5!u2sMG&?lC*>WcM>I_{N=ygpxa2b8IaiMso) z$*0z(Q|1?W@8`WKbv;xnNMPFf4kB>aNRcMF4oowGB$0p{d7L0c!cGE8VErAso`0N= z^U1ya4;`O_2p3{Mp!3o_)Rsy42y;5?_~qN{b+xENgi_safG>49X%v95tw&sYoc8=p zG+kPx{bjnNW%Tcn6*4$2=oGS_hQ6P18m|1n%9%wwMx$gGH^be&i#IVeM*u*P=(oHy zi>1;yid;p$_gy9?h4Rb3{_ zBb;7=B>Ky$LPsXM?HPUd?9k0!A-hO+M1r4xtceydu4rsQsOyo!Jd{06SHzEF`Qr5B z;(*}>OfNa)tC&6&ei>I46uVCnQiyi4Qp0*ci;-#0u#J#fiaCow)A)bMQ^$r{>DK&WSJZB0+VnypA%(IP}+?<3~@j5DQb z0G)D>Ezg?vfxehi#xi?-rGodQg49@Xfp3ub^1a5Xdk8v;FOaj2KA8U`c z3aT`gv~96Q=8cXy9?t#n>P7=>zF11?LLC?v8KNIV_`C5qL7-D{qq?mvP?gJd&grs-4C>z_y zhkx2OLx(|)2jSZsVX1D_Wy$SrjR@`cWESKSDyH<4?Ha7)UmK=e%erII$^Mt z4EcpEA}}vZ8lv(hk-Ms0ePTIJZ1DI=2JLi)bG9i;sN-upKmD>ewqqD<3hYHt+?MIDU^KR`Q9vn=eR9T81d;p=p~K&9Z~Lk);>N ztH?I6Pv3-K_;LhaU$dFj3<=m(I&W`Dl=|Ht()RLTA=9Tm%wb~rjp9&-ZS9-3Q1Hq- zzV0~6lckZnoIK>`ja5C~6Xl3Z9vp-B!lg zcDb7SCsl3jPg5MW$?sQ|H{CL3QYphUlrRg)naTU?4-O8h$1uR*EG#S>BF33Gx*7E8 zaLY`hlzH>aNP$KmW|z=SayBG(K4$Voj9~}d)JZynZZTw_)#>AI(dvgtX%X^qhUhGg zzi;~%{h`@{eYe>e9zHt9wL<0OT5=Jj-?D`21}%X|Tq>6YNnU?h&Y4!x%P#rN3Ekn% z#Z!3*jyqjH&=)oLx)s-4=zHlVO$GpQ18y@jL;Bu3*|ecjQ7@9H0SuM#ynxMiI)((m z%D$fkq!h0UbTLX5{>1L1`~3w*(qoZlMY7rWoE*Y0&*eL^?A)JHpLTMa>pia^SSsY# zJGPS`$zj*O}D%P2SXiB=;i^5ml>)WMQ$A{|_ zkdksbjZWsza<6*1{~t-|4JaQ+-w9%g?qU{!og(%g4CLkVibSrkLKh{>X~HWe&6EK; zC|UbBYloOXQEdL!;vLf@9WH(?f(7~&ME|tme^7+c9tg7^w&S8A=TlNG-S5@M^h+Gb z`mJF*h=$({<39vRV38@#2oH@+bn`sLl6DQ5@c{F!@hJu)lKpiGgTr^{KS<#nV!*C& zpSP|*ub(?|<{@-x5!52ntK=?G$)%Mir)XGOSz9A%N>GH&sjY36%oaYdGnTTj1vP{> zBA*(&!&6aN$+8#*>Mmpf5=YGKm6ZyhW1>!c_dEG_wM@%ROhJEg8K^+Mh83U8fz$lr zD^$=*=1(eNwdGBBB=7<4gy`SD4jMprN3af%;?U8=#?izm?LYN@`M{xLg(~dC5b05? zPV+HCy_PEdT5=#Z6i1YN8b=Qj^WHL@M{6CtBie!-`2dxbG-&SuGf8;mFJIo~@3)ty zdpbJ0351Y02bt=A2IqhMW0xuy=*z{3J{o8DD&qw||41dM+uBz47`@Z>kb)el{w_kg z7?w*Lii-ZNvj6g>OY{k#sm&KSuBTBi7d2V!9oSb zIXUkJ3hbu`!5d7@)*t7bKcf0*8%`iNP{RsOH!Ih$xlX#e%jo%C}5ITc;Qi zQe>w|vjSt@ZIVJ7mWorr?d`9dY#;t&b2dmTgG^YI!DH$W z2O~g$(E{}LopX&{xwN#^)v2yBLW^cwG_P$zj5n^F&dxek|8sJ9xakfGVMF9h;UlJb z79MT_GEINpuq83nfN4zx%8X+SyuU%sLJ*5VB23m8Sd!r+SlCfE>M%!cQOd-sTPrAx z{SHKwI+n;VumS1(H^`?^`8KCFZY`&@zj`1UFU`*YNmh#f%bfxMsN?8bPaK-AJ=!K{ zYocre#c~vBr!6@Fa}NAGC1P>x!YN+c)z(Iy6e#n9n+Nx+U9cH+%*>f~}PB zvWDDZ9|xic35v@W8XzSjBWq=X_wO?J$HihFMu|RwL(WqK8CAo2y035O3gOg_=QDL2 zNICaL(r`#fSX$&kC*^<6fZQ|C$sdg7{-piPTt6}TnblCC!^3#y-mRQ@PSyhFrfK87 zvhAb?>#*Q#)2#ZeOF7OUErRBx9SADe1U@ z;qp3`pBxwTcgmM(l8}Of$ZQtdWl2iHRq(sM&nFfQ{W4*0@HU^Y!1=@*JDoi58v($H<3>6W1%N~^GF@Ma3}&MHbiFC_w`3=sxKkm^-;sBN8Q?q`*YnI zWGBYX7>E&GlRDa;$Hm_#CM3W??r{!*KjLAI&k1E*@Wx*$SKl4ipN_mw)bm|w-pqY< zyxg_`{yz5gsZ}=9ft-7ME$4@=t|A*Ga6w;UTzosh_#6>JFSg#N{~*A=f#|OuV57%( z3J*j-PU0eZP$yS`U`5yY1`Da17BjTcYbSMg7yJJzV*TDz@VZvsB*B#r!B>Se?nMt1 z-vb--W&$Cw8Gc^xc0FslOS5gi;0Mm#+CVQ`G4HM3er1tcHw+a8kgcvPi{H+tf4^d~ z_(oYpJWeUW0=1d7Rk~G_=M(|*`;&1kWpY=UUfX7zLS&1Mjp+I7$<|lirqZP)e&sJr zvOArdleM*hW=s74wj@@pXVWf}R0+*7!VUclIwPZgRQ}QX&ihlb%Bp=|xq`qwJm4nS z&htaaVL&=rB}jf|5(M0Lg;q`o4;joGCaz+Vu6E5soM^2Q-t!m+KekidRO!K?H?C-+ z_M9ofg3*8MT}(Sgb=x@7f5jg|?9RJkMZ#cXga2@$1TC6NzWceVclx2S&1DB#s!h*U zYw>rMz_ZRs)uJTgmxgbT?aub8eOuowEnjB2N$i@I=k&9#x?jD_jRvG~+hQRB7?=g> z9Z(hkE7896(=CkUJ)4V(ovQsEMKIgYpl10z9Gqu?fDDuB>Pcv*8iKF0 zn$fSMsYx?GR*L1|`1lyKIS6{`Yip;?9U)6v$HwgLnu2az&;~p?$+pP!D!F<0_>C)V z?JP;1vCkf)p`ax5uLF3`y>N=IQX{xKCMAW}$ZYrkI?j#e+^sds_hrYm-SKrx_+f*< z_Nk)mxG&hJh2{Un8w#^V)2e6dwsdyqNXYG_5b*c;rARp>mnS6LONeBsdZNHcQw`LD zRM18$tln)O4!lpKUA4I5f~m80zOQ7LEsa1!E?^vz&vg^ZiURjp`ogH!ykOg}78#H> zse5-GEX!kG>qooo2=}h;a}j9R5BTN|jQT0B&PaT#Z+BbAzx*;-84>n)qIq2tfA8|I zo3aQ6G-ACeaOd}6R!hx;Z?rQ=HnV10qlD=^qpCijeOypLU8kHkb&xIm3p78MZtmb| zIrHg)E^rXF#mnx1Sufaq67WrzgAAY&SFq`GAy7GWP+eWUjRks~{}*$M&WuS+q^{Ix zSrPJh>pl#t=dZ+^e|qu@3w0k8tbp~?mX3?CSC4@mhsrkh5s=^yeOn>7GE-Ai-B$vA_AxY?_&?jpYKhoQpJz&+yFxniOvh#x5rv)H zGtW*=BDi|3Y;9S{C`()+<#X`~t+B>|ZDtCq8c8RM4|;n5D!rZxU)Rs0Wy`vc3C6OT z&C?H{KY9gRU$Xg|c~K=v%(~WWc$Li__obDD%h#QPXKm_K@V^t(O(8@idmi=g2RT8r zWw^T&59s6u|5iyWySk^w6BhJB3V4~cTc536lRCUhx2&q3V_n-I!wriC2T2wEio_zaSPM~ROgKe{cM)`aA`Etn)ps=f>k4h94S zymi9<-DBQJ|Cc!Al{Ey1MGAfR?X>*VSNC*;wBqX@I1=k@!a0w8c;OI;)@8rkvTyT84DDEATRE^zpPiooru0i+DX%^(PZTXhodHwjU8f;hVCh=M#4 zD;L@CJDqyLDA^c764b8vD6#$5 zJU@cWZt8MHe_WH>^2d$~)9P8&CC&&`^}OuZ7yLC^t%X5*xId;fuN0IwcF0IX6nr(l zrNi=IkG7*BiN+}zk9`bJ)%*tx_3hFiLFx?Xx3S0@2K{G{L%3Ya6rd`xc2I8Ud+-Y6B&G~oBh=iA7QP7k)M3Nrc>lf}2wMeYsfH_cM20a|n6q2G#aWGJ$ zlhv`)$a!+N69-wQ=SC(k|Kig@%XZR!^dga;m+~z1>p-x{@Ax6x+T6a^@<#fuXtN$7 zkh=K{|Lvy7+v z#zwve$lI|(&?14?gdj-)fe6M|CdcooKd>l3^<_v$SC_bhrNiK7FEmg8U#p1bJ2E{D z*a>k)a6neTq}z0E}A;iGdy=!Z&}Ag?vcyzejp7kznWDIuklG|gNik#2%RNZF8akc1 zN)ya5f4|eX=EffrlCyf&ED_=R++%`}dmlCBFc4BUR3sJpbvXnTvCZ!}M5$_90Vi<= zXlsv;q%6p8AdfWv9*FbaIYI8de#f%`_obj|*G&DXnF{T3vaFG;x2o3};>TJiLuKTx z8vvtu*eU_TGZG>CJ7R{7wBpwZ-{wKBeSV(YCu)(Wm%W;){EsHQ4Y!Y@?E*lY_|D7r zlh}Iu4;ZNSq&`(H#8DQEyvdOr${q&PrL0gC#M z0XGz(jaaKfptTVG4pPT2_vLBxCI<8Vf=hqv{L9(>|Ky2bo5*r%q z49jCT8Ipm%cX4q6cx8Pn81{R1mQGShxgMkm(}%b0>tRwxwR30go>fc=Ln%CK7jSYH zu?n^fcp!!6_IS(v4t|g$%W=qS3=~pBe+O2fk?!kgAN>;2=Cd<{`A9(9YtSmmOQ{%a z;Y9)B8A|2rwi~T?a^!vuxP6M0^~hB7+6exx$Wb=$K7N@NKjk& zAV>L#6j@nFsHESdi`|ni(z&WqO&R~9F>33T*Q&hNrb4#8=kd5Soy+UFyM^1{GcquQ z1)1Sv*>3H2cUCjgF*sWe=stn?E(;epAaJT>9r;TyCYRaSFT$R9-rNl}>??FB}UqgeTG1{b-H!vxB_@0&fzS!**9U-ZqUN{&u#@Kyi zgK1>EH#{)gdQiCs1U67~YiGAdygO;oeJ=kpfXDqbp;O=OuzlY@f?T4)u|cZ3A+}!$ za)@Y;c!--FbBY=*hAjCuX5pkt*dSIb--vU84F@gggOoQh9pUfCE|nS$U!UsoIBa?x zM~o8h%{S|J>hCj;PTJVeFfiDCd;-xCwt3p}oT3~>Qd0lN&51r8yvE7aup>Mqeq2eQ zqoi4nZRU5o)rO5DW`6e9XQYseXVncV##3I_JXARgo0yCFl?dLn5Lbi?w6>3i&&MPw zr329^-bHDPa9t{pSy4YGd7k(kSh(PACZu=LWb2#_v%c1t%jzAcKfJ2$x!UTG#I!u}Ig0*(_)`u6Yn= z<-CeuXr#^1!wdw7@QJV-L@+lZ%JeG*DRijfZ;sY9`_*pI`BN26=W;^kNhUVAKMGZM#+6Wv7gagz2X0@iS?sTw|-rn9?T%z{# zS#>;-*CpwlQgG+>@2rlx(IGTaNlCxBQZ>ixi5Yxdk3rVkoB8_6Zuy?&(w82&P8n6r zzOsR+{Q22j8%g+QP3*(Tqg`={dylB#v-gRpq)4KXSU#6bx-EazYUcK?uCBuNp$x{A z>U6O>&F(7m=F(D9boBJGQhMWGIcS800GToV!!n1sj!xP;GHg$bA5BgCqun`G8pWU& z$GNfRO&G8%MMYI=uUxyN@{*EXS^GY=E1cYM)NXVxt{fwF@a+ta!z8-7?7CPK{JeQn zSju$08Z4hmXnU*FyqKX{%j$1;aUCo<1WXRUp z)%9Joxc?${X#+VnR%N<$%hJQ9T88Sg<%mxpXEC#O!&s@VP1;yb4=0_=Vv1DjQc)1TodU>QA1k&e?K2leA57iome^_M*=o)wyyn@qHeq5F>G3M&f@YLWF^2!?7_@Qm zr(5#M5?Yl@xd2$S*i6an%`y?Xul4mMg9hr@^z`<=xi~%5%YPY+&&c&k_U5f94(hWA z{%Ia=QmA!RdF9ZbGtQckUnvt~{$;^LMah#ue{oJ4Q+!|yA|fu|iU8;Th1gMifIkfm z4)N#Pv-notidu(T+;_ozX^7DBedOR~w+`-_#;O*iKO&xjAF(Pc@LnIffnVldk6 zq!QbW(_M4UqY>-WT_m=*b-W+KICH9+!zqm z8lV3)5t^;fg+Y_~Iau9H$!hI4i}G!lbXYxH+2jXPQc_-bS4^5S($XsCD?@dmDk_tJ z#3ZnkrkAj~s-G!DKK2K6VJ{k2qRkdoRdH}|DCdrY-MTt%78ZpOMv?EZzW)9{2e9g1 zRMO!B%7f7YHYp|J8XTy5SuB@L95%E{gR#FtJWEl*6*xZ**{9d`Rbr_QUK+)mf`U3D zOc2}F{!ZP3;*rVSvs7Lu693y~3feT^)YiZsqF78v@Ucq+O#3$n2( zji7i0^KL;PMBo~Qi>=QTN{RStysx`=qS@Fh8S(o4FrBIWU-LvwwtA;?T4_u(3pi}v zMAfQ#mTfH5Sdj}|eliIsTA0{XovS@)aAUW*Iu>(o^xO(tKnn_QD4a<=IbeDA*iR|{ zbQQlmy_=?WQ%9+JG$4&{9%*7 z;z21uD)+Lu<%GIr)YMRYD)zHa^DvmX7Y7D zR-b_lQhvGSiD*iyA5$SQQK4$Tj%+)9b%K9Ed}$94yTBm2{!vzTS?P&F$e`V1>ZUZ7 zVbG0vcUH;{;4!xB_}~k^HUUiF3<$r*_#?kJe2>;9scL+lz)DU=ra>RujzcY&m>4^# z+ZQUv=>-iqQ8WDSzkpmu`ueV3*6|tCO%baVdC_)}Az?)d-K9Yw8VuiIzAs$5@IKJ^ z()9h*-R)H~q>An>ml<~y5&5!RIYAIAp0Df<+KB85iwg?qXld=dzQ91EDQ~LW>5LA? ztI%GURAYV zA#gw4Ry3isM}!=bx3{;4VM3y3UcY`lpxYCe!@^GkF*N2Cj7&;*X$m;B1 z>3-!KF&@`nPx!fH_E<;=x?SPm;Gor9P4V^#*&Ud;@rKX&g{RK1{&?n#zcU|Bc_K=5 znr90?GGqUt@K<;d8y~-Q(}ndFl=e(Hv>P0;Yc*2kOh9n5y85fQrm5+MC;Hh7= z<&nWX_uepi2fDZZc8_8Q8fm(>bDjfOLJdxK>J7Fl^!HkImfc|s<>jmd!RQ&Rz}5SB zC-OS|oXE#XnYQ(dZlLc}Glb~3yP$wky27YHqCjmQgaUy;A8-K#g2ICUUa@>r9^P>G zpn+E(tOsBLX=_gqYhB?#ifR9pS5$PWzyA!hy_qrx=rwU;aplL4VE5D0(J|l!k4UZn zzYZXO#>&coaiv@?&NYmVj!rEOm$x(tm?(7gB6f8Qp%jO$LozZl}>6mN{wQr zeC1Qnc*02f3t(8yV$du|7YmcG?P)g4R${%MsG8ersnfv=?hNi4Bgq;CBqd-8>LyiH zR1#%!fPJ$obecJ*Jj)d}?Uj<2e$x-;Yv`AwHltHTOJ*vdn&UF`zX}!4v>{I(G_IWS z_Yzcmfr|w;5(az@0(5AH0YR5}^^@xqDu1pneJH}SU-kM;$ z4R?BKMFks(Gpb5&HsW)vM@L2b^c3ZTB=!Voj8kzncw=kpTR=dK*|_+; zxr4p^Y_$bG9v+o-^~X}3v(BbKd}f`$v(<%l`HwqTgzj%DM_iozXukIi4&pY?&dzFt zijP-Yd;~ZOq_DGCfft%L*RJur0ZSbMK^2jE`0@Imr!S1^Wq|tdZvOwKE&OLIHS%NP zA=0uh8!dNUC5M`oP3V_1sg-dUZ6_#_Q+w1LVAam-d@$)25b*3-R%E_0YEjjkxy&ml zDA2v^zsXI%&N3Kma5}g+;Bs8{QQlv=x3^ENb$O{`XSY+7=+NYdux%3cd^|1rZEaqjzb5Ue%oOH9S7>t9IQ3gWemN&6+2e>)PdLVN zs9ZYl+(jg5SY<7Ie3%e@c!O3}N@m=Y?$>v$B<%aSYOmn)vY@M<@B=Gw6MuTKwd1C| z8THSQhQN>=-NgIhQQ+^o53a{TtWDEL!5($o|C@GBqD}H@_efR8oopeX&*70kO)t_mbG%gzv@+>xj5P4%60DIRy$jdnt zdHg9>X%xDhUtl`R(pPuxs$}-6atI$Z<~v(r2WL?$o(dP5t#my)77vI|*X1@m%u7kp zN*=6n9_*SKz7IXyeGnBG+K?Td^t9^rbZ1L>agP7*kCzI#K}4;I@nK7z-MgVH^MsaeHE_OIbFA+S4I zH44z1DqZ5Y*nB-TDCSaE+kiSxefa)Az1L0*djtL8z(5+CQ49)4`plBI zcgyFCrL)~hb`oJQZOuQLx@9Zi;v(rCnR=zOnf@xtR_tRx#cX@NK$qLRf72^3g-AuI zq6Iijh2JGlfx3;IW$CE$k5B@aqn}31Kco9MY?{x+r%O@|+0^ar^+iNTvAw9SIx1%@ zD~*R{?pBzXf3Bqnd4p;(KdBzcg|6RX-V>+K=lo40LyeYdt!_io-X;kR-nu^pzTEHa zR){^d($`n7aU49Uy#K{`LOg0j>wPeh;FS^9)b#TX=^6_P5-Fcgx8ssO2j7n{Nne%r zgEb?OD-R5SxkuIQ^_kn}GtIt;rI6&i@JA=aAa2W7=5;iE?vn1)it`|^bz~x18k(^1 z;ElMo)meO8JT5g2W#!lSON-?dWo19gGH96YhK%WA={S^CRCCN^l%m^2)l4G`M)u(YibKE2aLT~~ob=}$+F|^)9{!h`q8Dj8?{vo#oMz?~X#l)acH=O4BbU&8k^5_=sQlj2yIlPb59iD3YqCTi^m45-R zHcC^c$<+<|GmCsI;4M{JwFM^?WezTWJpcWTBiW72v45@0K|AbcRBP*9#dNvV+_jzl zSO&(MwbvG_tFxG|M_k4?s~ayY8JSy~^*T^)R_gh)=a0|+@DScr8tw7*$4Es|OZrib zAI_cCGR}Ttd19^)`>%9dyzX7)hff|7j_Q5md_iNF+%k!5C1qtaZtt%Hg+d4gJxsg9 zj&9y$^((ndcYb0SO0TtT_q|Hxdzzd#@N_BtPH(Bj&tdZ}F&Br#+FjdvMyT;Yi_kDI z09A3mqSvy79oP7F%m-|c#I@OE8RKj=Zf@=bwk6L}TY53fGx=dTA}(v0PoF-;Grj3H z4&nE_Mv;h2&U>-BB#v8zPUZi)pZ7@n+&gQw+RBjMVRJ~(>DqK`xjQ}i?KdO!Op`Hf zR}HI;#y^YUfu?(x4ib63zOaFTzc1NycuQr7k}iMhniNbH|LmA#H<1KDKjs#Ed_~c3b#{B$^Tq-Yz+H9t%)?z8W&BmtF?nI{O zx4NbqJ_}%G!2H`VYijpAeCE5qmz-faX7x8*<9gHKbpI9b?dYxzudN;|Hvc)_BV5w1 z%`YmNZ+ve(U+1>}?WsQ&Xmqt({lI0jx#Oq;9sbdH=@2mr zjDQ5cW2U~C$HcefG#-Z&W5`M#U)yLAS*Y4v+Rjk}o?TwvdT@s>H3h#0^#rdXh}Vez zPg%?a6jcU{*}x z)?w75Kdu`fCG&xnb{(&LV@QR3Y!t;KMfJx~A2z3%`E6JvSVjevpT? zvsv>@x5C8yXkxq<9&B-N90;Jqj$h+p zaYs9nF7^d={hbr3|F14V7U#b`s>MHRtY>B}6|@VACzF=&DN!vTLA1{=!pm;=3~w9mb&*6g`dj38WA$OsoP=b-`v6Q2deD=WR6M5$`HS`+liXwkZt65TrBo&2}ifXC-}45z?~+>KtE1_ zDcS10C%%@I%3;^#b4MF~`f|Qi_ZaLgd7SKl2V!?yyr!$7Qr#aswDjlN3jd26Hi)Wx zusmG;S3v#$7k#0bx<E;5HwH(WJxfN?3yZ5oAjyuoecfPx5-ePTtd!98k^81GhV zknQ4vnB3X4HsO?-ip6%%UCpLGI==$BclCMRJZ3i<2gw^xiIj3~2$ZJs$|OVoMQ5mf>?!#2P9 zXD&|&YpMjO_Dy;rw(_ELD7N`xTgd)cBEz^Vq!d2z$KcGKjMrQNdLrwEc71N$wDO!o z+ju_kZxC4|E8-0GK-=_;adA;U-8;7mS%;&t>Z}itJn)-k!L{iNw4_ZJamcdG$T>pN zSKTe-LH$L$@9+(5YXZb78dEp~wt*bhlx8yEkeOsLA7!C|)Ba2pGMur8G|y$5-u6{q z!l25*Ck_<|rVfwyvplugl9JrU2iXK0yA#g$NxTai$hLKqIbx#^{X=!<)~Ou zTqFZSn8YJUOF#`~TNcPx0nsQ(NE3Hsft~+7_Wm~%^Zz$g$M{EcNRa;t)ouTc+e!Zo n)&2g7+y5hS`+rq5=pC^z{vJHXTH-ml0EncR+`AI!C*S`8wxM(* literal 0 HcmV?d00001 diff --git a/doc/source/rllib-env.rst b/doc/source/rllib-env.rst index ea91c2e30..7aad86469 100644 --- a/doc/source/rllib-env.rst +++ b/doc/source/rllib-env.rst @@ -74,7 +74,7 @@ Performance There are two ways to scale experience collection with Gym environments: - 1. **Vectorization within a single process:** Though many envs can very achieve high frame rates per core, their throughput is limited in practice by policy evaluation between steps. For example, even small TensorFlow models incur a couple milliseconds of latency to evaluate. This can be worked around by creating multiple envs per process and batching policy evaluations across these envs. + 1. **Vectorization within a single process:** Though many envs can achieve high frame rates per core, their throughput is limited in practice by policy evaluation between steps. For example, even small TensorFlow models incur a couple milliseconds of latency to evaluate. This can be worked around by creating multiple envs per process and batching policy evaluations across these envs. You can configure ``{"num_envs_per_worker": M}`` to have RLlib create ``M`` concurrent environments per worker. RLlib auto-vectorizes Gym environments via `VectorEnv.wrap() `__. diff --git a/doc/source/rllib-training.rst b/doc/source/rllib-training.rst index 7d11bad07..9cd46ea44 100644 --- a/doc/source/rllib-training.rst +++ b/doc/source/rllib-training.rst @@ -66,12 +66,12 @@ Specifying Parameters Each algorithm has specific hyperparameters that can be set with ``--config``, in addition to a number of `common hyperparameters `__. See the `algorithms documentation `__ for more information. -In an example below, we train A2C by specifying 8 workers through the config flag. We also set ``"monitor": true`` to save episode videos to the result dir: +In an example below, we train A2C by specifying 8 workers through the config flag. .. code-block:: bash python ray/python/ray/rllib/train.py --env=PongDeterministic-v4 \ - --run=A2C --config '{"num_workers": 8, "monitor": true}' + --run=A2C --config '{"num_workers": 8}' .. image:: rllib-config.svg @@ -224,6 +224,79 @@ Sometimes, it is necessary to coordinate between pieces of code that live in dif Ray actors provide high levels of performance, so in more complex cases they can be used implement communication patterns such as parameter servers and allreduce. +Debugging +--------- + +Gym Monitor +~~~~~~~~~~~ + +The ``"monitor": true`` config can be used to save Gym episode videos to the result dir. For example: + +.. code-block:: bash + + python ray/python/ray/rllib/train.py --env=PongDeterministic-v4 \ + --run=A2C --config '{"num_workers": 2, "monitor": true}' + + # videos will be saved in the ~/ray_results/ dir, for example + openaigym.video.0.31401.video000000.meta.json + openaigym.video.0.31401.video000000.mp4 + openaigym.video.0.31403.video000000.meta.json + openaigym.video.0.31403.video000000.mp4 + +Log Verbosity +~~~~~~~~~~~~~ + +You can control the agent log level via the ``"log_level"`` flag. Valid values are "INFO" (default), "DEBUG", "WARN", and "ERROR". This can be used to increase or decrease the verbosity of internal logging. For example: + +.. code-block:: bash + + python ray/python/ray/rllib/train.py --env=PongDeterministic-v4 \ + --run=A2C --config '{"num_workers": 2, "log_level": "DEBUG"}' + +Callbacks and Custom Metrics +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can provide callback functions to be called at points during policy evaluation. These functions have access to an info dict containing state for the current `episode `__. Custom state can be stored for the `episode `__ in the ``info["episode"].user_data`` dict, and custom scalar metrics reported by saving values to the ``info["episode"].custom_metrics`` dict. These custom metrics will be averaged and reported as part of training results. The following example (full code `here `__) logs a custom metric from the environment: + +.. code-block:: python + + def on_episode_start(info): + print(info.keys()) # -> "env", 'episode" + episode = info["episode"] + print("episode {} started".format(episode.episode_id)) + episode.user_data["pole_angles"] = [] + + def on_episode_step(info): + episode = info["episode"] + pole_angle = abs(episode.last_observation_for()[2]) + episode.user_data["pole_angles"].append(pole_angle) + + def on_episode_end(info): + episode = info["episode"] + mean_pole_angle = np.mean(episode.user_data["pole_angles"]) + print("episode {} ended with length {} and pole angles {}".format( + episode.episode_id, episode.length, mean_pole_angle)) + episode.custom_metrics["mean_pole_angle"] = mean_pole_angle + + ray.init() + trials = tune.run_experiments({ + "test": { + "env": "CartPole-v0", + "run": "PG", + "config": { + "callbacks": { + "on_episode_start": tune.function(on_episode_start), + "on_episode_step": tune.function(on_episode_step), + "on_episode_end": tune.function(on_episode_end), + }, + }, + } + }) + +Custom metrics can be accessed and visualized like any other training result: + +.. image:: custom_metric.png + REST API -------- diff --git a/doc/source/rllib.rst b/doc/source/rllib.rst index 8caa21d9a..aa0f9004a 100644 --- a/doc/source/rllib.rst +++ b/doc/source/rllib.rst @@ -29,6 +29,7 @@ Training APIs * `Command-line `__ * `Configuration `__ * `Python API `__ +* `Debugging `__ * `REST API `__ Environments diff --git a/python/ray/rllib/agents/agent.py b/python/ray/rllib/agents/agent.py index ac7ef6dc8..6cf18c236 100644 --- a/python/ray/rllib/agents/agent.py +++ b/python/ray/rllib/agents/agent.py @@ -26,8 +26,18 @@ COMMON_CONFIG = { # === Debugging === # Whether to write episode stats and videos to the agent log dir "monitor": False, - # Set the RLlib log level for the agent process and its remote evaluators + # Set the ray.rllib.* log level for the agent process and its evaluators "log_level": "INFO", + # Callbacks that will be run during various phases of training. These all + # take a single "info" dict as an argument. For episode callbacks, custom + # metrics can be attached to the episode by updating the episode object's + # custom metrics dict (see examples/custom_metrics_and_callbacks.py). + "callbacks": { + "on_episode_start": None, # arg: {"env": .., "episode": ...} + "on_episode_step": None, # arg: {"env": .., "episode": ...} + "on_episode_end": None, # arg: {"env": .., "episode": ...} + "on_sample_end": None, # arg: {"samples": .., "evaluator": ...} + }, # === Policy === # Arguments to pass to model. See models/catalog.py for a full list of the @@ -184,7 +194,8 @@ class Agent(Trainable): policy_config=config, worker_index=worker_index, monitor_path=self.logdir if config["monitor"] else None, - log_level=config["log_level"]) + log_level=config["log_level"], + callbacks=config["callbacks"]) @classmethod def resource_help(cls, config): diff --git a/python/ray/rllib/evaluation/episode.py b/python/ray/rllib/evaluation/episode.py index ebd6ea784..754fd8d49 100644 --- a/python/ray/rllib/evaluation/episode.py +++ b/python/ray/rllib/evaluation/episode.py @@ -7,13 +7,15 @@ import random import numpy as np +from ray.rllib.env.async_vector_env import _DUMMY_AGENT_ID + class MultiAgentEpisode(object): """Tracks the current state of a (possibly multi-agent) episode. The APIs in this class should be considered experimental, but we should avoid changing things for the sake of changing them since users may - depend on them for advanced algorithms. + depend on them for custom metrics or advanced algorithms. Attributes: new_batch_builder (func): Create a new MultiAgentSampleBatchBuilder. @@ -23,6 +25,8 @@ class MultiAgentEpisode(object): length (int): Length of this episode. episode_id (int): Unique id identifying this trajectory. agent_rewards (dict): Summed rewards broken down by agent. + custom_metrics (dict): Dict where the you can add custom metrics. + user_data (dict): Dict that you can use for temporary storage. Use case 1: Model-based rollouts in multi-agent: A custom compute_actions() function in a policy graph can inspect the @@ -47,6 +51,8 @@ class MultiAgentEpisode(object): self.length = 0 self.episode_id = random.randrange(2e9) self.agent_rewards = defaultdict(float) + self.custom_metrics = {} + self.user_data = {} self._policies = policies self._policy_mapping_fn = policy_mapping_fn self._agent_to_policy = {} @@ -57,7 +63,7 @@ class MultiAgentEpisode(object): self._agent_to_prev_action = {} self._agent_reward_history = defaultdict(list) - def policy_for(self, agent_id): + def policy_for(self, agent_id=_DUMMY_AGENT_ID): """Returns the policy graph for the specified agent. If the agent is new, the policy mapping fn will be called to bind the @@ -68,12 +74,12 @@ class MultiAgentEpisode(object): self._agent_to_policy[agent_id] = self._policy_mapping_fn(agent_id) return self._agent_to_policy[agent_id] - def last_observation_for(self, agent_id): + def last_observation_for(self, agent_id=_DUMMY_AGENT_ID): """Returns the last observation for the specified agent.""" return self._agent_to_last_obs.get(agent_id) - def last_action_for(self, agent_id): + def last_action_for(self, agent_id=_DUMMY_AGENT_ID): """Returns the last action for the specified agent, or zeros.""" if agent_id in self._agent_to_last_action: @@ -83,7 +89,7 @@ class MultiAgentEpisode(object): flat = _flatten_action(policy.action_space.sample()) return np.zeros_like(flat) - def prev_action_for(self, agent_id): + def prev_action_for(self, agent_id=_DUMMY_AGENT_ID): """Returns the previous action for the specified agent.""" if agent_id in self._agent_to_prev_action: @@ -92,7 +98,7 @@ class MultiAgentEpisode(object): # We're at t=0, so return all zeros. return np.zeros_like(self.last_action_for(agent_id)) - def prev_reward_for(self, agent_id): + def prev_reward_for(self, agent_id=_DUMMY_AGENT_ID): """Returns the previous reward for the specified agent.""" history = self._agent_reward_history[agent_id] @@ -102,7 +108,7 @@ class MultiAgentEpisode(object): # We're at t=0, so there is no previous reward, just return zero. return 0.0 - def rnn_state_for(self, agent_id): + def rnn_state_for(self, agent_id=_DUMMY_AGENT_ID): """Returns the last RNN state for the specified agent.""" if agent_id not in self._agent_to_rnn_state: @@ -110,7 +116,7 @@ class MultiAgentEpisode(object): self._agent_to_rnn_state[agent_id] = policy.get_initial_state() return self._agent_to_rnn_state[agent_id] - def last_pi_info_for(self, agent_id): + def last_pi_info_for(self, agent_id=_DUMMY_AGENT_ID): """Returns the last info object for the specified agent.""" return self._agent_to_last_pi_info[agent_id] diff --git a/python/ray/rllib/evaluation/metrics.py b/python/ray/rllib/evaluation/metrics.py index e82e1f419..fadf2a5a2 100644 --- a/python/ray/rllib/evaluation/metrics.py +++ b/python/ray/rllib/evaluation/metrics.py @@ -59,9 +59,12 @@ def summarize_episodes(episodes, new_episodes, num_dropped): episode_rewards = [] episode_lengths = [] policy_rewards = collections.defaultdict(list) + custom_metrics = collections.defaultdict(list) for episode in episodes: episode_lengths.append(episode.episode_length) episode_rewards.append(episode.episode_reward) + for k, v in episode.custom_metrics.items(): + custom_metrics[k].append(v) for (_, policy_id), reward in episode.agent_rewards.items(): if policy_id != DEFAULT_POLICY_ID: policy_rewards[policy_id].append(reward) @@ -77,6 +80,9 @@ def summarize_episodes(episodes, new_episodes, num_dropped): for policy_id, rewards in policy_rewards.copy().items(): policy_rewards[policy_id] = np.mean(rewards) + for k, v_list in custom_metrics.items(): + custom_metrics[k] = np.mean(v_list) + return dict( episode_reward_max=max_reward, episode_reward_min=min_reward, @@ -84,4 +90,5 @@ def summarize_episodes(episodes, new_episodes, num_dropped): episode_len_mean=avg_length, episodes_this_iter=len(new_episodes), policy_reward_mean=dict(policy_rewards), + custom_metrics=dict(custom_metrics), num_metric_batches_dropped=num_dropped) diff --git a/python/ray/rllib/evaluation/policy_evaluator.py b/python/ray/rllib/evaluation/policy_evaluator.py index 4d72315e8..4120e6b4d 100644 --- a/python/ray/rllib/evaluation/policy_evaluator.py +++ b/python/ray/rllib/evaluation/policy_evaluator.py @@ -71,7 +71,7 @@ class PolicyEvaluator(EvaluatorInterface): ... policy_mapping_fn=lambda agent_id: ... random.choice(["car_policy1", "car_policy2"]) ... if agent_id.startswith("car_") else "traffic_light_policy") - >>> print(evaluator.sample().keys()) + >>> print(evaluator.sample()) MultiAgentBatch({ "car_policy1": SampleBatch(...), "car_policy2": SampleBatch(...), @@ -102,7 +102,8 @@ class PolicyEvaluator(EvaluatorInterface): policy_config=None, worker_index=0, monitor_path=None, - log_level=None): + log_level=None, + callbacks=None): """Initialize a policy evaluator. Arguments: @@ -162,6 +163,7 @@ class PolicyEvaluator(EvaluatorInterface): monitor_path (str): Write out episode stats and videos to this directory if specified. log_level (str): Set the root log level on creation. + callbacks (dict): Dict of custom debug callbacks. """ if log_level: @@ -170,6 +172,7 @@ class PolicyEvaluator(EvaluatorInterface): env_context = EnvContext(env_config or {}, worker_index) policy_config = policy_config or {} self.policy_config = policy_config + self.callbacks = callbacks or {} model_config = model_config or {} policy_mapping_fn = (policy_mapping_fn or (lambda agent_id: DEFAULT_POLICY_ID)) @@ -280,6 +283,7 @@ class PolicyEvaluator(EvaluatorInterface): self.filters, clip_rewards, unroll_length, + self.callbacks, horizon=episode_horizon, pack=pack_episodes, tf_sess=self.tf_sess) @@ -292,6 +296,7 @@ class PolicyEvaluator(EvaluatorInterface): self.filters, clip_rewards, unroll_length, + self.callbacks, horizon=episode_horizon, pack=pack_episodes, tf_sess=self.tf_sess) @@ -342,6 +347,12 @@ class PolicyEvaluator(EvaluatorInterface): batches.extend(self.sampler.get_extra_batches()) batch = batches[0].concat_samples(batches) + if self.callbacks.get("on_sample_end"): + self.callbacks["on_sample_end"]({ + "evaluator": self, + "samples": batch + }) + if self.compress_observations: if isinstance(batch, MultiAgentBatch): for data in batch.policy_batches.values(): diff --git a/python/ray/rllib/evaluation/sampler.py b/python/ray/rllib/evaluation/sampler.py index 503f52a12..3d9ee4b0f 100644 --- a/python/ray/rllib/evaluation/sampler.py +++ b/python/ray/rllib/evaluation/sampler.py @@ -20,7 +20,8 @@ from ray.rllib.utils.tf_run_builder import TFRunBuilder logger = logging.getLogger(__name__) RolloutMetrics = namedtuple( - "RolloutMetrics", ["episode_length", "episode_reward", "agent_rewards"]) + "RolloutMetrics", + ["episode_length", "episode_reward", "agent_rewards", "custom_metrics"]) PolicyEvalData = namedtuple( "PolicyEvalData", @@ -43,6 +44,7 @@ class SyncSampler(object): obs_filters, clip_rewards, unroll_length, + callbacks, horizon=None, pack=False, tf_sess=None): @@ -56,7 +58,7 @@ class SyncSampler(object): self.rollout_provider = _env_runner( self.async_vector_env, self.extra_batches.put, self.policies, self.policy_mapping_fn, self.unroll_length, self.horizon, - self._obs_filters, clip_rewards, pack, tf_sess) + self._obs_filters, clip_rewards, pack, callbacks, tf_sess) self.metrics_queue = queue.Queue() def get_data(self): @@ -99,6 +101,7 @@ class AsyncSampler(threading.Thread): obs_filters, clip_rewards, unroll_length, + callbacks, horizon=None, pack=False, tf_sess=None): @@ -119,6 +122,7 @@ class AsyncSampler(threading.Thread): self.daemon = True self.pack = pack self.tf_sess = tf_sess + self.callbacks = callbacks def run(self): try: @@ -131,7 +135,8 @@ class AsyncSampler(threading.Thread): rollout_provider = _env_runner( self.async_vector_env, self.extra_batches.put, self.policies, self.policy_mapping_fn, self.unroll_length, self.horizon, - self._obs_filters, self.clip_rewards, self.pack, self.tf_sess) + self._obs_filters, self.clip_rewards, self.pack, self.callbacks, + self.tf_sess) while True: # The timeout variable exists because apparently, if one worker # dies, the other workers won't die with it, unless the timeout is @@ -193,6 +198,7 @@ def _env_runner(async_vector_env, obs_filters, clip_rewards, pack, + callbacks, tf_sess=None): """This implements the common experience collection logic. @@ -211,6 +217,7 @@ def _env_runner(async_vector_env, clip_rewards (bool): Whether to clip rewards before postprocessing. pack (bool): Whether to pack multiple episodes into each batch. This guarantees batches will be exactly `unroll_length` in size. + callbacks (dict): User callbacks to run on episode events. tf_sess (Session|None): Optional tensorflow session to use for batching TF policy evaluations. @@ -239,8 +246,14 @@ def _env_runner(async_vector_env, return MultiAgentSampleBatchBuilder(policies, clip_rewards) def new_episode(): - return MultiAgentEpisode(policies, policy_mapping_fn, - get_batch_builder, extra_batch_callback) + episode = MultiAgentEpisode(policies, policy_mapping_fn, + get_batch_builder, extra_batch_callback) + if callbacks.get("on_episode_start"): + callbacks["on_episode_start"]({ + "env": async_vector_env, + "episode": episode + }) + return episode active_episodes = defaultdict(new_episode) @@ -270,10 +283,11 @@ def _env_runner(async_vector_env, atari_metrics = _fetch_atari_metrics(async_vector_env) if atari_metrics is not None: for m in atari_metrics: - yield m + yield m._replace(custom_metrics=episode.custom_metrics) else: yield RolloutMetrics(episode.length, episode.total_reward, - dict(episode.agent_rewards)) + dict(episode.agent_rewards), + episode.custom_metrics) else: all_done = False # At least send an empty dict if not done @@ -312,6 +326,13 @@ def _env_runner(async_vector_env, new_obs=filtered_obs, **episode.last_pi_info_for(agent_id)) + # Invoke the step callback after the step is logged to the episode + if callbacks.get("on_episode_step"): + callbacks["on_episode_step"]({ + "env": async_vector_env, + "episode": episode + }) + # Cut the batch if we're not packing multiple episodes into one, # or if we've exceeded the requested batch size. if episode.batch_builder.has_pending_data(): @@ -325,6 +346,11 @@ def _env_runner(async_vector_env, if all_done: # Handle episode termination batch_builder_pool.append(episode.batch_builder) + if callbacks.get("on_episode_end"): + callbacks["on_episode_end"]({ + "env": async_vector_env, + "episode": episode + }) del active_episodes[env_id] resetted_obs = async_vector_env.try_reset(env_id) if resetted_obs is None: @@ -429,7 +455,7 @@ def _fetch_atari_metrics(async_vector_env): if not monitor: return None for eps_rew, eps_len in monitor.next_episode_results(): - atari_out.append(RolloutMetrics(eps_len, eps_rew, {})) + atari_out.append(RolloutMetrics(eps_len, eps_rew, {}, {})) return atari_out diff --git a/python/ray/rllib/examples/custom_metrics_and_callbacks.py b/python/ray/rllib/examples/custom_metrics_and_callbacks.py new file mode 100644 index 000000000..eec7bffb5 --- /dev/null +++ b/python/ray/rllib/examples/custom_metrics_and_callbacks.py @@ -0,0 +1,66 @@ +"""Example of using RLlib's debug callbacks. + +Here we use callbacks to track the average CartPole pole angle magnitude as a +custom metric. +""" + +import argparse +import numpy as np + +import ray +from ray import tune + + +def on_episode_start(info): + episode = info["episode"] + print("episode {} started".format(episode.episode_id)) + episode.user_data["pole_angles"] = [] + + +def on_episode_step(info): + episode = info["episode"] + pole_angle = abs(episode.last_observation_for()[2]) + episode.user_data["pole_angles"].append(pole_angle) + + +def on_episode_end(info): + episode = info["episode"] + mean_pole_angle = np.mean(episode.user_data["pole_angles"]) + print("episode {} ended with length {} and pole angles {}".format( + episode.episode_id, episode.length, mean_pole_angle)) + episode.custom_metrics["mean_pole_angle"] = mean_pole_angle + + +def on_sample_end(info): + print("returned sample batch of size {}".format(info["samples"].count)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-iters", type=int, default=2000) + args = parser.parse_args() + + ray.init() + trials = tune.run_experiments({ + "test": { + "env": "CartPole-v0", + "run": "PG", + "stop": { + "training_iteration": args.num_iters, + }, + "config": { + "callbacks": { + "on_episode_start": tune.function(on_episode_start), + "on_episode_step": tune.function(on_episode_step), + "on_episode_end": tune.function(on_episode_end), + "on_sample_end": tune.function(on_sample_end), + }, + }, + } + }) + + # verify custom metrics for integration tests + custom_metrics = trials[0].last_result["custom_metrics"] + print(custom_metrics) + assert "mean_pole_angle" in custom_metrics + assert type(custom_metrics["mean_pole_angle"]) is float diff --git a/python/ray/rllib/test/test_policy_evaluator.py b/python/ray/rllib/test/test_policy_evaluator.py index 0e0d48c21..102a2a6c5 100644 --- a/python/ray/rllib/test/test_policy_evaluator.py +++ b/python/ray/rllib/test/test_policy_evaluator.py @@ -6,6 +6,7 @@ import gym import numpy as np import time import unittest +from collections import Counter import ray from ray.rllib.agents.pg import PGAgent @@ -150,6 +151,26 @@ class TestPolicyEvaluator(unittest.TestCase): result2 = agent.train() self.assertLess(result2["info"]["learner"]["cur_lr"], 0.0001) + def testCallbacks(self): + counts = Counter() + pg = PGAgent( + env="CartPole-v0", config={ + "num_workers": 0, + "sample_batch_size": 50, + "callbacks": { + "on_episode_start": lambda x: counts.update({"start": 1}), + "on_episode_step": lambda x: counts.update({"step": 1}), + "on_episode_end": lambda x: counts.update({"end": 1}), + "on_sample_end": lambda x: counts.update({"sample": 1}), + }, + }) + pg.train() + self.assertEqual(counts["sample"], 1) + self.assertGreater(counts["start"], 0) + self.assertGreater(counts["end"], 0) + self.assertGreater(counts["step"], 50) + self.assertLess(counts["step"], 100) + def testQueryEvaluators(self): register_env("test", lambda _: gym.make("CartPole-v0")) pg = PGAgent( diff --git a/test/jenkins_tests/run_multi_node_tests.sh b/test/jenkins_tests/run_multi_node_tests.sh index 398413691..380730fcb 100755 --- a/test/jenkins_tests/run_multi_node_tests.sh +++ b/test/jenkins_tests/run_multi_node_tests.sh @@ -320,6 +320,9 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ python /ray/python/ray/rllib/examples/cartpole_lstm.py --stop=200 --use-prev-action-reward +docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ + python /ray/python/ray/rllib/examples/custom_metrics_and_callbacks.py --num-iters=2 + docker run -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \ python /ray/python/ray/experimental/sgd/test_sgd.py --num-iters=2 \ --batch-size=1 --strategy=simple