From ca864faece2b4f7439e6ed7324a6ee692bdf71aa Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Thu, 3 Jan 2019 15:15:36 +0800 Subject: [PATCH] [rllib] Documentation for I/O API and multi-agent support / cleanup (#3650) --- doc/source/index.rst | 1 + doc/source/offline-q.png | Bin 0 -> 34438 bytes doc/source/rllib-env.rst | 4 + doc/source/rllib-models.rst | 2 +- doc/source/rllib-offline.rst | 123 ++++++++++++++++++ doc/source/rllib.rst | 6 + python/ray/rllib/agents/agent.py | 25 ++-- .../ray/rllib/evaluation/policy_evaluator.py | 9 +- python/ray/rllib/evaluation/sample_batch.py | 38 ++++++ .../examples/parametric_action_cartpole.py | 4 +- .../ray/rllib/examples/saving_experiences.py | 47 +++++++ python/ray/rllib/offline/input_reader.py | 5 +- python/ray/rllib/offline/io_context.py | 12 +- python/ray/rllib/offline/json_reader.py | 53 ++++++-- python/ray/rllib/offline/json_writer.py | 28 +++- python/ray/rllib/offline/mixed_input.py | 10 +- python/ray/rllib/test/run_regression_tests.py | 2 +- python/ray/rllib/test/test_io.py | 121 ++++++++++++++--- python/ray/rllib/utils/schedules.py | 2 +- 19 files changed, 431 insertions(+), 61 deletions(-) create mode 100644 doc/source/offline-q.png create mode 100644 doc/source/rllib-offline.rst create mode 100644 python/ray/rllib/examples/saving_experiences.py diff --git a/doc/source/index.rst b/doc/source/index.rst index 7f542c9d0..af9cb8440 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -94,6 +94,7 @@ Ray comes with libraries that accelerate deep learning and reinforcement learnin rllib-env.rst rllib-algorithms.rst rllib-models.rst + rllib-offline.rst rllib-dev.rst rllib-concepts.rst rllib-package-ref.rst diff --git a/doc/source/offline-q.png b/doc/source/offline-q.png new file mode 100644 index 0000000000000000000000000000000000000000..324288b416ef43635df743a478ce5968b7c3a85b GIT binary patch literal 34438 zcmb5WbzIfm(>D4Aq(wktQxcK_QWDaF0&cpbLpr6q(?H5icb5oAOM{4%bV;LhcgKDg z_x*dG_jAs9-p@Jb4+VCx*P59%*EQG7;)AlH3?2>z4g^7X@Ru)CA?OYY_yz=5JWqs|Pdnw_GDeuXk7Vr9_?TN* zFzq%}?(z)ik{}pRB(dm12qX&!l-*oW3I6>d4aWomkAWvHNW)S7eQ-nfI~Cf}>S||4 z$G!;7Trb9bDegy)9tjAXE_KJi;cyZXk`k&d7opW(Nkk+h)gJqQaSgN;ecaqmx<1jj zCoC;AUG6jp%(|%Mq;+(3;Dly>{rWYID2htZX+19?xo?HD`7B%Ce}6FRIe4zLv@{P* z=nrX5I##crhtZ^rD0=+$-p0CF%i~ilS0bZGl<_h{|IL!x&+4B)e`XDpbUa!c+B2Ym z{I2#p6B84|!^3Z`FU1eK=()MMOQ;&1mV1IR@$DunmhJs7r_syUnZ1*2Mq*VneU>ywOqh;5IJLgC%;I$d?FSi~iKf zN{g7qZ8u>l6+{1PKF{-B&X&6mgwzqauZlIgCk(4BKehjX0)Ryj?de_{MUT|-Um~jB zOxFO*oz8ie`6F*gKQ5Q~Ul+54EhXAFpEJ?Z(?5Lp_n?<^psz1ZZSC0WF{zxVXB`6^ z6QBA{z|@ouVvEma+|<;R?C#s7Cl=$Qqc0J$>FFHI!FnNAyx>9Ljh3Tfxj@eZBBfz* z;~!WR`dV5~O@7|Z8G35_`>T@pb?lZ)ou2!=ue*y&Rfloia=b3pJbvlEmWu&Etk2XdwOIACp3y* zA#%-kP2>cmZq)6+{lPdFnyz_cbMff0PL<{O7j5?$$9F@P;Zae@g4_HT?AkY z&MUvb+ns)E8v62?{~{~VNJ&pu6l=}Sei+=g2SN#sP&YKBfx%9E9IKAX8l&)RuE zZ#kLln$`-Xn)0o8#aq3aF$`8S-yD@hk4O0?^l~OPWg@BS?B|=~o8R4SpYdExlg@E$ z4L~VtJP1qG9?eSrfWoAh`px@vr>Ca}SX+RJY2_V_9}5RZy12jC`F-2DM(wf*ZxL!Z zg8$8wX}Q+ZNcw_#=K;|HabzO{oT@WT$Q1`V*`DsKuw`pO#|iyAWm7iW;*XS}@y%5& zYQETDgCp)e5HjVZm?$$eR94;_e2$-&pI^Z)*MGg&j$c;4!CEo9R{MHyp-ud1uO0l_ z)a2xgwVWtf#L~Wp`9S*hsyH$!-%5f&k3$oL`7Su0CY_PK7wZM29~t)oRTUeF_jDKB z8r0O(V4Mbe%F18ewsj(;oq;dl2)m}^@m`Jwc950UhH_wrKHQJWf&b6~8g`p67utf4 zkB>i+K2~LoQxe(#@rhoB+W*S2L_=3olNmmoV&9w;A7A6V+Mgp6rI;pk9ItC<<;4M) z_VT(Y)oe2SK>oco@kN#;c%P zGG1*nQ9o-3M5{vTeXM4mzgXZ5dw28?Cd^&LSbE;ew58)g|q)g)*sf(?Q1CIT8wwu;ps2) zLw)yI*Mp_*0kO+pw)`e{2_CXvno{=qee3bdA$wLfVgLKV2K7GYPH@D^K!*Qr(3lgM=1uV!D>BY}cM;tmmjONneov8#<;g$Zz7m|6m_Q4-f@S#M zoMRI)y+oMSPUBNOr8R7`9_tY(Ez-4bdR6L!V0gsHh;9;a zFUTyT@ka$sX~p~;{@2Of=W4h6x3&Fp<#KCZY?A;2qe*7Ye-eX07z#B&2&YzXR>!QX zX|YtD@87>)QCZo2vOZG4jj=}67mhh-$!kh4D0n`mX{cXorw=kZEfxfvq^>@CJYm@4 zx;YM#?&)-`y+88$w7bpF_jv3*A^q_1aI(Qy5@M+&d8UdtycDx`0uYsu%i50*M3nrt zw3kf=_HZ~hG&(xEHC4U27$t6ZqDuNPZ^)HG(C5s-+&o{Sn30pSN% z^)EhvCuL;66XA-un7!^SerLI2|TSTu-aS6DNd=P?~Cy zOT(;ULFKn^XW-Bwcj`AeplAj-LC6z}eqK@c>x*NMD;ut`)z#JI6S)E#)r-fR4jAAR zA|70x-G@GUdU_+_0^T&PLi&|7CTz)mvQ9zW3bCCKd4zOSCmN)pqSD~D?FE45;PCL; zPh4Syy%)j$#h}RxxD=#_ri+bY-HVT*p`n2aF@Uf%SxIjPnHjz{6IUiMYiXi+T{-_Q z9|sp#CW3;|Uu9o_PVehh@7FC%oa`-u`yNqOe&U^u3Ne3%EDoj0$I=-XI29WjEl;(jP4^R62rFA}a0KI_P9n-cdrvOG6aRUIxj=|sg}@Vo(zy8-8+OHP$~$*^2H33xdAs$$aU z^6_lv3C>?qnx_s6AVuCX9@h!|f+0&(W*pjASWENqEW}|a_mrO-1)9--Eo1GFy8pm@ z--7)BuUG8#BtF5M?#z9JHv3dsSvA?~&d3J>GY;*)%S}*dxf&Z*l5MX1kTFVPp4Z0{ zF`R|%f50>E^aP;QcaG9IpI)>tF?KqZ6j=TKYQzk1Y%Tj^mt1l_99s{p0_5k3(k*lf zZsQ*-eJQ|c9}3)e=h9MBZOgTinZ&plY5dNN0Hhpyi9+9I?GEIs0I(cyPB1ewx1VSH zLf2-GKRY}7``6g>#4tI3`OM%17$-L`509KDq;Yif*wlJyf3XuSfC27wpcU_0rw3>s z8YDrGd~KW36!oU=QJ_G1oKg{qu=@@bk7-w`pfePp4n<#6N3FJD(g1>>Q*Av4Rt5;w zin$UE$nby_EhjkrRFP?~h%z&ryH~pP?i!5&1E0Fl@7n8J!N=a-2=XU-jIx$5+ZhJnsRxxVb)^2T6G%7w@CX(x7!UAW65r1L!0dIn(HQ z7*76ViIfKVs>LpwcW|32omawjN~nRQ0RN0nNonvrT=Bm-%m9`J6zev>A1wFc;^K-U zuMPvzFyj*v0F8}{ivvuSe(lSg+w>cN{Bz#BqK@$YZ}L7haO!5v@Vj_)ZaU|+Spptt22>kx{fFmTQ z&-){rX6@{k)vP-JYDH0tUhMiK0n?d1TM(e@)Qq=de?^n!end;D5PY>*?t!7tg*P((*p@m}~*0FmiFfD*(vfKt;qT7k|sc z)BUacQw61SRCc-_56rTSSXYC0cme>J_gKV(Jiy*@aBu)kJ3E=SkJ0&~VQOw}2gu(@ zzC!yueE)-PX7IWQ5IbdtEjPdn*k*M2O$JC1tN@B49tfY@);MZ_5d!MPm*SxfC^P_p zH~hNh<||v1m7qX#UW^bxZJnyN(dU>rS|0&SEh8?DgphFPBqKN!7bXVkCQxN`eKl&o zxdaFR{@INlN93=UwcL1tg#-#Guxs=iHX1G?qn{!1_y-3cyjkW zJ^|F#h$?<_w%7qPyaLtBW<@WDjs17!&HuO(bG5vaItA1z;AEBprrL^apwfwB$FdK+ zZr6NnUeV{Mr=bxqqVgP5yV6|2&(H5S=d|Dl`-GsNpxd&jSj{nA@hlc_)HfpYAVp2C zC4$l`6;{}5G`aiKw{ol2bh#%k!jEX_T*>rsb)eHTPzW#nI&#Fl=cd{BGF&10T;5Dd zN-E$G)fFdnwBJwD)z$U#WLx*^>$aZ}l+UfQp0g;MnVIEK#{>8=<;T$0(U}@i5?4s( zQ&|NWLMUcr{Y!B%?o7_eIs$<}J!RMW8nEi=$wNLW0ZK&&Bkh_mU%m|WO9W$}^M@0| zVEANQx5W-MO_0#bhLqnmcC#0Nf^+J9Bk;#kCSzn^aB;mqi2iy~Cav4J+(l8w)I~z; zMCF@hwT@54uGrM=e0(&SvrItYU#P8hLb#E_p>Q!g<52P2sO$mH`#V*6 zTck9W?;DE%a_ngLPJwFvK}OyW<-`T0Fy-}#tCJBgAbF`svtO8 z7e$S~mgd4uL~pH__nTdcHzK`wO}ddrF*VR3Bb~yl^VW#HW-2j}>^~o25<~v4_%e%C zbhG-ReDUVaPv{>*W4Y2>hu(*@0(ZZO2XaIIzRDN(zn>MBl!;vZZ(rE`cv%Fs_Ff9_ zD80V>uLG6mgZ}Gn`X5{_RUQTFkkNew1k> z9uY&Wix>Y{M3%#Uf5c8`I37W6q&1GR>!2&4WZ@G))aB<&i;GRP`8hc`*NWp}W6!Sr zBBY6C)~Pz<5)#C91+M;G7+N1 zYI-B`%+Ig6aogdkR=lX|hWel;s0qVa!m{qJvT}0bo4wQk>52hvP$sa(_L`In7gE;L zOh6#+AFbc|*E(e7w@8A1o_b{YPuCyE-7}eb?;aD4Pa6lcVpHjHB`Y=S9?Ap1n{;_WJ-={kj9}+-;OYhaR(~s`o z*UP5NLJYm;k+Mau9JVg@_Oiuu`sdNMaM98E?IYevM^R5oVapVHKB$c?OV7^C`jAO~ zR72Kk%ilY#Q=(C%o-CiAS!T{lp}?dz7-eT&ovcTA@FpR$xpssX;b{78)nl<7Ub-p=VBK9dR4 zVyZ=6(>b`3`ly{CB=I}=X90N(8nUMkZ^$AYtvI&J`7r50rIkfjpl#uYJc~tH615f6| z)xMPHDm}eSieGhd%FpPDId0ED{jiB#{DwMIP#1B>B*`te|&tL|IH*n z+?EBwFq!bwPG47h)Y>cs?ZK8HYP>#43Nn?eY_9eSg zb@ru;$`N-YJt<4&Hkk@*;k;f>K#HFKaL(|l;NUN%5ezpXE7_T`#Su!2H~KA|fb)fr zK#CBEyDMri3YYHnB^(5U5vkV;(itjdtUmwcsT2$e0(1%D*oT74TXosIB0Tkzo3)I% zJ-B^5As~qqvf;s73Ww4W!%r4F4drX#)rOU>fxwKTL~uma!I*%o+(2WEHTt8`0S@2r zMn2b#r9bT>_*5c3$77nGM7`1?_qh-1#LATH2LuVn3RSr5=D*wqEGjI>@Cr~Krk&_j!=lJ@>_q<~wG_;^VF-gkc z9#vwEU%U4}%idOx=6qpEH6zo{mLn$(7Z+m4v`pvvubj4q&8V6ZXMc~E#u;IardT+>X`6k&_SiKb_s`e@P{|&EQf13I7A1_2Ip*{I~g> zQJxWZXiwarK+Q8J`yd~Hg5$ge*K zgX8ujV$qH2dtCcLTt&fD*bxLcvqmy$cjzL}HwgdbUG=ybv`~PAtDE01-~Rd*yJ>a# z26l=!$&Z|@R+hNOe{(3%N;Xl!zW+5Am2+QLW5fC`ezV^nvOkiln9z8bVKd%We}ey{ zwEoYeg=TaeDgce0hTT8wng+0L3OGs}rQ7(g!^@!@tYxguzxcUTQKa%)msMMu8j z-lTO6DJ=Z@DEN6za;YM{WE1&%I40CQH%DI9@B@l^to?ap9TlcJS-x1XJ>6p1u(Qos z>uXy%DPVU-2Hj!=TWR9iw!7hsKXO({q&a+dyB3o!L^;FBjSG@OX~lha$}jp_yT7zw zkx0__eo|!%i5Q)&QkE${R~^b%XO{%Bg>-My8RvnZR#EJIMqx$Y}a@+!w3RD2-%GHz*$^UPPatSsgk)+ZU1ij_4&@5fMpB%#MM>}~5Ekq~8% zv#A-_dze&38=q~+d!d%E?o~TH)URJ&NJX67rQueeo2zw4E^ErOv#}|?e4haGl~;tb zK7K?gU?;ztcR#DN^~3iRMo(R5zrwwG6xu%GI_tkjlUB2q+e#tITG3B4TP=>P8 z5vT@!E#hl;es8C=55{1)l_oHF!5jF<3|A<;=YGvgb7vq)@}`=qM}XP z{txIz8M^{r`#yI*O+|TL5w(N)6b1op@>jQhtWe0sonadMAZr4w1lYaZS)*c{nV7Vc z80#>-#82cuTZT(a0wyZwYo;HRmFQ%t7D2R(e07bD2qUsIMNG)kGqiS2sCMX)$!7P|q75doN7=odrvlH_cga&$oXg~0Y z8CDPEkaRFWkEOJ$y>xfd^JT{2lG82I;I-Rwj`+lhGEtrFhdwMuDgTsgBi6yT-e%h4 z>n?z{-SD*ACj-f~6dVh)3xi2FI(BK>t``hrxOXL0hAIEp8xWo_DV!pDO$ zp%@Ly0s$S@i!_qF)d!=>56Fs!YDD1-xDvUIm`0P%qG;$6(F~6OoH|l+5gz^C@TZR9{@#Pb-0ooFy)mH)-)FY)wAqOZAM+LCoT<#82N+N@BogDEx zQp4}ZGeA$Nc02eG3A{1 z#(IagHmBXpSHVY*_IEkWRn*;+Ca|)Uu@D}g=d=gPA%eWI&W$6c2sizkAv3{-uw}t# z4A`-h$owjBfG^2{1m!AOxtaSQ?x)SNuMttc;lH}Cj^*`)G);ChBfxvyBcaYZ?B zgBUc%R3NUehTkhv^^ujYQpS?;pA@`J56x7PDa3!Me~|fx=nlEMHRmq=B16_HU8pAF zB?Fhx%qE)KM>1NT;YRO)+6Xs}UN2+qaiRz}CV*9!c5W7VxLN%Rrb))NA7N5)2ny3M zV#uJkuUqUOB?gF}eBVXHn07`Gk^zVLm|j;+TmLl+&?eS~bm8$Ac*V8jH@UM!ka_OPj6l;lI$=$?vx7x$o8DTlc2i|F(%nAgTXR zU5G+`V{3*A(h{;8CfSDr^+G?5BjkdhwC^KYh~~m~T4--{O|F@Z(;W4F424mDy2~pM zBn!YaS3Kt4SkJY$@K6#?>M1ty>j4@bCkZdOea_j?&qJ}X_dh0x z@Om9}wJz%V!?N}^GLvqmZCqShSmN?k_Y}gEdbVI0U@y_>geQ!s01*qHoZU%3b@qx2 zQ)NMiv*&L<_<|2H1aOAo8a{P!TgfeA3al zF5BsLpXc9wGef&e76*>}@p-92dP|ltlCyE|^Ikmn1HEjcBj>Rs+3^GlqdC|1ZKIZN zcX97aKIrP^QH<<69i67)FF9#EMgeHr%*lEFZSRif2PGud@jbD0&g``I9Ak>@)jNSC zXU4TQ-(*Xy>{L0Y23~p0(2%AF%n2_r(~0e@Tf`r2!qA`?CVi)a7tOOfUBXE%UdOUw zss{XyL1PrYXM01Ai~Wj+vlk5j9kk<_CG*VDn~x1{Osk-=L2dEIxu<>v5NWQ3%a0B- zz(I)3?Zjm+1*Zna1b%IId+UcEQ^3ed|7rS&4GsFX(U=Ton@=A)E%Td^h6 zdw+h+j#@ol)zpmH7inZ69T!fP#u7`%4CpI3{KuY6)k3e&-aE@xZ%sEK57?OEM`X0Tc=wvUZZ_k1&k6PA4>^k7j~V5@EQcpcB{+-!6AK4_%I*Xnb?Ivp7q86}v@ zDk@Zq)ZbPJxtf{f0pUyt1?ZCqhR}*{l;(7??6DrR9O4$nTFkv?*FJ$MLo6xwM1|%) zV_%d;H)ez8+Yc|iltN@5xRA2yyFo1bPREV@z3U>-w@HErJpy&5r5xPc<;Ny_;wmAM zuNDvE>-C}6sx};9;n~Ep%$?2DT%9fOyk@%N>=X_Pem?HJoc0UqtZ#)3CZkvt4#Q{I zFdCuQkG?@FQB0T+Mx4;>{WP1h!-JbvI7S*fTCS0Du^jIUb&uHsOABcianCh}x{0Qu zy{AWk(*K(ZNPItw9bGy&kbP^^g)eDf@Ta7KyuU=nvkDl<})sXM_z?MCjzS<&N`uGnA-OypsLMjjBcJ(b8>- zBT~_%&2_*ICHAf0<3^4j-t>NP3}zpqn+wII4hFfnjfkQAm%;?M0rZmh1p|dbMEZ>y z{jXjg=BNb{@?Z`euey;riV!k#R9pqiQT#D%{Wa(fB}UvZ26mT+-LQS$*L?M zmV%E3>ztc3Udqs_qDNcuexiF`u#p!0i}XH*kCV_q-z8_15>6`dw^PCY%@C&d+5bm| zKn2yvUIBt@v@c;WqrIgDkx>#GzGCQI3|5SAI6%v#H;d*iUDCbZ|;?}shBmF26(zQ~glq~oU>Bd()EvOpp{UXq&2+0H)y&k|EfwCO3^W=8}iKQkla z&2891w^Y?KzBuL{@?yIL)Zw%Q{*}g-)7}NGdpM5%zisR&^Vhm zN;D1puIo2T@0BY#{9m=iM6Y`xICsJgOC3gZJhjjoMJ1Jc5a+JF)q2#)MirsNZ7FJZ00pp=%w>B7W1IllQppkyWDB^lKfG;0?1_qZs(t93}suIcT`C1cAVXirlx zR7-j+*Jf;ZwrH1-laRz1^X8E$K&?36mBuXYz=gI7y)A!qMBHfHitlw;$9zEys6`X7 z7uBkdTrMiwOffzw7C-+%24#(7A+-9=EYoxJb)W^Qp=sra1k3klrH!V4c6QUKD2s*w z4`}3~uaqO2Hn~d`TVpXWx9RVXCj|xqkV2M&0fU!!FeB53t*d48hD~`SzJ^QZfRlf* zu2!NE!*(YPit40?-#5>?w2+8?fxgIZ&G%u*LKsz0a6Eg6si;u3wl{AmQ9j=*(vx9z z^~!4%u(1C5`EeIxh;p%buYU1JI_9P^N&b7>KWtO!)8$eC^7|OF2|6i$Ug()hOFU8< z>^k;JENXgS%-L4769GNT%l+J#4mY&b+gtQcr4nYOJ0F}8=7uaA&JXQJD36@&FGeCN z?ZrKL{Pz(pD6K#ynX9!&h8NqNIV>^<`Uds(^j95mNa{zfJOy$B-@#pR{M||Jg?9Jx zRhqnOzu`lRKk2)N`u8fOr)|L)t45j_eSHQ=0p7F1RodB*Ds}u)`Q0|Sef)sCT_{t_ zJSSXL*IdRL$VpkL{ao_zHEG-HxpC7{(E)AX5ySL8iNy*Xeo=JaMf%_%vt%W--P^xZ z2n~89Ju+g#(X^QTXK+JAZ}tyI$Z+1%t#Ik*8sjk$ZpIm}M~S+FeYgzEW2YIoL9*oLsnS&2QqueKQb5 z3s+2ccA%oifu21gHjk^%kd_gqI)+-*6Z{P|>vBpFq#nR-qwWm%JZRsrKze(^;}3Se zXG#PIiGecqKF>$2B-l)Mx%f2gc~tQ2P^{ATQRF-8D>zCkd3D8a@`zt}ii z+>)W;^iQSo%C=$8d0^rN5^07S)}X)xMXR9Z^SZ>u$^sm4NVWo4BW{&K{U zH;W_r15_BR{zZ|-f=d-=nNGJaC65`YtMYJQ z{^QrQNjB$%aLz zLTa3oiY7BD-6-A5UxVE_@7l9#J0Ps;2Yu!I@s3V+=Lwo+td2<|#y^jswZb~kC+h@F zqs5@`nxFlOlp@}(1YdqV$07`5^EX>vv3<<(e$=woz?aC=y|CkAFu;gYkH4>QV)NBm z%VWveG$qgnlq;ebSeR~G5_<`Bkk&d0!i}e5>18I3iji?SZo#q-a-L|p3dv@PGz{9? z-AmdMSIxP8KHS=Xc`t>_(jg(Af^KxX>p@_cm8M2N2c#wZM&mVZmUiA-McYXktk57B zIyA@!*HCu%W!2mN${9ffm5I#NF1pRSgX$DbbRlWzH4-TrJrf zZe*ztQF=t|=w*_6(}?_lKWS|+(1-1FVHhu1H-hn(Hbf{XYPDUME8z!J97Rp*jKa5p z&hSy#f88)dSt(0ZwUGr2@!L!N+1#?Q+0ru)U~w->MS`%|(u=pvUf2}AyxR(qT)mk8 z7QMRZdzcwF-k97ArwQTn1kcZ;NGbojfAKvIU6H_PyBbaYlj(RF|H0YyW3xb_I|7|) z`TF)W;#knGQeFOJ`wJEImu>ze?M^Ua9I3$q`cZe@?Z0IOEIP!Eel0}__clAppkr32 z;}%(Xo>byvod@k)$9TIKV%IX(Rk%-hV7V3xBmTP%YlC#2({{C&KFwh4GqX%XM`vT- zLsEOZq@n_`8f;HdGrf&q07?-0tzh(;lvOLEeSid&!SZ%t}E;+_P6o-u|6d(tG%!k$fo_)1);^yN$b zqOk|bjgbM|(8m5#iLd3&x5AQ+Y@97GdgR4*-NvscGACQk<0xX+^rOtLJZ2$+oF|E+ z$q%RC`B*lLcWF@|&afe!2c7{O*=a!x9UFMiYZD)-s+f~Y@h)K;Xp*R4aTi;%DInqW zd3$W;oe!8H@i}We0t);GX*G`5e}ivEOyBUxPEO9AfT~}Vgc(aQvc#k@cLw!Y>Rh5_ zcyuwcmK0{Xyd`8>jN)@4Q&{%*PN2vUGsVNMFHp%l6sOy&pHH3D?jt}fxHv_63B7BtJDMcn3?UR1A&C{W(I4+z2zbe|T$ zn4zX-^Xtk<_uY$EbL)=Y4nIUdjtoqRe@Hi$(V(+(LBQ+*j|OZP4wRY1}U ztQPd}KwQOq0?hauG$aah2Oo_?bPf{;P0AJNEHQ%TUa#3#PNKOFS9ve@2;8F)v#yp# z9^W)DDb%a`6@U36LPwWw{mr-g5j>9(C@-0jH7iFSQerTrszs;+*zujdt6)M!ge&*m zebYxiQIkKWQEtDmbsa;?&L)b;=f4@^!N)7P_X?+>G#ur&FTYzRNMmqRszPbERliiaWnMm0B^$BgYAv;^Jmj{!f3=odQyfs_rfxc(8}zD)GK3)z~P-4A7F@KK24Q z!10t`TT|>A3>Ie^P9cGK>i50l-M!_Sxs7*kf_&p6%vc3!AYL4ZLiBn*n*XWyUzgNf z8oE2sm{=JtoT)lLZ#nRvDzmaO+r>g^J*(c@;(6uGuw%?iwug#eCV}?6Tq#FGKdn}v zNG&FE-P+a;X6fBBLk$W{d{(ENd<`<@20h|xhlic@Yb=vbF0aAe9c&>{&?P07vZ|IC ze^}10tEOhc%RgUT^tcN_k;RS^sEx&eyL@~PDm*$zp`B3C+;mimiYz{A#xm>M)Q_!QkkOv?i9G zc-!u&=Brm=9G7A3=zTWXU&KAuIM&!ZKx`p^jkmJjNeI1|l7+~IIZ+@SzZun5kBhwg z_Xf4_5_02mejzQVDOpxo6xx2%=0_%5DILtX+V;lQVKaxTvD0Q?#OA|$OD{V$#E8b9 z<5oYwOyL#nu0W^AXEO=-KPV7(d1Gad-vkBPs9yMpNf2zg3Eu%DjFa0Z%FUs-(oyTW zi!6fJh2^fOIIdf~ptdHF%%l7A=1=m#uxcLl#quI1KgntgnjQRs&vd{M{Q5qWIqBl& zNt)?B6WY~fcci6e2rfpA9>6q2gq@c51VDebcK*g;M4_eAFQCwMObnkB=gpR$?)LFs zwq>q*ME=HQ_BUE#0U7EoPy~9)%g+F!z&) zxrq!4xF0YN=FI!8BPjT+fBcNdVApcEbf3Sb4g3E7Q+tC@cOfiWkQ4$kFVxk7Z%6`* zjEb-ep*&+A;2n%Ze(*b)OaK$Z!hXN=hb${J zn~r;W%M9IT$4aU$s`C^Ex7~WDsXDQP2)-3G`G{q7E@Ib$O@#d|yR*!=6*~^F`d4@NM*8)W zl1$+Ab2%T87_g1?ch}ITs8#L2GNVRE43h;@sP8^Q`P+TeXJ1zI7OuVQ6y59;xwlaR z0GXEX`uROv-e+gm6bz8|jwn4&VEC|l4kxn6&u8-gwr=?Ud-7n4`wgz71Z^)4N~?fH z*w46mnSAZl8-gKxv56 zXsM~f&dpkVBVZu;tTcz|>ccDY0erdeC(VO53+`W*9~ z!Q8;dhXG7t!@02BUW|ekT?ZHKeh$x3`|0V;eU4|(NrS{hw{UP2llt;6cm0p#wasOY}9pZf;iwdUi7~m3}5rdl*4n51Xugs2* zdCiRYdPobV2pEb6%>Zah<uftDnR)KYRCv5Ae$(tGSlS?4^s*j~pHs?+eMG-)gV?POSaJA(2mqy-n?%@;1{i zrG_gDW8U$(fxPU0^9Ibki=O6-qdb$f9*$AsKn*b0z5c53HGz;}=qFdi_{Es2X%clF zyktm~6;Fy}q;%T7@2??! zj)X>q3lxkUE6&?$S-{coA+|0Y7wzAPP%k$_=#pTR7!fsn``Y#PLm&u7psb`uhv%ln# zEd*@Os9pZ{+lL%x9PM6itb`$gYmz}GNu}l0$;iE2>mh3eHhXx<5$haqEjx)&`zbg~ddw3ky3LY`vB(87se~D{%*J4l%I9D}PeF+IT`;52WFwhb^Fym;3n+ zbfD*^vC#15Zou(cVGP*hy6ftcO|K~L`e*M?f~=K>-)SouHSB4tZ^~9zZs%xFXqqb> z1|q>Y&C7_BtngnC68e2!lbu}cyfVMp>Ctbv(NuN_TRm6ewl%0kQ09n1q{8AMt3`bpSE+5CTMRf&gDzlG(`i&`eaoYy&-CcRQs|UoT`VZe9 z(Rx!B^LedTFjIS;b=#CB6|1xE)^93-zVgV&)9q>T#m|&tGiUJE;;j`GPaZdYyg-$> zf7`od*Le*6AP;9{d^J$cJK-V!l3QQN&!~$BRRH;QJGa2Eitkn-Nz1SeQLbftnOzo| zbkP;C@QIWS`Xxq*7Rx8}c13IW9w9%P{2nc^?)^H%#KquXkV-7Ioerm3eSfSAdg>;|w zr82R5Gr`x8N~L8a8Wb$6kbftSxJN6k`THiv;b1$OpS-n?0p_OKqaqY0TnOCw+{`HM zPgLJ1X-NrI12greyqzm`Tda9b;m3M;zQT4Nq5u3{0O6!fHMt_N55g4|%e76Uvsu!} z0N(~~>~2;2`N(%zxcdVs5Cg+_mPH_Nn}&oeXlts+;$( z9-t2#-5NoH0fz&<+5dvqB(l2E)$!6b7>`H~X<@}WwoIk{twockT;rRxH{MNzC#)@9+GT-G4e{dn1~UEpA%}=H z4`U!g`g?6GwdefAG|2tF`Sx%c1$6=q3?COsa|(+w`L&d+BsSff${b$B2vEFf`c4Lg zYUil5NXH2&7SPNn^QTf>0=Pc;G~b2X1iaBKDaHbyEQrJvE6ZF^5F?rQ6R6+AEGqySItb#g;#^^P&Wn@2G~15{RRtw7GxL5QG~?3p?4dlal;HY1m3 z+c$l|Zy)zgRyTbqRJp6X?)dzfaDYS!P%^MFO!6iUm8Dlmu)94AaAb5YQCibC4#s07 z>iIOhL~liote3k&oCBJdA`IzFu1IPL3G+P5qa$&^zW}EQItu4d6p$>W%(#x4 z^o@^~s=Xh4i6WAx2zzLPc9-?I^t02`mUjbVaJ2a4t&0=+PcbHFHV`Y{@#2uy9?sTUsr>*Pz|QQziHM26VQOYF;>yw5~}dvtmYa;&(3jkxBwd zQC!aS2o#{7J@V1VP?rsZp)+^>+`wB>QpQ0iC!l&NsN@u40Dd~E;3mMousHE*hVFGj~i zxd@My&ngWQV(u*cC46P@r`&4}|LiD=z?zx4Gbjv)>MrdQt171Iw{SB$&$g7&_Z_gr8 zaP;p%YxeIgU6wc1IAxzmA(};n@^GMG`W*b#rU?9Rok5XN`&}-k z`R3qxi$tDXm4VVoy7)f}gGm!>NpM;79t`x-=|+4sNn?Nhb9=k2q{n&c0DMFuBDxW@ z%&1Vpl{wq_zOZa8Uq{sMV!jh;)pEVSDN7;o7|ubtCEn*oNy&+-U~rg5oR7|P?9+TR z3rBPV+NWbUf?h{AFa`YSk?>8Jf2)RXVQIa$qQ`r%D!_|Au6yyQxW{T_nDYosKPgLe zGuxhTs%FU$xi&P9@OD{C6r^|j#2KP|>2ZgeLI@R*hVl%*1>4~(EXX{P2mF8-$V)4i0+i&+j@c3SH6G$#3j)}@+SpR6$g>cw#a(RHzT zcbX&3NTX;#c#^Y@wOCI-Ei~TcUBmw>?Ja=n3Yu=wjZ3f)2p%A~1%d{b;KAJ?H~|6# zcS1r0f_rdxx8NS!T>?Q5?yh&p|NV9UTd!`_d-bX)IOo)<&7RpaJ>9EU_X-20Lcrs2 zW;yCs_`Uxni%i7e5eCGGXQq_b*w8!+gY%0F;d|vP!UhqJzk>*$nI%Pp-Kd%oS)sq$ z%0=+Yd^!sUA3wh3)>zgH<25bVyxF@W06gM%!NFzpfMtpjL2*XLn7#^ zbB}>5Yt|qq8*Z_*AZj$25@zI!J#FyWB7_oWMP&RIJw8Z(jpMv=iXL(Vdn1dG zLeC`W)5ETPHc%EVq{t%gm~N?JjU31P@z`H z{$NQHQ`1$hIUz*9tQA^)9#gk^Tt8Ph;)mj}x3ELE{d(7>)jrmOyu4^vq4W7Z)oeLX zA{%{uwk~jZQ4UCundot*>8GyB`;y6_RZ;nA3>4!u@Y4K#`Dqg)=@anYXGMCRWR>%j z7VnG$=ke@Etq}ZvJWbvl8U&CPzXXz5Q*s{QePJ;&I@ zR?;E9QThRz=LDvU<+bJI=Ytk0fbCK#etp~8)g)@#I&f{R^?Se(f*MkbUAMOd)lZ)t z?K#ZGJA2)ioDn}HJUL9R7rE@G?IV$(t%CgRda0-Ir^EF)^vv4Q($Wt(0zPTL?jY!- zFEUV6#y8-}3!{1q!#v|wD(GfE29>(pKrAWhh~SUVBM)cF z=NqImw)2+yeH!sk;_gzT@*|Nq^Q0&&%%4ZoJajRP*ps%Cvqk_F4}Gmi1^|x1zf;No zs4?RI!x}vrBdDEqp3eIe2cFg<34=P6;Fkt=Y@f=b)@0y&8}?#D$w9D(4VhQ1F9b=> z^H5Wj2e1#TYg^`b{Rzl!iwH;^lF0ppH}-;cpC@`Kh~~D#=$Q*Y68Y5jr|Ly5~SG zB%XwJ(e8%uthBAIEoi?%Kk(~CEK?LBsL+#%;G=w$2~%uDjS{sB{`PHRa|J_EHjP?4 zRqpY}$Xix2p6?jJ`x@GoGT3#GSkN>9SU{FmTJf@p?{|TU3WD6z(O-)B_d%8xWGl`e zMvg$t{hbln1eD#VPWBolc<=O zBY~0;z0~jZH2>-tRq>pcUh9I41GP-L)m7Wq>eMMyQ8i_dn--K<RSdY8?VQB$>)Me{qWQfp4@ zZgdVcJgu#na@DO1j0tQD&e1g=7sRB0Bf!HWP+}927Lk_jrlyQEjnE$IpIK*a7B(RK z+9MQc`lbK)_TbR*!=U@3`?H}2iv^f>3XM+n>kwL~+uZGF?Yml9a<;#JD%&r4#wtuB zm%3Au6eh*Icdt~*Ls*dgxI-L16lxY72Dk-_FsO}T?)9~sr4TXbAm##<(X>C0I@^VRE+{N?ZLjjpxUo* zGvEuPk5&01WjPoHIRra`e6V@KI7#ECRA#3a`C39 zHg+xa$Cfy25TEvV?l_-lnx?gB(x4F=YWZq-rxn!^WaR5PRVCzhr+1sxU4U?k=h1{> z>o=ZmSaJV6a!qrLRX259kug4s&>?(jaPhv$YEF1@vC5OgDCA%%g#OKo7+VwnMEcL| zP=kOgEQh~lPVuW9RkWXlDeOf#zJh5dBKi*uWY8F*Go#r#!zU9WR})BqAZ_w`4D?vR zy?f&L8C%m$*=ZP#bqF1%>J`QYu?|Hy-kS&X$*|v>T2?AAe92Krk`3$bHd5Abxx7Ol zmq@CdIl4Qm309`-85MEW3|FII426kAqi$fx%+H*k+;B)aEv<{c!*=bu(tflzf4=mk z{hEIc@~1`+(b3T%7xsokJv}a72V4-R)X&1Q1Hv@&h;0Ys6649NpdClfnB8fQp8Aqq zN6u{P%FuyCyXM`6(~74?t*oX>?pK?z`;|TZo|irFO&+KD-OPWN_&u8fsafDC)nMlu zbxCB*EYcC$MGP`Sj042sX%p(d5;JNV^p-v-)U5B+-%Un$uDqN_6%CV)I~IDhs%VTc z>@dDXc?}_}S7zLlJv)cHk~a0Sx9~Kwm0_4c*D|~_8>cbAr4$`=(32*q{#L%a_RBoR zYFC zHi`V85YhZ1PW{%iC6(3|37S@`PK+W9`e{k_p&DvgyF_}q^#sKd0fR|gJKZfA{ZQ>2 z-Fx9=9L-dvX|Mtkg%hkp?DC7-W`rYfyoBzvrxq@*nS3!)T1l7MF$nrNkr==!bTp%; z?+k0ghBF%YoTqj3NHpxlW`jAo?{_~(Z%OL@JbT{01>Yyoq}&GgZ)V14ex&r*>vgXU zygTC1VhQK8F6ATegrrD};zW*YuP-0bN}5;E8d}6fg`2(f(lmB)95_5(Gn{20|GPsj zgTwmX>++2M{a|nJiHG;9+zoFVHn|X~kW2j^jCM5*jo-{~U%!4$f%A>7)#c5bH}oNb z*0Fs^(C+T;%kwZ1{GYtPNtV5Z43l-{9!KyGrua5DUan=2VTHi?c}fP?Pw5OFbF}5K zQ83CSzIwU7jv7JV?ctlLI?%FuIE@*nsu-S$N9FqM~l6;r0|-GsG0=-IUm;%ZDKMt&L8{~Jrk{8Lff z?ppt*#}eg(od_L2iN9SOd=5sBh_XQyz@>T}PcVZ%B^M_fW87Y!O4s-TiZdsBDGh%@ zJvZ!2&0bnL{BfNE39P@@AA9D_8oW#g$rEp>&|oWq`{4*BQ?AEb{4(%iuN*&1(W+B) zp-b$DY6KorXYy`}#;j4ReSD0lfQP|Ym0bWImhRmh4Ag@2@G-2GS@6fWz~TCZH9rV=6j zRm{}P<+c#%VR;twHu+4#m*a~Zl0unuZDa_xenEuFF*{LFg4eF-fHIWn$S`67zqKt|%Cc}fz$_r<(u z;Jc{}GL&peN;d1^hV_L76;C{A-!Zj_qCly`rr&GZ<765z{ha-cDM%qBrMxa}_6+W| zMuBr<^~>MAb`1h3P)%skNr%~^NXjcj(J;iepk*}0>sN2geJkFv`73Rohg~&w(mt{p z40_J;*^*K`z|9yp7q8cDR`>mMV{reS)MhZYeBxnCVy3k3M~ZHV&-8u%)C^Qlr_hvr zxNe^}V2hwfmM|qv7$W%L@w+|jmayKB_BlQ}esreivc~kNSnQ$U zlxK_7U0;xK3C~vt_7gNk{_k*ciElCIu!*zSUl&vm4bTuxVg#SX6?m@|`!3whyx?DB z0$W7aX@E!b&&4PIvQOz~{uJ%R8lb&LVTeL5zM?dw-JBz_BzSEFY4Li9EzU)R`7yeB znzVF%_t(hVV%CkztxQwSlRz`IKIzQ=uzy)J{(`CyIRTM1-6Sd1r!~m9Xz_L85+4dA zPJ62tXIKixC{f)8Xb&`!8i=A1Xas#Y&19kx?{1TO;%6GUADh((aZn1I<{mc0-)#rX%Soc#}C=gR%*SYl*kqjmK z=V8V$?oW9z{cnTDuC#hgeo>g*XIc(62hy1jE8r?1$n2-Tk7z`t>;!nenX0TZ4ieQh zfpRgy$FarC&r)8=JVumxA+G%n{R%F1IBMZHq7~&Ts*19DdxnvCT1d9!2f57q(4-^V zR}H?N-ofW>Y<|xp%_Lp+dY2D3&O<4rZw?}IcNSnm)!tCK&(+%%B4}fuR3=MF5}$t` zw?YRT)-O8um>qv^fKLI@Uh)l<%B*=<@!4vJADPkNy}ZB2Fwm5d7Le)?6qw+dpa^eHC7&YS+z# zsDWXfM9p~N)fDF`f{!d7@Y3XW$2@86v;!i7DH4;E-7r)1giy^Vtj`7CoW4B%Oq}vH z^xLahm?iZ>!Ri;MV3#jk7U4zqU{n~^SRN$LQQ7@UBTRotW7Or<1s1a6U?&o)Abc-J zgnrQf{)-CArdWu)YlFq%#di^$=#HQ3z76jg#W^^C6Y-HIoV&$n9F3S~ufC2$V5p`V z6o3DA#Z^-e+(p9)eWpaE%&t#OQc|%90hQZMP+h&-gD^%Szxu#_vZj^8%&C&ZK_tTHjuK2HJOR$#Zj>Fcqp*@Q zPjWx308qFdL%Y#TrL@SfVevi%cun)Q`|Yh^nW5C9$b>)mM8n6Jqh+IGVxAs%Jc24g zLx-H8=C5%Ho@RLg85ILx37oRk>;=Nw{RMv5xn`C>@Hoe}@whm;(9W<)4|D_2u=ml-S@ zW7e-WVT!*saej;_`sGI|y%yAMEEUkVxmJt$^_;aFANbK?KdvEm=*Diq0gPbyZ&`Jw zllNsTCft4h<1$wE->;XC;Ux}%H!b9}aS;7Rv!Qk^shdw!Uf!aMeD+Ox)gVG-VQwCP;qWC6O{u-n*|VaQO)`+M~8X&dTXx+@f@&HTvj zK3*B+P)E;7fhA&)2{H7lR84t<98_0+JJWfStHs(!u$i5L&w#<-tpUi+Bs?bso`Owx zpS-ir6N(*wxAr~TCF#sP1HV5sf*Ao}RD`eAoL}FQeO%6TjDM>f)5f~sJZ6ra#imRx z@xis43v}-jAOV3rJ z|Lq^e+&9We77Be75s6u|$9^ADg>nNa8FALX$`+Jlf%@g&rxN=hDz{xZx)krFvV6e(kb(D4`!R)FwzvglgkSLqPK_TFtP_WN%QZ;#iD z_M0jM@zA5A_3wt;h1D6OwMaN8D17*@j$5Jqq#;+ZqZUg&=L}h!1JMuvM5E@O)z$}< z{Nd>i5&832-5emPq*JY<)Kx395u;Mr-a=7F;t}BJ#icDLw8+*-v1>BQF$Yn2Qp(=R zkzSo7n0$ZdtO{HjI)7@C@OXNwpI8B63OO9>3j)tV4N5}ke$4H)Jg#40a3xqiSQ_7$Dlb{ghWZn_j2VK`M-lBAd=Scc z$7*7e7rWVA56UMPPHvxK3*T(p-p@OXVQU!M4y&zry=0|<)P5yqu1a^xyW-oqyQgFv zDt3hyvc%7tNkdeqiS(~FM3VVDI+Q7WEMBuvf2G2a8I;Vgw#M((?UH{{6(aqO`c+s2 zY9MTj9uW2z1Ik-0tG7;yIN!|wqKdsISSPSy)D3y*j*8p!8Ew&aznpp7{?+f0;6@R3 zdOQGdgwXxqXJKJ^1_w%fnENg-FR8^`)xPlRTA+qrg_G)TpF<)F9vZHPtp;DrIhf;h zf+aE<4_D5mcZMGw$Ml!31Dlm}1fnT}klVs6H0Wo%Q8H))s5YXQJ$O}<=#?(!?S1Q) zi{pshbBt%aM=1j=-KF0XR7Cmg_qGv@o@Yf`4#5*YkszKDgVWzJHL=Y4THg8VS+U7M zg_u88oIk4Il$#c7(En|q8V08Li)6j`Ony0;?)`^lX)BHD0i!Lw=c%8vdVDLQMk43d zjZ%NZK2o|OZeiq=VnJlg`bOoTa=!1PM;@&g>od2A3Po=O6(L0}fB(AyOGio57V;PO zRMPKo6-J&|L6FE74T8blF*C1H`Zp}331~60n%6^3R~>zh610711dlOO+Wb!SiM!TqqneHs$i(AW-_cEI2J%ZnWQQ(cvb--{56L!-d zHI!dBp%e^X==-Q8LbZ!F1rBT_Ij91kh2))j%CeWYF$gu1Zm;pO4wXk7#70WhBGFfNc1Q(=O-KwQp z9zE)Tb~oZ}sA8(a`!L=IqH;j?N%!JkaJC2=It;UZw!b77Rl!t_*@hR&;A>A+34JrP zbChgc5%2^_2X%&_87wa&E7qZj>&7Cv1AEWyuCrMWT2zR&Uw8C)u04MYG)-utBasM; zW6<@Im|14(J`BCP4WUc9@m^I2{X-H<5M{@Hw5IOAu^sM)4-ps&N+(p5=@Oy%dwvZF zJ2nIE$=HbShp3>fUt+~jkbZzmO6*(0?{DvBj8XiUq^6$-xch7+W*Pdpb?nZAWm4y$ zI>g=@nlwPpo6zi%@+jf5ym9T*?>gThVr{pkd`10XLxKai7I;Su@>AmJ92`^?IMIVX zD+6tp5x8Knwbtx&F-(4{|nD#8)oFtW7D^CaK4h5My~x!oKY2@ELC zdcfOp2J>MZ<$a=m;|xbmg-)fm)y?Gt<=XRlgAl_Yq{yE4b;PPMkxUG!Qz{MNtND?q zGocbimo}ZYNw>aypO7JT&O}}lo&9H12_qTkQ0*^!K( zD-{yXa#Rlr9MGrvP)*HLySlWLEg1((8E@sWNX}R%QNRb~f|elTugXkc#6T-ze#Ucz zmffsaNpw|=gNBYj$DLBFlSvo-a|+LsP&HQx^fWp56uYo_kZ*=6lk-M1Z{1s?JJ(NY z7@Ls&I;n!PgSV|k4j^HqUU0lzu46EE%xP5&81+sPwd^ALDaXi7?!PSH>Li-XT?bS1 z4-Z7d*}RD?;F26w*Iyb}}7_Dtuh)(Z1Th^EG?O{cljHK~eGij8@J4pNj450iT>b zJjkoWNG?Z_wG-MhK(0cTPM z)oc1Pr>cr8PlXToO4HK}|8NYFmE5r4|Ik{y~&gpjiT{ zgRdo&8{W+AN4u!j zc|Mc0@vZTf=uC|KV^2ZS$O7Tm4_kTKk7yY#*4cs zai-6~se|3+RO(cMiGp1T_XX7ekIczWz@|vWj3cgXP}m^Z6Cm3rYdjsKUN-ADz)>?< zV?~q^-H9ssl%)>he$_gKkpQ@Z*>vm4m?&BW5ybD08VzdRwzFm^2%V1GwRP1jmR+6k zYe*T$htH-0Y*8Fif^S&VLJ`b9ZnhZ3hbBomQs|P-z&cB}PUJ-cHxlT<^Kp_^-r%N{#=VDaoYW$lxVhT_IL-O` zJZ$D3_>3el7SSnRYJ9iT=z;sIDHMUBp(NKCs?eJpgOcr9VoSzxbW9QmKp{;^Qop#R zXRhruekE=a?>24Q?FKmt`-6}lT?T_ubQVS4A~nJ3j(UQYc*3sSf2GR)v08jj1}inWT7AHb!s^GiJh*N{b-Rcb|d`QN6bv zxKJ2GBiO61dWbXZU{2N{ zx7D5_kO+>c8&xlEUfOkh0cM>m;`4&_RRt9nO%PktYErKf|6mLl9dhhIqa)f%(PAyM zE!51bOp+zWrulv^%zi^LC_Eq zUs;=t4TdH}zPc7E(Y=0xU<9JL8>Z8(_{e5q_i@(G*}9Hr`o2uasFbMeJ67mQ4{(yO zLlwNpkadPBd1{|LAH2A48G|jLGVEuM0-^u8~wiZJ~_qFH&fg#ZAxS`?35_>}AT7{{ytn8)N&iVE0 zv|*s(V)C?q)9D@OPq)CGS4j#Lyme9Dc!l(QT0bwEpAySq&?O_i>O;Ig2C5^qv(Vq) z;TeW;FIhnz^~CT?nsOQnA0MBo_S4R3ss*J0IAuy69-j5}?qRCl4^@k=NlrD2Uc4sp z49*Y20^p2qH=`m76lHGi3L}#@A2lU!=`0%##)P0+pM#AP$xUz^pI2Acw#|ECGj9E-Y3YyUXpBWR{Xv)s;;WCDDx* zGQK1Z0E{4dwx2v2J_wL7sc_bm2HucEI!JBSal|~Yfe#zRsIgNA9hxtu*FL<&a0K!_ zah}#THknp+Q?8cM5s8$8B+GCjU_0eZY^wj{7zUACvzDvzeyqWVhmdGNmu(??f+wZe zb}s>XU!xmi7Td=)&FF+ObPU3b!+s%UBxx*sSy@@*_d!1Uw06IuH__M#WB8u{-~?k8Y8cYQ^td zNl9X#E0i<9zw)OKEN9L@?@1Nw9|))X-B$?p;r^iIY40+0lNTdFfhAa@nH8xHpH z^uzMJfG89#N{I#l;PUu0H|@q6uMR2~vKIhk!jhvcU!o*uLlTcDKgsGFvD?+8oE_w#u4}_^j(01g)-PA+{Kb#% zJjo{~leH`Dc`qINM0lDGMmd`2jH=8H^s2<%WeRf;3Tx&XsS;Yn7YKnJ zCBfRqX1%uq;pkg2)QF$c2tuL}qo7RkIGF{stm-G%V;;lHybDK!$PWOwr@qp~t=&PRB4`4MA|$DcAQy*< z+%0Q>U2)I#BwI*@PxqZ`8k`@6a^d1Z_Z4QSc+yTe@ka5VIgejI6WRRjN4L#slnc}( z)U%dozOSN>6{vxgU{}b9ZGlGBcaMfk1R(SkR283ORfecbWYzI3kUr&*TKO7mk!@~v zZ<_!1fEGWUd4|E)+HaXAZx)8-$%r-DCE775^>G9k?FNB!?Ke*#X0o^2CfMFGWhHW| z8|pHRHV;S62020YEiaD%_zrL?lLURWVv6iS|9mGE4GY4O#L4%5AsXQX;xakwSm6|q z6nOy36)vHQ&y=X}EvWF{b;+z1u!k4O9}5`ADIrK zQkrfiUrapo;}$Ul7#f4L`zvG!gON+S=lEzoNPUfjshRRMr+G6-U3`Iii$I~9qc|m1 zk}|_Uu6<4SkUcj8@FSCyV#^{MA`v%uB$$URPN~Nqe6Z!;-9Cf5CrAc}cdg}}eP5}M z;Dz^QcbjUPGBI=K2y0#rpkO*bTAVv-J%7KoBKL_v0)>o1=7e9Xj$xzb-D#~y#(JAx zyuz3F7#F0iEt<9=`61POq1NIibvkK;7Cn$(Rz_eEU;FaMV$&^COC40J=zQafwa)w_ zXOuJembM%JT}|KSQ#tP;TbbY9(PDWrmZL*KWR-ANpCi-AIBBFKWYa2%Ub#}6!MiBsy`HMgmcLU1CTJ6AudP05yqn^TtFa38@F&)Y0 zPej8q#iD3fyWHK2C-xmnEBU0+BGiVz=4wEe=t_z7glg{yy6z6fk~)OL`mcMM55|{! zl{5C|YmPs6IfDfEt@TWP@*7Ku!8_nfF_GDr_~&y<$Vk%4Z~`FJ+_DNf=Mpya|v!Sd}R6FrHA9!QyqiKaT9AB6J*o?Y{%~QDI6| zDlF|AMfU;iC`94o7e#Lal5>qs0Dt4Gr2`xSZ2=#U6>oY0JQ|$jt0&}eVl47Wmbto? zOq{vnjT(N>AD8{0e_t$g2YVs~p#Dh#A?Gs$$EQwB?mi3MQ@t_wM}I9+YB-~Jr|STW z2OyDsFA`Q6OgLehASu3CteP5)x;aG;Jt3jGVq*(nFaU0{Mio>AS65m(tZ@0@kDFof zkws9GJ5(h_#>2j);nHp?LZ!Lm1-}*aINrW-VXkN(4)b?CN>puXMb9z1It@hBR)*E}c;5ItuW?152|vDyFW>p52i zXRU?A;RUd{BfBV~Ur>HfxVcZZmzjDGKgJ;4w6sRO=u2Yh5+qxFHdgjJNb>0d_xq!P zl{$G-x#z~hmbsp3TO`6m2rUR%6?;sHMj9^sJPC2hr}OjW16*DAUN_xWN32YIvy<`( zn2>1`7L-U&%0H*3`ZzbsD;zUOF2>lgE*2J&v>1k=8WHY178kJ}8vm;OCjvl6Ca0zp z>+IhHTeT#st46Qp_;$3Seb*KO`2rNT1yK~8_*6qw+x+5Lke}<;xR~z0ROX;Dz!P@n zY*atrB?zH4{Fd^5Ucb$k&*3BpZ@dp5a=ST8^$Y$X|;>XM;G?`Hb#jfnF z{B|;12!y3ZD>qqSF9=Q{_YA*N7hU^_rdq;6TH`U~39ycK{@|qe_XBABc+QtkmnBB} z9K9I?%{hKnI3q{kAVKD4Fi9kIRe`Q=AXA!)vu|l_ZMJ#qbFS-5C!^&EVtZiZK=$SI zXx{T|Z0OnT)!eZL;<~s3g(ZTXNaj5$L>RSYvUlc*wL=-wV9G?E;qw?V&28}6L8PI0 zg^meFTtBhxukl!kqS_Mswy`&{bUJb%U5E95!HdtlPC*2u|I7Xkh^Eb_eOe2>F}}U# zUr@mm_xB>`12D0h$|Fc8OSV9byMlB7e6soi&hKf!lVTU1E=Z(miVXE3|8nS_UdPC; zZT^p=G+4TRqpP6+AX?y<>_qMR<})qk3%bw3D4CZ3$QX}U8Yb3chQf*Uat5duSey9M z5NYG{EY$9`7_fH1i6J8EMah;s&k%6-CNtAiNRqe= z4J{**SR}IN|BX3}ye}!++pSY|>8&{%OWN)O?X8&)Gk_X}t3u*gg;j-vShUD6@+&FR zNngmGy&ueT%~VplI2@ZVxjzMra_&4+h~f~pe|U>gZ;)U3kId6Y8hD}ifBKPd*O4T{ z6n6ep*mVY47fv!mzuZsrd8!-8tPq@<>P$D=wp6)T8&48d^5XdILMDDX?@S*84aW{D zg!o(Eakb`>tV3z31S-Of#(@C2nsvN{F$*n6>6Z7a%rR!#ACznqe7z>uctVq$D%)!d zHv3^L^1*PzwQ3Q`W@>qGB1LNbJ!OJI*GE3`EFVsPbJI?@j>ek8-WygkdNncpwKMSt zPi18CCAXcLZ#1z;l$F(tPSWp0Wkq*iHkW09l+Vx2<0yO+@D#s0N0++q$|NEjPp}_G zhM~#E~J1AVv?U8O|#?Ui76=r zX9UmbOe)~SwfT0>n^axj?Q35zooTR#)pum7DE9O!eBuPbDW^4-HUpibcf@V>KDkar zI1)rd6c1cudT!meve!CUjOz{8AfvBZaBeS0LWZH3q0jvK=}SOqAW4)dgG86pwi)b5 zp6Hn}RW7>hyIgO+Tz4|S1?OUM+WD+Ow^8I4x5%sQ`=ztVKNh?9`6ZOFF|OGvnye`z zc9>#f;n*q75I*Q-Y&x1Lvt%L@k*EN>I4sMF3!~1<&7~JO%fnghMU+S$g6~$QmjqCX zNBfM?-|rcQjRw~&wXl2nrq!v~ewuA}=03*gUN=Or{UgnISy}B;)izjj;V}q6zLTG@ zl(}FcDdA$vNhfB_qlwr(yN?#ceIrZu;z>8Tr#p|hx){9|;~gVe;tL*i4OFcwl$lB{ z%uG+m#l}t*&HP47;t%}{a|c!7?x7lZ37_s`}xml{3Uem;gWSZRkB( z%jxzDD5Yk8bwdm@a|h|{BHnzcT{<8x0s9ruLwGyr-Z#u?!nWbrls~mOeQW{wr4cn! z^3AOSHOTfZ5a1DF>`j+ZZ6)*JFnxsg%S-*4m>vJ3pwNdXBt-}zXUGYc{e$XD?sSj8 zqVM4zL!4peND@?;oioA^=O=nUJvdo}%&L#wpKRbIsOanPD3F);Ddiy3Dfd(NF!iuW z)JB3cSw?(Z8j(anJ)rv~xEN+cLZN&K=catq6|ZVHm2{Ju7Lv%ot__g9VitzZ13W}g zpfBUN%PAp-td33-f2&!`?{hjpAlQHR7;*FC{{zMLXwuPA8=jwUD`%d01NxUWuEmgK zTqSbb>RkK+;=Nf}SwOYuSjvKD(!iRjJa=61&2*oTh~0s}BIqfpriSOd(C(N>y}s#z z-9q!sva$HDhg)$t7GMiW5aS&+7~~OKm(!1pA}aNJ_v^dht;Gainc=Di+_o`!tO~nTH&nRn z+5!#G@6_K_vn6xpLiGgIz{wu{ZanwJb7S9yiJn13c({atr=izRyBv~`Hax!-%L1Jt zUSZ*8*xZqsjFT&B1S&)uP);rQ#@C0BQ=0K4X^`;;d}*1S;2Ku}7Y$LPVqv6Gz+D72 zCoF^dkh*Z44GC`^;9SsSYbw`NS{H%V0rJF| z8b|^kG=8Lb$z~$CxiwJ$Bolp<@k(^6-7c&8AfvfzG9ZVSti)d2SJ3QnvV=*JJahDW zE<KigO>2IZJHDr(}z07JXEm$$@u;3fZ%Pl4; z{4N?nKrjk=$~XKQQKpquIcCwW(9>~o@SsijJ>m0iKl^L40=s4t=hgU}P21{* z=3Cnp95h#D(TLCMTfs!LK~#=tFA>64DIa(xt0)K8w@W4=D_Su?Yt|PQ$_?$Jdt;9Q?a|wzyN$`j1-2aU}S;80(pS=`Q^m9PZM56s=wW=s_T ziN_s$&Hn3+mGiyk`!3)$HBy|Y8D?7$ie7Y#{JJ~1(>-r!cI%o1X+eOI0_-Nmz%>Yf zv{TW8>`ak@Em#jXvP>D?>=;K5iK;}3uOSv3*488n9ct|M5U0?2MMNR#@p`~B90pOK z`cveI_)pcb2}Ayw6kw1p!!t)0O9-nMSLqJGPxVuMVEa*Xwcfr5q4UqyZZyd;r;As3 z=w-~;K>AJM4+oY}cxN3ahpU4JH{{1}KkA+N22KBkrl+T;Kwcu~8#1*WT*R9=Bnf~s zlRY?f5qyY(5JMrJodz&l{V`02kpZ%tOY|erDEtXm^WxC47r^php-n{4O^B@h38Sf_ zJWo(J^W-owM6)@ zR}|9kK5|3<|340=?fZ~!Mkq~+C3P_kTGZX%{Ud~d4qs&t(vR%b>*@4YUQNK@#v!Fz z>BN{l5LK}MZ4^#<*!uc9(08AIee}sh7+noc9U{HHr6YPa!*(&5_5d7~nE1Y$LaWv( zGOyr$MkQ6GfgmCv2xN=help<&R#yJn;ruse0b(!v3vZKK%fN-9RP$9V0ff6$0% zXVk*XE?Iru*T7Ya_3A#{nkZH|jPR4_@eds5*; zZcUXaCtAuJym3Lb6w;=QxTsnR2IJppK%1+RR3LKMU${0D_ARV0SX+~_({k69`pQI@ zxnI9;VRye`SkAoYeU1RmD;Ct{kGCkzuBYd(({Ln^VqRz~yYKv>Kn0-y{Rq(?voRpfn%NM9Uh>25P|kd%rG& zSg{Vg=;yyz6y4TYYmy7<@JchT!q$YYwo^h0!;7Z2*N2)5%MNW~#lGw-isAqM!Em?6 zN-nEdLm2$U-4Vvg$q6S?dMy?CnbGO)xxSO)!`%M&eEyFqPkW>G`tt4ffOL3(=l%I` ztZz_ax7WZcd_AKGdemOL`#(d0pFe*Bh3F_*vsIbUDNU_cfr&5Uf0$R~Rb@rr^f=7F z#uoHg^cJpLjj;NZ*|y5?q`m7a*0ZLYKzDs~UdMHeBMaPoB^|zaQXr?~-2fZo-)|s; z=ot^@9ew#+c7A~%ZShw4r||t^+c{ktdQ!7A!5`KCIkW#`!Z#xDW>M&w33*}V@xHfn zzjh84zSDZg#&q87(jGqmilG%I{>-G^jsrclGf)pEVRZ`c+XR7_Yu@ zBT*-vBDMvdxx3oKd@52pgFgZf%k)bQjs*G)WENEN>;8^BCr6i^gTv=!-xWxNrF9no zm-zE65K`kn{$x-xRt6h_pw}E8pM;LMKjbW=qWCEaUEUlIHT4e+5ZSPpYTvc*-)pJ6 zFJBTQ16g-{bfP>pTyX3saNj#m?b#ZX?AOBX3YF-6j;DPGl6a)@)L!UR8Xc`hP)UW- zRQ#K{O;u?!uqMBwgL@re%p?X&QydB0t0jINb%2c@hsDnBF4^iUYSedCd`Isb9NyaQ z5`EB9)#vwcQhuu}>gf0g%?QPBYisspMo?hTqwT@Pbte}#4C{(DxR#FZEBCOf*t;WG zb4_I>X8u!9%>nv@3Ea1-@+DzGb$IMvj=1@yzV}luo>Y*jnc0K#8wz#|2ZpZtsqMv= z-2WHLQl%Sty4LYLQt>W1JKl)x;`jEwR;keoC@Pk`D6bgFFH1AVyVN8WJQf^Opa8m8 zZuW+qeH0{WfQl`DI{of!l~hCrgDTxnt<7XS2j-%eqc~pY)E(>;sI%hwvL>+j-dvQ` zEO?AP3B6Z{Srb)R>JB#nriDk~?oITT9)(n_7<@~yzNg*`McjP*6X`Wa-wRbXLW}@0 z^4$e~pLxP$nD5d3Gd{OcQV5Imm0R|p!p%~IM!=w9X3!qPzxftLda!I_`5zxF#J}C= zBoF`LF%9ZZbD!gc#TWV?ZYTOQZqI6bL8W=jn$)wIgRO%5H}BnUeHV3n2IMtabx&`O zeVSf31JL zKb+LPC#A-Jc0HgPJD@FKw->#-xLb4ki=Q%9-?@DxBU&aGUY>)5TcBbmpCpF4Y42mo zou+PqYRnqYFy6{0@%?4Fbfjx{dH(t<7G3kd^)&T=--(<=QzQ>gz=IzUElp`i@n5e& uKxGU4=cNqSgO~E}P1rd6|NfZePMDu44DX8sk`rjb4arC-h!=|)`Ts9_VQ|F& literal 0 HcmV?d00001 diff --git a/doc/source/rllib-env.rst b/doc/source/rllib-env.rst index 0409f3f0e..055ae18fe 100644 --- a/doc/source/rllib-env.rst +++ b/doc/source/rllib-env.rst @@ -271,6 +271,10 @@ The ``log_action`` API of ExternalEnv can be used to ingest data from offline lo Note that envs can read from different partitions of the logs based on the ``worker_index`` attribute of the `env context `__ passed into the environment constructor. +.. seealso:: + + `RLlib I/O `__ provides higher-level interfaces for working with offline experience datasets. + Batch Asynchronous ------------------ diff --git a/doc/source/rllib-models.rst b/doc/source/rllib-models.rst index 53b7c839b..5e3628aa3 100644 --- a/doc/source/rllib-models.rst +++ b/doc/source/rllib-models.rst @@ -325,7 +325,7 @@ Custom models can be used to work with environments where (1) the set of valid a return masked_logits, last_layer -Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_action_cartpole.py `__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN and several policy gradient algorithms. +Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_action_cartpole.py `__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``vf_share_layers=True``), and several other algorithms. Model-Based Rollouts diff --git a/doc/source/rllib-offline.rst b/doc/source/rllib-offline.rst new file mode 100644 index 000000000..00a6f08b7 --- /dev/null +++ b/doc/source/rllib-offline.rst @@ -0,0 +1,123 @@ +RLlib Offline Data Input / Output +================================= + +Working with Offline Datasets +----------------------------- + +RLlib's I/O APIs enable you to work with datasets of experiences read from offline storage (e.g., disk, cloud storage, streaming systems, HDFS). For example, you might want to read experiences saved from previous training runs, or gathered from policies deployed in `web applications `__. You can also log new agent experiences produced during online training for future use. + +RLlib represents trajectory sequences (i.e., ``(s, a, r, s', ...)`` tuples) with `SampleBatch `__ objects. Using a batch format enables efficient encoding and compression of experiences. During online training, RLlib uses `policy evaluation `__ actors to generate batches of experiences in parallel using the current policy. RLlib also uses this same batch format for reading and writing experiences to offline storage. + +Example: Training on previously saved experiences +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In this example, we will save batches of experiences generated during online training to disk, and then leverage this saved data to train a policy offline using DQN. First, we run a simple policy gradient algorithm for 100k steps with ``"output": "/tmp/cartpole-out"`` to tell RLlib to write simulation outputs to the ``/tmp/cartpole-out`` directory. + +.. code-block:: bash + + $ rllib train + --run=PG \ + --env=CartPole-v0 \ + --config='{"output": "/tmp/cartpole-out", "output_max_file_size": 5000000}' \ + --stop='{"timesteps_total": 100000}' + +The experiences will be saved in compressed JSON batch format: + +.. code-block:: text + + $ ls -l /tmp/cartpole-out + total 11636 + -rw-rw-r-- 1 eric eric 5022257 output-2019-01-01_15-58-57_worker-0_0.json + -rw-rw-r-- 1 eric eric 5002416 output-2019-01-01_15-59-22_worker-0_1.json + -rw-rw-r-- 1 eric eric 1881666 output-2019-01-01_15-59-47_worker-0_2.json + +Then, we can tell DQN to train using these previously generated experiences with ``"input": "/tmp/cartpole-out"``. We disable exploration since it has no effect on the input: + +.. code-block:: bash + + $ rllib train \ + --run=DQN \ + --env=CartPole-v0 \ + --config='{ + "input": "/tmp/cartpole-out", + "exploration_final_eps": 0, + "exploration_fraction": 0}' + +Since the input experiences are not from running simulations, RLlib cannot report the true policy performance during training. However, you can use ``tensorboard --logdir=~/ray_results`` to monitor training progress via other metrics such as estimated Q-value: + +.. image:: offline-q.png + +In offline input mode, no simulations are run, though you still need to specify the environment in order to define the action and observation spaces. If true simulation is also possible (i.e., your env supports ``step()``), you can also set ``"input_evaluation": "simulation"`` to tell RLlib to run background simulations to estimate current policy performance. The output of these simulations will not be used for learning. + +Example: Converting external experiences to batch format +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When the env does not support simulation (e.g., it is a web application), it is necessary to generate the ``*.json`` experience batch files outside of RLlib. This can be done by using the `JsonWriter `__ class to write out batches. +This `runnable example `__ shows how to generate and save experience batches for CartPole-v0 to disk: + +.. literalinclude:: ../../python/ray/rllib/examples/saving_experiences.py + :language: python + :start-after: __sphinx_doc_begin__ + :end-before: __sphinx_doc_end__ + +On-policy algorithms and experience postprocessing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +RLlib assumes that input batches are of `postprocessed experiences `__. This isn't typically critical for off-policy algorithms (e.g., DQN's `post-processing `__ is only needed if ``n_step > 1`` or ``worker_side_prioritization: True``). For off-policy algorithms, you can also safely set the ``postprocess_inputs: True`` config to auto-postprocess data. + +However, for on-policy algorithms like PPO, you'll need to pass in the extra values added during policy evaluation and postprocessing to ``batch_builder.add_values()``, e.g., ``logits``, ``vf_preds``, ``value_target``, and ``advantages`` for PPO. This is needed since the calculation of these values depends on the parameters of the *behaviour* policy, which RLlib does not have access to in the offline setting (in online training, these values are automatically added during policy evaluation). + +Note that for on-policy algorithms, you'll also have to throw away experiences generated by prior versions of the policy. This greatly reduces sample efficiency, which is typically undesirable for offline training, but can make sense for certain applications. + +Mixing simulation and offline data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +RLlib supports multiplexing inputs from multiple input sources, including simulation. For example, in the following example we read 40% of our experiences from ``/tmp/cartpole-out``, 30% from ``hdfs:/archive/cartpole``, and the last 30% is produced via policy evaluation. Input sources are multiplexed using `np.random.choice `__: + +.. code-block:: bash + + $ rllib train \ + --run=DQN \ + --env=CartPole-v0 \ + --config='{ + "input": { + "/tmp/cartpole-out": 0.4, + "hdfs:/archive/cartpole": 0.3, + "sampler": 0.3, + }, + "exploration_final_eps": 0, + "exploration_fraction": 0}' + +Scaling I/O throughput +~~~~~~~~~~~~~~~~~~~~~~ + +Similar to scaling online training, you can scale offline I/O throughput by increasing the number of RLlib workers via the ``num_workers`` config. Each worker accesses offline storage independently in parallel, for linear scaling of I/O throughput. Within each read worker, files are chosen in random order for reads, but file contents are read sequentially. + +Input API +--------- + +You can configure experience input for an agent using the following options: + +.. literalinclude:: ../../python/ray/rllib/agents/agent.py + :language: python + :start-after: __sphinx_doc_input_begin__ + :end-before: __sphinx_doc_input_end__ + +The interface for a custom input reader is as follows: + +.. autoclass:: ray.rllib.offline.InputReader + :members: + +Output API +---------- + +You can configure experience output for an agent using the following options: + +.. literalinclude:: ../../python/ray/rllib/agents/agent.py + :language: python + :start-after: __sphinx_doc_output_begin__ + :end-before: __sphinx_doc_output_end__ + +The interface for a custom output writer is as follows: + +.. autoclass:: ray.rllib.offline.OutputWriter + :members: diff --git a/doc/source/rllib.rst b/doc/source/rllib.rst index f78aafa47..2e4c249fe 100644 --- a/doc/source/rllib.rst +++ b/doc/source/rllib.rst @@ -83,6 +83,12 @@ Models and Preprocessors * `Variable-length / Parametric Action Spaces `__ * `Model-Based Rollouts `__ +Offline Data Input / Output +--------------------------- +* `Working with Offline Datasets `__ +* `Input API `__ +* `Output API `__ + RLlib Development ----------------- diff --git a/python/ray/rllib/agents/agent.py b/python/ray/rllib/agents/agent.py index 26981a4c9..cce3f449d 100644 --- a/python/ray/rllib/agents/agent.py +++ b/python/ray/rllib/agents/agent.py @@ -130,7 +130,8 @@ COMMON_CONFIG = { # Drop metric batches from unresponsive workers after this many seconds "collect_metrics_timeout": 180, - # === Offline Data Input / Output (Experimental) === + # === Offline Data Input / Output === + # __sphinx_doc_input_begin__ # Specify how to generate experiences: # - "sampler": generate experiences via online simulation (default) # - a local directory or file glob expression (e.g., "/tmp/*.json") @@ -146,9 +147,14 @@ COMMON_CONFIG = { # metrics will be NaN if using offline data. # - "simulation": run the environment in the background, but use # this data for evaluation only and not for learning. - # - "counterfactual": use counterfactual policy evaluation to estimate - # performance (this option is not implemented yet). "input_evaluation": None, + # Whether to run postprocess_trajectory() on the trajectory fragments from + # offline inputs. Note that postprocessing will be done using the *current* + # policy, not the *behaviour* policy, which is typically undesirable for + # on-policy algorithms. + "postprocess_inputs": False, + # __sphinx_doc_input_end__ + # __sphinx_doc_output_begin__ # Specify where experiences should be saved: # - None: don't save any experiences # - "logdir" to save to the agent log dir @@ -159,10 +165,7 @@ COMMON_CONFIG = { "output_compress_columns": ["obs", "new_obs"], # Max output file size before rolling over to a new file. "output_max_file_size": 64 * 1024 * 1024, - # Whether to run postprocess_trajectory() on the trajectory fragments from - # offline inputs. Whether this makes sense is algorithm-specific. - # TODO(ekl) implement this and multi-agent batch handling - # "postprocess_inputs": False, + # __sphinx_doc_output_end__ # === Multiagent === "multiagent": { @@ -503,9 +506,9 @@ class Agent(Trainable): elif config["input"] == "sampler": input_creator = (lambda ioctx: ioctx.default_sampler_input()) elif isinstance(config["input"], dict): - input_creator = (lambda ioctx: MixedInput(ioctx, config["input"])) + input_creator = (lambda ioctx: MixedInput(config["input"], ioctx)) else: - input_creator = (lambda ioctx: JsonReader(ioctx, config["input"])) + input_creator = (lambda ioctx: JsonReader(config["input"], ioctx)) if isinstance(config["output"], FunctionType): output_creator = config["output"] @@ -513,14 +516,14 @@ class Agent(Trainable): output_creator = (lambda ioctx: NoopOutput()) elif config["output"] == "logdir": output_creator = (lambda ioctx: JsonWriter( - ioctx, ioctx.log_dir, + ioctx, max_file_size=config["output_max_file_size"], compress_columns=config["output_compress_columns"])) else: output_creator = (lambda ioctx: JsonWriter( - ioctx, config["output"], + ioctx, max_file_size=config["output_max_file_size"], compress_columns=config["output_compress_columns"])) diff --git a/python/ray/rllib/evaluation/policy_evaluator.py b/python/ray/rllib/evaluation/policy_evaluator.py index f5c250aa9..aaaf47c45 100644 --- a/python/ray/rllib/evaluation/policy_evaluator.py +++ b/python/ray/rllib/evaluation/policy_evaluator.py @@ -187,8 +187,6 @@ class PolicyEvaluator(EvaluatorInterface): other metrics will be NaN. - "simulation": run the environment in the background, but use this data for evaluation only and never for learning. - - "counterfactual": use counterfactual policy evaluation to - estimate performance. output_creator (func): Function that returns an OutputWriter object for saving generated experiences. """ @@ -309,8 +307,6 @@ class PolicyEvaluator(EvaluatorInterface): "Requested 'simulation' input evaluation method: " "will discard all sampler outputs and keep only metrics.") sample_async = True - elif input_evaluation_method == "counterfactual": - raise NotImplementedError elif input_evaluation_method is None: pass else: @@ -388,6 +384,10 @@ class PolicyEvaluator(EvaluatorInterface): "samples": batch }) + # Always do writes prior to compression for consistency and to allow + # for better compression inside the writer. + self.output_writer.write(batch) + if self.compress_observations: if isinstance(batch, MultiAgentBatch): for data in batch.policy_batches.values(): @@ -397,7 +397,6 @@ class PolicyEvaluator(EvaluatorInterface): batch["obs"] = [pack(o) for o in batch["obs"]] batch["new_obs"] = [pack(o) for o in batch["new_obs"]] - self.output_writer.write(batch) return batch @ray.method(num_return_vals=2) diff --git a/python/ray/rllib/evaluation/sample_batch.py b/python/ray/rllib/evaluation/sample_batch.py index f576e4f14..a96ab0b1d 100644 --- a/python/ray/rllib/evaluation/sample_batch.py +++ b/python/ray/rllib/evaluation/sample_batch.py @@ -306,10 +306,48 @@ class SampleBatch(object): return out def shuffle(self): + """Shuffles the rows of this batch in-place.""" + permutation = np.random.permutation(self.count) for key, val in self.items(): self[key] = val[permutation] + def split_by_episode(self): + """Splits this batch's data by `eps_id`. + + Returns: + list of SampleBatch, one per distinct episode. + """ + + slices = [] + cur_eps_id = self.data["eps_id"][0] + offset = 0 + for i in range(self.count): + next_eps_id = self.data["eps_id"][i] + if next_eps_id != cur_eps_id: + slices.append(self.slice(offset, i)) + offset = i + cur_eps_id = next_eps_id + slices.append(self.slice(offset, self.count)) + for s in slices: + slen = len(set(s["eps_id"])) + assert slen == 1, (s, slen) + assert sum(s.count for s in slices) == self.count, (slices, self.count) + return slices + + def slice(self, start, end): + """Returns a slice of the row data of this batch. + + Arguments: + start (int): Starting index. + end (int): Ending index. + + Returns: + SampleBatch which has a slice of this batch's data. + """ + + return SampleBatch({k: v[start:end] for k, v in self.data.items()}) + def __getitem__(self, key): return self.data[key] diff --git a/python/ray/rllib/examples/parametric_action_cartpole.py b/python/ray/rllib/examples/parametric_action_cartpole.py index a1438f0a2..5af79c20e 100644 --- a/python/ray/rllib/examples/parametric_action_cartpole.py +++ b/python/ray/rllib/examples/parametric_action_cartpole.py @@ -175,10 +175,10 @@ if __name__ == "__main__": } elif args.run == "DQN": cfg = { - "hiddens": [], # don't postprocess the action scores + "hiddens": [], # important: don't postprocess the action scores } else: - cfg = {} + cfg = {} # PG, IMPALA, A2C, etc. run_experiments({ "parametric_cartpole": { "run": args.run, diff --git a/python/ray/rllib/examples/saving_experiences.py b/python/ray/rllib/examples/saving_experiences.py new file mode 100644 index 000000000..9964c127d --- /dev/null +++ b/python/ray/rllib/examples/saving_experiences.py @@ -0,0 +1,47 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +"""Simple example of writing experiences to a file using JsonWriter.""" + +# __sphinx_doc_begin__ +import gym +import numpy as np + +from ray.rllib.evaluation.sample_batch import SampleBatchBuilder +from ray.rllib.offline.json_writer import JsonWriter + +if __name__ == "__main__": + batch_builder = SampleBatchBuilder() # or MultiAgentSampleBatchBuilder + writer = JsonWriter("/tmp/demo-out") + + # You normally wouldn't want to manually create sample batches if a + # simulator is available, but let's do it anyways for example purposes: + env = gym.make("CartPole-v0") + + for eps_id in range(100): + obs = env.reset() + prev_action = np.zeros_like(env.action_space.sample()) + prev_reward = 0 + done = False + t = 0 + while not done: + action = env.action_space.sample() + new_obs, rew, done, info = env.step(action) + batch_builder.add_values( + t=t, + eps_id=eps_id, + agent_index=0, + obs=obs, + actions=action, + rewards=rew, + prev_actions=prev_action, + prev_rewards=prev_reward, + dones=done, + infos=info, + new_obs=new_obs) + obs = new_obs + prev_action = action + prev_reward = rew + t += 1 + writer.write(batch_builder.build_and_reset()) +# __sphinx_doc_end__ diff --git a/python/ray/rllib/offline/input_reader.py b/python/ray/rllib/offline/input_reader.py index 3ee7356a4..9af646427 100644 --- a/python/ray/rllib/offline/input_reader.py +++ b/python/ray/rllib/offline/input_reader.py @@ -9,8 +9,11 @@ class InputReader(object): """Input object for loading experiences in policy evaluation.""" def next(self): - """Return the next batch of experiences read.""" + """Return the next batch of experiences read. + Returns: + SampleBatch or MultiAgentBatch read. + """ raise NotImplementedError diff --git a/python/ray/rllib/offline/io_context.py b/python/ray/rllib/offline/io_context.py index 055bd714b..0e947c86a 100644 --- a/python/ray/rllib/offline/io_context.py +++ b/python/ray/rllib/offline/io_context.py @@ -2,6 +2,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os + from ray.rllib.offline.input_reader import SamplerInput @@ -18,9 +20,13 @@ class IOContext(object): evaluator (PolicyEvaluator): policy evaluator object reference. """ - def __init__(self, log_dir, config, worker_index, evaluator): - self.log_dir = log_dir - self.config = config + def __init__(self, + log_dir=None, + config=None, + worker_index=0, + evaluator=None): + self.log_dir = log_dir or os.getcwd() + self.config = config or {} self.worker_index = worker_index self.evaluator = evaluator diff --git a/python/ray/rllib/offline/json_reader.py b/python/ray/rllib/offline/json_reader.py index 61dadfc4c..01a52ed22 100644 --- a/python/ray/rllib/offline/json_reader.py +++ b/python/ray/rllib/offline/json_reader.py @@ -16,7 +16,9 @@ except ImportError: smart_open = None from ray.rllib.offline.input_reader import InputReader -from ray.rllib.evaluation.sample_batch import SampleBatch +from ray.rllib.offline.io_context import IOContext +from ray.rllib.evaluation.sample_batch import MultiAgentBatch, SampleBatch, \ + DEFAULT_POLICY_ID from ray.rllib.utils.annotations import override from ray.rllib.utils.compression import unpack_if_needed @@ -28,17 +30,17 @@ class JsonReader(InputReader): The input files will be read from in an random order.""" - def __init__(self, ioctx, inputs): + def __init__(self, inputs, ioctx=None): """Initialize a JsonReader. Arguments: - ioctx (IOContext): current IO context object. inputs (str|list): either a glob expression for files, e.g., "/tmp/**/*.json", or a list of single file paths or URIs, e.g., ["s3://bucket/file.json", "s3://bucket/file2.json"]. + ioctx (IOContext): current IO context object. """ - self.ioctx = ioctx + self.ioctx = ioctx or IOContext() if isinstance(inputs, six.string_types): if os.path.isdir(inputs): inputs = os.path.join(inputs, "*.json") @@ -74,7 +76,23 @@ class JsonReader(InputReader): raise ValueError( "Failed to read valid experience batch from file: {}".format( self.cur_file)) - return batch + return self._postprocess_if_needed(batch) + + def _postprocess_if_needed(self, batch): + if not self.ioctx.config.get("postprocess_inputs"): + return batch + + if isinstance(batch, SampleBatch): + out = [] + for sub_batch in batch.split_by_episode(): + out.append(self.ioctx.evaluator.policy_map[DEFAULT_POLICY_ID] + .postprocess_trajectory(sub_batch)) + return SampleBatch.concat_samples(out) + else: + # TODO(ekl) this is trickier since the alignments between agent + # trajectories in the episode are not available any more. + raise NotImplementedError( + "Postprocessing of multi-agent data not implemented yet.") def _try_parse(self, line): line = line.strip() @@ -121,6 +139,25 @@ def _from_json(batch): if isinstance(batch, bytes): # smart_open S3 doesn't respect "r" batch = batch.decode("utf-8") data = json.loads(batch) - for k, v in data.items(): - data[k] = [unpack_if_needed(x) for x in unpack_if_needed(v)] - return SampleBatch(data) + + if "type" in data: + data_type = data.pop("type") + else: + raise ValueError("JSON record missing 'type' field") + + if data_type == "SampleBatch": + for k, v in data.items(): + data[k] = unpack_if_needed(v) + return SampleBatch(data) + elif data_type == "MultiAgentBatch": + policy_batches = {} + for policy_id, policy_batch in data["policy_batches"].items(): + inner = {} + for k, v in policy_batch.items(): + inner[k] = unpack_if_needed(v) + policy_batches[policy_id] = SampleBatch(inner) + return MultiAgentBatch(policy_batches, data["count"]) + else: + raise ValueError( + "Type field must be one of ['SampleBatch', 'MultiAgentBatch']", + data_type) diff --git a/python/ray/rllib/offline/json_writer.py b/python/ray/rllib/offline/json_writer.py index 03401e9d1..d190d93e3 100644 --- a/python/ray/rllib/offline/json_writer.py +++ b/python/ray/rllib/offline/json_writer.py @@ -15,6 +15,8 @@ try: except ImportError: smart_open = None +from ray.rllib.evaluation.sample_batch import MultiAgentBatch +from ray.rllib.offline.io_context import IOContext from ray.rllib.offline.output_writer import OutputWriter from ray.rllib.utils.annotations import override from ray.rllib.utils.compression import pack @@ -26,21 +28,21 @@ class JsonWriter(OutputWriter): """Writer object that saves experiences in JSON file chunks.""" def __init__(self, - ioctx, path, + ioctx=None, max_file_size=64 * 1024 * 1024, compress_columns=frozenset(["obs", "new_obs"])): """Initialize a JsonWriter. Arguments: - ioctx (IOContext): current IO context object. path (str): a path/URI of the output directory to save files in. + ioctx (IOContext): current IO context object. max_file_size (int): max size of single files before rolling over. compress_columns (list): list of sample batch columns to compress. """ - self.ioctx = ioctx self.path = path + self.ioctx = ioctx or IOContext() self.max_file_size = max_file_size self.compress_columns = compress_columns if urlparse(path).scheme: @@ -102,7 +104,19 @@ def _to_jsonable(v, compress): def _to_json(batch, compress_columns): - return json.dumps({ - k: _to_jsonable(v, compress=k in compress_columns) - for k, v in batch.data.items() - }) + out = {} + if isinstance(batch, MultiAgentBatch): + out["type"] = "MultiAgentBatch" + out["count"] = batch.count + policy_batches = {} + for policy_id, sub_batch in batch.policy_batches.items(): + policy_batches[policy_id] = {} + for k, v in sub_batch.data.items(): + policy_batches[policy_id][k] = _to_jsonable( + v, compress=k in compress_columns) + out["policy_batches"] = policy_batches + else: + out["type"] = "SampleBatch" + for k, v in batch.data.items(): + out[k] = _to_jsonable(v, compress=k in compress_columns) + return json.dumps(out) diff --git a/python/ray/rllib/offline/mixed_input.py b/python/ray/rllib/offline/mixed_input.py index 9e9a53e69..e67946c9d 100644 --- a/python/ray/rllib/offline/mixed_input.py +++ b/python/ray/rllib/offline/mixed_input.py @@ -13,20 +13,20 @@ class MixedInput(InputReader): """Mixes input from a number of other input sources. Examples: - >>> MixedInput(ioctx, { + >>> MixedInput({ "sampler": 0.4, "/tmp/experiences/*.json": 0.4, "s3://bucket/expert.json": 0.2, - }) + }, ioctx) """ - def __init__(self, ioctx, dist): + def __init__(self, dist, ioctx): """Initialize a MixedInput. Arguments: - ioctx (IOContext): current IO context object. dist (dict): dict mapping JSONReader paths or "sampler" to probabilities. The probabilities must sum to 1.0. + ioctx (IOContext): current IO context object. """ if sum(dist.values()) != 1.0: raise ValueError("Values must sum to 1.0: {}".format(dist)) @@ -36,7 +36,7 @@ class MixedInput(InputReader): if k == "sampler": self.choices.append(ioctx.default_sampler_input()) else: - self.choices.append(JsonReader(ioctx, k)) + self.choices.append(JsonReader(k)) self.p.append(v) @override(InputReader) diff --git a/python/ray/rllib/test/run_regression_tests.py b/python/ray/rllib/test/run_regression_tests.py index a542924bd..51be4e4b1 100644 --- a/python/ray/rllib/test/run_regression_tests.py +++ b/python/ray/rllib/test/run_regression_tests.py @@ -21,7 +21,7 @@ if __name__ == '__main__': print(yaml.dump(experiments)) for i in range(3): - trials = run_experiments(experiments) + trials = run_experiments(experiments, resume=False) num_failures = 0 for t in trials: diff --git a/python/ray/rllib/test/test_io.py b/python/ray/rllib/test/test_io.py index e45550340..022960517 100644 --- a/python/ray/rllib/test/test_io.py +++ b/python/ray/rllib/test/test_io.py @@ -3,8 +3,11 @@ from __future__ import division from __future__ import print_function import glob +import gym +import json import numpy as np import os +import random import shutil import tempfile import time @@ -12,13 +15,17 @@ import unittest import ray from ray.rllib.agents.pg import PGAgent +from ray.rllib.agents.pg.pg_policy_graph import PGPolicyGraph from ray.rllib.evaluation import SampleBatch from ray.rllib.offline import IOContext, JsonWriter, JsonReader from ray.rllib.offline.json_writer import _to_json +from ray.rllib.test.test_multi_agent_env import MultiCartpole +from ray.tune.registry import register_env SAMPLES = SampleBatch({ - "actions": np.array([1, 2, 3]), - "obs": np.array([4, 5, 6]) + "actions": np.array([1, 2, 3, 4]), + "obs": np.array([4, 5, 6, 7]), + "eps_id": [1, 1, 2, 3], }) @@ -49,8 +56,7 @@ class AgentIOTest(unittest.TestCase): def testAgentOutputOk(self): self.writeOutputs(self.test_dir) self.assertEqual(len(os.listdir(self.test_dir)), 1) - ioctx = IOContext(self.test_dir, {}, 0, None) - reader = JsonReader(ioctx, self.test_dir + "/*.json") + reader = JsonReader(self.test_dir + "/*.json") reader.next() def testAgentOutputLogdir(self): @@ -69,6 +75,40 @@ class AgentIOTest(unittest.TestCase): self.assertEqual(result["timesteps_total"], 250) # read from input self.assertTrue(np.isnan(result["episode_reward_mean"])) + def testSplitByEpisode(self): + splits = SAMPLES.split_by_episode() + self.assertEqual(len(splits), 3) + self.assertEqual(splits[0].count, 2) + self.assertEqual(splits[1].count, 1) + self.assertEqual(splits[2].count, 1) + + def testAgentInputPostprocessingEnabled(self): + self.writeOutputs(self.test_dir) + + # Rewrite the files to drop advantages and value_targets for testing + for path in glob.glob(self.test_dir + "/*.json"): + out = [] + for line in open(path).readlines(): + data = json.loads(line) + del data["advantages"] + del data["value_targets"] + out.append(data) + with open(path, "w") as f: + for data in out: + f.write(json.dumps(data)) + + agent = PGAgent( + env="CartPole-v0", + config={ + "input": self.test_dir, + "input_evaluation": None, + "postprocess_inputs": True, # adds back 'advantages' + }) + + result = agent.train() + self.assertEqual(result["timesteps_total"], 250) # read from input + self.assertTrue(np.isnan(result["episode_reward_mean"])) + def testAgentInputEvalSim(self): self.writeOutputs(self.test_dir) agent = PGAgent( @@ -112,6 +152,58 @@ class AgentIOTest(unittest.TestCase): result = agent.train() self.assertTrue(not np.isnan(result["episode_reward_mean"])) + def testMultiAgent(self): + register_env("multi_cartpole", lambda _: MultiCartpole(10)) + single_env = gym.make("CartPole-v0") + + def gen_policy(): + obs_space = single_env.observation_space + act_space = single_env.action_space + return (PGPolicyGraph, obs_space, act_space, {}) + + pg = PGAgent( + env="multi_cartpole", + config={ + "num_workers": 0, + "output": self.test_dir, + "multiagent": { + "policy_graphs": { + "policy_1": gen_policy(), + "policy_2": gen_policy(), + }, + "policy_mapping_fn": ( + lambda agent_id: random.choice( + ["policy_1", "policy_2"])), + }, + }) + pg.train() + self.assertEqual(len(os.listdir(self.test_dir)), 1) + + pg.stop() + pg = PGAgent( + env="multi_cartpole", + config={ + "num_workers": 0, + "input": self.test_dir, + "input_evaluation": "simulation", + "train_batch_size": 2000, + "multiagent": { + "policy_graphs": { + "policy_1": gen_policy(), + "policy_2": gen_policy(), + }, + "policy_mapping_fn": ( + lambda agent_id: random.choice( + ["policy_1", "policy_2"])), + }, + }) + for _ in range(50): + result = pg.train() + if not np.isnan(result["episode_reward_mean"]): + return # simulation ok + time.sleep(0.1) + assert False, "did not see any simulation results" + class JsonIOTest(unittest.TestCase): def setUp(self): @@ -123,7 +215,7 @@ class JsonIOTest(unittest.TestCase): def testWriteSimple(self): ioctx = IOContext(self.test_dir, {}, 0, None) writer = JsonWriter( - ioctx, self.test_dir, max_file_size=1000, compress_columns=["obs"]) + self.test_dir, ioctx, max_file_size=1000, compress_columns=["obs"]) self.assertEqual(len(os.listdir(self.test_dir)), 0) writer.write(SAMPLES) writer.write(SAMPLES) @@ -132,8 +224,8 @@ class JsonIOTest(unittest.TestCase): def testWriteFileURI(self): ioctx = IOContext(self.test_dir, {}, 0, None) writer = JsonWriter( - ioctx, "file:" + self.test_dir, + ioctx, max_file_size=1000, compress_columns=["obs"]) self.assertEqual(len(os.listdir(self.test_dir)), 0) @@ -144,7 +236,7 @@ class JsonIOTest(unittest.TestCase): def testWritePaginate(self): ioctx = IOContext(self.test_dir, {}, 0, None) writer = JsonWriter( - ioctx, self.test_dir, max_file_size=5000, compress_columns=["obs"]) + self.test_dir, ioctx, max_file_size=5000, compress_columns=["obs"]) self.assertEqual(len(os.listdir(self.test_dir)), 0) for _ in range(100): writer.write(SAMPLES) @@ -153,10 +245,10 @@ class JsonIOTest(unittest.TestCase): def testReadWrite(self): ioctx = IOContext(self.test_dir, {}, 0, None) writer = JsonWriter( - ioctx, self.test_dir, max_file_size=5000, compress_columns=["obs"]) + self.test_dir, ioctx, max_file_size=5000, compress_columns=["obs"]) for i in range(100): writer.write(make_sample_batch(i)) - reader = JsonReader(ioctx, self.test_dir + "/*.json") + reader = JsonReader(self.test_dir + "/*.json") seen_a = set() seen_o = set() for i in range(1000): @@ -169,7 +261,6 @@ class JsonIOTest(unittest.TestCase): self.assertLess(len(seen_o), 101) def testSkipsOverEmptyLinesAndFiles(self): - ioctx = IOContext(self.test_dir, {}, 0, None) open(self.test_dir + "/empty", "w").close() with open(self.test_dir + "/f1", "w") as f: f.write("\n") @@ -178,7 +269,7 @@ class JsonIOTest(unittest.TestCase): with open(self.test_dir + "/f2", "w") as f: f.write(_to_json(make_sample_batch(1), [])) f.write("\n") - reader = JsonReader(ioctx, [ + reader = JsonReader([ self.test_dir + "/empty", self.test_dir + "/f1", "file:" + self.test_dir + "/f2", @@ -190,7 +281,6 @@ class JsonIOTest(unittest.TestCase): self.assertEqual(len(seen_a), 2) def testSkipsOverCorruptedLines(self): - ioctx = IOContext(self.test_dir, {}, 0, None) with open(self.test_dir + "/f1", "w") as f: f.write(_to_json(make_sample_batch(0), [])) f.write("\n") @@ -201,7 +291,7 @@ class JsonIOTest(unittest.TestCase): f.write(_to_json(make_sample_batch(3), [])) f.write("\n") f.write("{..corrupted_json_record") - reader = JsonReader(ioctx, [ + reader = JsonReader([ self.test_dir + "/f1", ]) seen_a = set() @@ -211,9 +301,8 @@ class JsonIOTest(unittest.TestCase): self.assertEqual(len(seen_a), 4) def testAbortOnAllEmptyInputs(self): - ioctx = IOContext(self.test_dir, {}, 0, None) open(self.test_dir + "/empty", "w").close() - reader = JsonReader(ioctx, [ + reader = JsonReader([ self.test_dir + "/empty", ]) self.assertRaises(ValueError, lambda: reader.next()) @@ -223,7 +312,7 @@ class JsonIOTest(unittest.TestCase): with open(self.test_dir + "/empty2", "w") as f: for _ in range(100): f.write("\n") - reader = JsonReader(ioctx, [ + reader = JsonReader([ self.test_dir + "/empty1", self.test_dir + "/empty2", ]) diff --git a/python/ray/rllib/utils/schedules.py b/python/ray/rllib/utils/schedules.py index 41518e6b9..12efda661 100644 --- a/python/ray/rllib/utils/schedules.py +++ b/python/ray/rllib/utils/schedules.py @@ -104,5 +104,5 @@ class LinearSchedule(object): def value(self, t): """See Schedule.value""" - fraction = min(float(t) / self.schedule_timesteps, 1.0) + fraction = min(float(t) / max(1, self.schedule_timesteps), 1.0) return self.initial_p + fraction * (self.final_p - self.initial_p)