From be26a7b1b05564ec6fe33b08d26e2345611e4f8c Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Sat, 6 Jun 2020 03:22:19 -0700 Subject: [PATCH] [rllib] Support for complex / variable-length observation spaces (#8393) --- doc/source/rllib-models.rst | 15 ++ doc/source/rllib-toc.rst | 1 + doc/source/struct-tensor.png | Bin 0 -> 43407 bytes rllib/BUILD | 24 +++ rllib/examples/complex_struct_space.py | 47 +++++ rllib/examples/env/simple_rpg.py | 48 +++++ rllib/examples/models/simple_rpg_model.py | 69 +++++++ rllib/models/modelv2.py | 54 ++++-- rllib/models/preprocessors.py | 45 ++++- rllib/models/repeated_values.py | 171 ++++++++++++++++++ rllib/tests/test_nested_observation_spaces.py | 98 +++++++++- rllib/utils/spaces/repeated.py | 37 ++++ rllib/utils/spaces/simplex.py | 5 +- 13 files changed, 599 insertions(+), 15 deletions(-) create mode 100644 doc/source/struct-tensor.png create mode 100644 rllib/examples/complex_struct_space.py create mode 100644 rllib/examples/env/simple_rpg.py create mode 100644 rllib/examples/models/simple_rpg_model.py create mode 100644 rllib/models/repeated_values.py create mode 100644 rllib/utils/spaces/repeated.py diff --git a/doc/source/rllib-models.rst b/doc/source/rllib-models.rst index d0b8e19fa..1669bf1cb 100644 --- a/doc/source/rllib-models.rst +++ b/doc/source/rllib-models.rst @@ -220,6 +220,21 @@ Self-Supervised Model Losses You can also use the ``custom_loss()`` API to add in self-supervised losses such as VAE reconstruction loss and L2-regularization. +Variable-length / Complex Observation Spaces +-------------------------------------------- + +RLlib supports complex and variable-length observation spaces, including ``gym.spaces.Tuple``, ``gym.spaces.Dict``, and ``rllib.utils.spaces.Repeated``. The handling of these spaces is transparent to the user. RLlib internally will insert preprocessors to insert padding for repeated elements, flatten complex observations into a fixed-size vector during transit, and unpack the vector into the structured tensor before sending it to the model. The flattened observation is available to the model as ``input_dict["obs_flat"]``, and the unpacked observation as ``input_dict["obs"]``. + +To enable batching of struct observations, RLlib unpacks them in a `StructTensor-like format `__. In summary, repeated fields are "pushed down" and become the outer dimensions of tensor batches, as illustrated in this figure from the StructTensor RFC. + +.. image:: struct-tensor.png + +For further information about complex observation spaces, see: + * A custom environment and model that uses `repeated struct fields `__. + * The pydoc of the `Repeated space `__. + * The pydoc of the batched `repeated values tensor `__. + * The `unit tests `__ for Tuple and Dict spaces. + Variable-length / Parametric Action Spaces ------------------------------------------ diff --git a/doc/source/rllib-toc.rst b/doc/source/rllib-toc.rst index efacb3ccd..0f69f62a3 100644 --- a/doc/source/rllib-toc.rst +++ b/doc/source/rllib-toc.rst @@ -79,6 +79,7 @@ Models, Preprocessors, and Action Distributions * `Custom Action Distributions `__ * `Supervised Model Losses `__ * `Self-Supervised Model Losses `__ +* `Variable-length / Complex Observation Spaces `__ * `Variable-length / Parametric Action Spaces `__ * `Autoregressive Action Distributions `__ diff --git a/doc/source/struct-tensor.png b/doc/source/struct-tensor.png new file mode 100644 index 0000000000000000000000000000000000000000..52c6a6346f804136d10dbfa664848940ed3ded28 GIT binary patch literal 43407 zcmce-Wm{ZPlQj&D)400^cXyW#7BsjM+=IJIZ~_GPAPEFQaBD2MySp^*u21Kld1vMq zyr23y=i0rguBu(NYOT!|wGSY4R1#De7#MVg_p%x=FaRGI7}$@!u*XJ7-db{mVoFL>D1~=x71tPmDFg&LJiNdp8UigW^-k&EVaVmC2y(DeH4}JP z*jRXKct9E|n)W+moZGjtPYhrO&)O5mj;nydQ?S6da{cjW8%Fq$5(Wcy-}2t^ ztsW*A2}TR{yaG4tG0Pi<90rTlq{bGm05SLgX6yhFaDKp^aQerbU(9Q{zPkT{wsd{D zZ}PRRwL_>64(1urG_92|Q#bJh6H$${lMAMYyO7-ei04Q=UqQ_4!z|JSPA5`>=W4V3 zp&+Or&e7quCL6Tsc=yB&NiP*i8Re&JPGEcQ)z={*h$#Zs^vNduh$C_FPQgv1t3mv; z%I=C0tZ9X~?InZjX3Gmta6SLhPBlkxBJ44DT|ZyUi>cVy*{2?%Z5?qDZw^Pq^M=@` z&Q*``X-IN%en9|3(<`%Kowu7ya9r4@bz>^Zm|Mr}3z5WM`LBPc48#X<`-4RzH8z1T z&L09~+^}7>?teqzjx?7QtZ%%t0t~HjDF|Deel$0Y0wnzzO&^(eYQ)6WdHj+@4vRVZ zS9+5}kI7-KL&cWqIhPwp)A85~SR0$0oE>MCUpBnkfCZGn+N?li!Tb<6fbSrSmJd(jym}!(XbU*OZrIF%H&>NKqyjBH+gOQ1`*8bAOZSqNyQt z2Z_M?*Kp((fi!H z1-Ax^K1^oz1p)IhzI16C$eykR9d3uF?%bWNUiP0)Xu*Dt_ONG0Mosdr;>vKH;MHHM z3jv4L2>q1;!NQUCqg6jO#+qtRlS{u@F7W67Qa93v{SDYKmHTY=wlJpI&&IrwB=B-Mwmaf@QhsG$3Uo^z zs%(-GsXb~JDm$Y~C?Sq;zSMEFkO|>PlB*7?TH1xXW(<~az|`RJ{+`tTw6^NkAIJa0 z$7eOTvHD0nB1WYl@EP8;`sC>yPliVdVtwb|3lxT%FC1q9Ua`i0fO`6?-m8L>-Sl<6 zYq74H&sqbtXT(*Bk0f)&ohQ}Z5WIPYt?sADh$F$Et8w*B(G8)?Y`sF$QQy-lrqe3F zJkf0K?eR0npZj)%JJJnP*WdBDu{v(pJFb3?c=e)Wg6u{9Ni}&j`$vL8%`g}7NjDEh zgi{+g{wcqm)1O@mH*a{!YbY2BT2pdwbk~W4vXHhJd@M}J@h`jG`fh#V0<1}oDc6gP z@TWFBZX+B;TFj!0ox%k2Y64Vbu1@xIs!buWK~=;L}dn0<@>ib=3h~K6=;8yB$caO5h%xZ0|6a;K`^Ne*hVLzf@e%?i8+b- zPa9h}>1Ciry9~Y#BfbHTG)$14a(O*(ahvV5yvFt{3=E&>%-RHB5H3(jo@0EOm5ND^@7PFDf{nLt+!B|*jBed9?!0TDm#6A^ zpxNXR_{2IA^F`FK+7iom*x9NslpIvx716fq?+@t-bOUG-6K^y z#y9~91j;;GqdcCuZP}njmpI6MsSV!eCYuYX_a{i}4)^kZeRrZn_lxQ=N-yL z^XpJj9)&^q<4aYAU#7t4DsLP;QggHVsH*~jQ!6DvpDnI!qVq;tHj?%Ci`Ev@kvq%p z^F0Zt3 zIQ*v>i)2wktWo?2>CtEpt%@v9hnu(Q|jEo`&f50_X$-DyKd#|1jzrIBV?b$K{fiW zKv^_`ILET5{^&w}%KG^uJT&%%l z`#osfAfdRn+vBS}^3%{m6Cdk`Jn8K5%Ueq}8lo|DhEKF-R-SOIy`?_RJ zBBK0|$EIRcuX!=mJe7o{FVPmhSjdPSB9FECo%-I>{q_@Q~P#D+{ zxQwpfZK9X(d3>)J{fL**B@g$?VmKm_ts z(OE)85kvqILfng=;4m&&>X71=*FR8u5iyiUsZ)1ylzowXhyOrRdMmd+?%zZNJTqBK zxy1||vtEnBJHLK0Z5z8cT4!*`zDp&RyQ-K+u;v{kUEZaHloodr%HKvE0U()qZnwI> zRm7DQVne?eQ<6{aaG@r&I0ht&R2-eUwmT8clX+Fo-du|*>bJP$7Ke#G?0O>=HQ6QJ zAwyX6Mf@KiBm)mtct3gbzxR5ixJ{MEO+e!FbEbU_>ozPz&rZEidw1fWZF*tz6`obZ zuWQ4`8OF$84}`jWN)ss~p;~!>@U*SCgwAi2;_Y1UZ|7_hQV<}q}9_v@)sf9D4?KELw{&2A7Aey+v5A4NiXp> z!Ls-3aa*XMJc!_-w6RbkG_Na0_KH}RvPv`aXNp{_mexfa!rXH2Fnt6F*ix^ib>Bzb zg#>00jV}g4h%~9_=^w!j)!PTo@%17nDPCYs!3z7JoDL>sA`|{W%6Y>n*YJGW*JfiZ z%__RFQv}ewrKfWBQy(8g?PJ2&g?<&6ugY$gj8ys&HUeUv+g-#Ic7Ru+>j^{{ltYHU zM!OTifk+}TX0~Q=yChL1VCZ!`j|Fn~iL?ulAbJKtVqZ#%s?)P7Wbni!{kdJ_K?x(Rf3zuO&! zqMa1$l#sN6$Da%s5zH8s#S$RgjDA5qK`C#GWmOnj#hl6W7->sh&_R9KQQ#oUoZP>IL2gsWpFo%?`Xu4lurh{?PAn&3n(nuIxNIu6!)S0tU{%F zDz4kMIr>^MLQ3-56nO;KKY&J^XrZ_neMcGL_ktt&=$m^7qFgB%^b0=wMvI~C>D)5|qMd06@i<=+B%&u0@M){&(U4B2u)*Xkh z!DH>dr{_%AaLGH<6W)W~|4;z87_ry8z2a;HC<`$Z=9;!meW+Tnj=VVPcHRDK^%Erk zx8`ju$M(kKH1IR5B`L3_@KMwpbBofn$<0>>;Hxk072D@AE=o#DRvCw}HPV+^Sc&Of zA`M*#T#674i4)XJofM;j-}N@#|f)IwlGyXD~-z8)2rv zWSq3`%-WTl46xSF8nD^sjQF-TRZ!aei>)0oEPMaCI$y7l+v`E}H>Tpof_?nA#sJLO zqkD&qt3~iWneStBL4*rv{N3CvqgA0%pW}Sm(xWVXtUB`B8xIxUOi`6NR75pVX0bYnrcWHg0MNiy8 zFqsH_?^t!-<9hHYvnM{~>xpI(EkWN-ra!G&AnlriFp`=v45l#6elHi>0Jp}N9;L94 z)I4Q+Qb!Sg<*~{;aGlP|xg_71l!*=EqkaOd3E=)ux2U==_DZ<|f`j5!^vw^HKyyTJiwp#FUF13GkKB2n07j&HvW&+q-;y0WjV< zc;|}dr(!vZkADWYdp1yrPzlw8j7lt|e0EDF_6tcS{f1Icm|}-ihEQ>FQwN3S9_d|QIeurnZS63-~nPiZvC?*3WXh_Q)-jZ>wpxVQFhh4gu}7gCNG|(;I5v3jQ*EF` zG5e+LH{`u$*S7rQ&c|%``2Le+8l$zOz9>wU4qr0&fIS@xa=&#FJM6^-S2RXmmk>JD zE@oe+`z7(3VoFkpCzyiFXdqz(@CjjfxDV;CZ&gho&1EF-I ziqEgXjv`Bo%e(1^* zcu6-L2ZOyy!Ce^w_$48-rt(5@YaGW>!c167a;-4lhfvW)ZQv3`MQ#m8eHid} zz@+0>iYJ-EaP@5A^UzQtpbrgtaY3o8nMqSp5+Wh)ZF0a|gswe!5N>%>H;H0-8k2mE zft*6Xyx>6a@jA+_qVKP|UmjVp%NQ7o!RWqVv#Avoe(EMS;BW&eA&t0~oa`o`w}p&B zd}fhQM>>>csVz_N0y-O&wPx(Ypj81+*Hkkn_hgjrdc_0Ot)M;n!@Bk98t$0Q1wu^Q z1;y^msOu)trSXoJNlf0+cV~awtJKnp`n5oG`hYb!hjjbz#oae~FQ3H54639m&m8>k z0j0KC_`$o;9NS!5s(AyX^RlC~^Fb!45j10tQ+sY~Npj-`APaeMXg`5|kuE4F`)}S5 zfyJafcYnr>yUX`LapEK2N275eYk!ghY|u8(g!kBMYlVmEJmhL({z-NE@~G@W%F$D1 z22agx_vd-M%9RvbrYog77+o{Fdr))n*0UdNe}?9qx<>$zHKQBFTlX`=@w4V<5SWGc zC|TgCTY$sQgFWEeK15!Hh7$hzOX1*O8|FgWkQ1Wn+0svT2z8h!6xKlMKoJ>LMUR-E=#bQJ}Mf6z3{m*Zb@@XNkYvI`5rQ0xbs2cz(AHK za_sbUj1N4wDPB$%lt2r?^|6KtbH(tRMm!T4BX7ZlUwVsPJQb3kVa}N3?+<_ar16^w zf=(3J7P+LowoE1={`{kNOSpR#E$*&DfOWt9eY4X&H>KgPn}j(7#IcJ+w@LL)ea-1T z97#fQySpX=k=8r!yoHJS?l_^%Xoe>XcP8@$ZPX0^#f_Iv>*NcePeJh_9S?B@EC2+e z?W>7g`+#5D$IU+u4k3X2!5!6c1@Y?R0Dqwln+tOz@d}@H4Akz|o-D5iJoqVUNv2$9 zR*ex*C#AEJ{*`T{RsS=^Q&dr_pzm}G%Mu>Z1O0fTd$CYoc7tepB#FA6a9pXVs_lKN zcm%f{R;aD;wHKw#7jcuz`j7pDzdkYED0sUjz#xFUPC3Leb;oU}-9PYs6alg2R^aew zwMpLZGy{~zfB-A1^Y(b;Q6aFM;K&BUS6iD}YRb|$J2xGX6CnMJuBJtIZ(5e6UFr`r z^x5u*_rH(=lH9Y3)pq$-9=62B({WlRIY~n}^aow~X{H{^I920CUS@5DiV)}ZHr*t% zBX6#{^o)ckB9L7su~hPc5c3u;PZZyT{M@!&jiU0~p!`h-@FaaZ&dO>&YFWt3)xFW= zbgGzx(1_k96h&*Y&DDSPrWys$d2ESV{yJ!9C9VE1u_BnV43o^rPBsi8@~!ZLR5jc+ zT;2M9=Rj+qr%Ss_y+}&wk9@wdJo2p4s{SKsuo&j!~(qqH5P|4lWrF{X5*rMW%t z#l`4yvUcR2;CA=uh09K{%`wwx^vwM^+c>t0#j4BODU^e}op$g&Z8-WMHcEQd?{fLF zBMR~VuB|$U<0Y?uhzuCR-VpAD!0>sumF3`zi`?T3`BvdrAJxlEb6@#8Rd*o~Es0N=!-;>zojv_osXnyO^z}r84_9OHEn_3kzOZD%igE z*vK9=RazGM2oDQg3H=C}kd_Q@n;^m5l>a|(VKuP^vR3?0`Q`EwQOH zxX|yIQd4EMnNo9{p#C3EZGhlq2d-$Fr1k&LM*o)XE&caWNnKj#!*2?4Gh?i1RCw7G zLi`@qT)h9a#YP5diw&vkLG}N-UlsvCvm@8bslu8OIW1feo#X9rVNry zT*ZWGnc8L%k;G~5gsFT}v@iT$L(*p^PP-XKGEv5(lHz1WjRF|sEY7Y;CLsbjWJYR^ zm`Bn}qmtmp#_NnE2oEFLDFtb&Rf#$@X2>e=j2r0rJPf>qGoc7~db%Bqc}c z3pr7{!~WNE$N+j;D(SqzU{s<1iuHf_PZ|~$`%>BrFC6LrzEzf&1aK%LY7IOP!n%6i zae$>)1K>Zgw4ZE9_=E)isemE5g zgLl%WF~&tiGWcQYPSo$?#!phfEa!fEg)=<6(j0u zQ*-{60OoeZE8k=Zt7tOpU$m_&S5FW8*2&yj*;z2`bYM=7 zcC@AQK!m>PdLx*|!Sw0(2(X#4fgyaRvOkzu#2mA5&TkXRzFd*=QyFY_GYa`|4*O?!Fzj!Yk*m{<-CsVt@9Pq|0%Zf2SO^VE|knv<&zec2+G|^?bM1QjMp$Gaix~#X(EW&muglB%6-t z!)Y_*WN5-hfgjsf{>b*TmsXrVHp79|&-eWqd`PrpFo5u_{71#b-MS!Okho2$*voFZ zAeWz~J|z&H{3lp*faP22clcJa5~YeW9ZYiRTbduIY;BREjvoNTQ>9+c)I& z>}Ni8ZNzku12={~vTV6obp00fL0bovK30M0t#xnA`{Na26vt&6ujPgUrZfOyz%8S@dXpxdkkIJL&kLcaQ2}Ng zoj1=Lg+fl9m&czZqphrE1gxE<;o-b)u74b*EGDA(&svIDAPBJ5|7s~3u}~G(g4Z*u z{OC%e)CrKemX*)*xwX+k3N5FFH0pyfI+2FZXT#vRjf3$K5{W?3j|nLyZh#CZT`T96J&wqWRuxR5Imga9mt7=$Iy8e`q8l zM6_`g@%kXewz^*On=M_B0;*37k!IaKqjA6oaRk^}Xr~!xeMg&$5pet=PpNCV7IULJ z&AwcdpTW0bAc~pJd=l*GuFaNs4xI>%;{KO8-3ltVPnnmw1~j}7neuXB}XZdQGJnE2sz>fZk5G%QrP`ZbGCG2cd? zjzV=+9j`UfxGOfFqC2lxM@x%s{7fQHPDe98p1GGv%H?LS_8=i@3Ny=Yxj{ueXy@qK z7N}h{63PJ{h+)Vdi6nJuiWCpf7+)?KLoIPa=76H14O#$iipGHXhFqG*k=6zR0z2Y| z-#Nx@g7pXn8lN7A@X17pthFm$mj28gPj)2M*}l$(5Vu7ywQ52$*1z44rR#Lxz?r&6 ziSWiczanw69)Z#uW~uV1PF;u^`BP{x&gEYr9^7!6Ydi$*r_bj*vuM&3a7;hbV&od- zeL5KpjJ`ellT`4Rp@y$Ji&fJ&Ezc-Q-e}aEmnB)>8jj{J$5a_r7uwH%qB|eW&&S85 z|I`z3X9*h58{A(NLAZvb&>%0+n%)}Lm2q^u#KNe27xBB6T+HN2u9Zi>DKikQb+^^0 zF-Ns06mZstD7D`ZcrJ!Z8qMGPT%RpN5S^1bY)y?5FDQYY$zhZqm;RAXNMpSu8fhrX)Mgbz4>LPAJ3^>J{2 z?Jo=Hy>Pi$+#WmZPtnO|x>5S>GiUMRn{gQSYu^*VlTo__zW6#+>8h77oBd^V+nzME zr#$~%F9nzO7@j4F>NM~T-CC#m1J8xsQo7f%+V$4qyiT2NI*DYdl9?7e5TW7ZmjO#* zGGw8&$d*<*B(aO?ns`UZ89{V3#a~|FKTY|*l{TChK@c6Y*or4Mr$i_{C|QFj?df`2 z=VHM^=VHlz(RJC8tLe1G-h1oICr%rg7$_i-#Q==hmR&6g9pc|5Sz?G;G@C1Za}e!2 zo53PEVy0FpgW=RF<%CqK`3*=gg$7q6M>o&u!0$c=(_q*u6i#HPj@q zxtzvD#3M#7k)mee;jz0TY6|YZz)#5ge~|M%sro^p2{xn7DT&o+OPq5@JF>q0;p}h; zuYUuKXh#Fv-~Yj$s{iT%Zr%4PDpQ$Yh17J`=5S&jcC)!k2b%>>2JK%%%=iC;C#l1N z?{^|b)V_0nO%lPT^GZY~uHtuIizDH)3x}7uKxvk7>X;bIO_?l|;TC`96hTCbmO7ea z%HZ7OCi=lb_Xis*sB9zA+1#AQ_+XahJ`P6E_nNW6aXB(P9A4|wU#BWVsa+N?L&Jez z=~4*q6iFe71#`95G6Fu1sR>F0P|ES!20Dy?#pTp7TvE@B7UPBK~7kO*=VvY-*08C>>!4AOrzOun~ep z(b?v6m1^35n9gq04WbMPZ(I*FFcLHEgjw;JH;RrO&In&a@DXA|_Kug#O7hwpA+G*1 z5Rn`&dAia}ebsZt_4;zh+H^IRV+wBfT5FE(j-Z|Q9~Q!i&k}9s5bfZgIYCABmWf2` zV#xRZO24sq=Q^6j=Y6;8)io}1+5~sE;)Xet#$s)v5(os8(stJCL2+)>M`Toh+Lsia zYH&e;&59rfi?(c&uqvVv7BV*=3X@WPzRD;KQKeSeap|OSJz=^`^+={PK%rnSHTXIPf{tQ1H$3X{YI5@f`Ve zw`fi1GH;|aSq%Enc4zEK$uzY4udt$pbVmS-eXjPrpD+4rg5Clh*1_aGQJ5g8yE%dnqDuYX3S0Spcf?yof0;v8a- z2?i^sF>kitt?{JM76*eK!FY(aZC7I=x-ITZPOT^WXe7MIvp+vxMSM48LH2`m_P_?gohxzS-TwDoi-!fv*TNus@}Y2bXi)>fg2y+!*J=ynRM{c*fl zi-bUVtY1i@HID zPPKO1bhslL%)|QIYZN%(iJ9k5GQ$JKE3%6tft%xfV(~SVGFE|0qus2JVGF$bX$}zT zzidN1dN?|VNIn4GW%;DY=N)9v(-m8NlQsV0;ZmUYr=-$7Xhz~0G&raO)<~V( zLbCYm%`bNb3t(*If0%TD70)hr(@jc~Ee;7E5vLqj>A10>gC}?YM_>FRWKM<4eo7O` z<%U6X(A(?b+4IAd_+2=FV)SP$AqW4*Fzaz5 zJ}-fF|61;KZga1@(^W;5!RQs0Z2m=^yM2LruhkZh9;lASN#Ac&16 z+l-_z+(q;-k=KQ>XjgXrHtXRGcyup$p`lGk@af(#aBeSwZu1h zp{xN1h!|x1TqYhzb00&^3#B9e&BV_Sko?leS7M)UXI*ko(gNA1i##mJ*(hA}4Rdm# zrsjUXm<(A*N)`G%j%`bQj@))VrF8#%HI9u6$O|~jq_A45v%NpA9oJgQcsgxWwPiaw z5_%Lf>xTEg8Wm6!WYw-DGWNfa1I_N}!c$R68^fa&b+kecYX|m+5(zskR`EAcQ3NsU zO8^h(ds-*8BsyWeRUzXBv3fxVb+P4-)z~jz)WxIQeQ$Ig zWT{jJM0_qkeY#ci-Q@v5k9EWdX1i>nOo=PT>>(RK(h6K zyWF0u&z#&?8%wN(0m)q7recvLHzEkxf0i%jiEX>=M6YCFbAF$S;FJjlP!#K?v+Ch! z11QY)JZ)>OM!TUExI)c&zAAS~FRVL1z>E_q;PJbxIADlDb?rTEP(k}iq0MER+*s?; z*OZVDI8Dt8R+B)C%_L4eUZ1u2dP97DWEbmk8xvbo0Z!~DV6;-RLZ-W^5+&w^ zP;qaxCDXs>&K)mSkDkt_ZVY^IAu#h+P=|)pAvwOIi|wY0HX=@(u*96+3Yz;R9gOx~YlnvF#n z8C>>=_6Q&$4Jd((NW`;}8?rQ?vYs6AuaRv1{W5TMQ@IgLeK8#k_qGm`{dep*E<5O6k@wK?Yw z3w`;d#eQ|lwXt>DdM5vLdqNkXXjKq#AyhTGb&uNz$5;ts2dw=0+D#sCMG+*bv*W4d zOOFhX5@LgUj?4q6VDqgsBC?Bkg+G2d_hL6Qgy`St!oH+&RxSZyhr+yE?2U><)!Wya zE!AZpAEdd8uvaiWbs& z$dLZybo~6oTNy#Yp=*_Tkc;Olp&DP021bxf3kULjLa=&g@xfPsDsQ^5$D!#^D$_!f zD=oAyBJC}nfl)!_OQy=*ukEh_q1?oQ}#;kwEMsPzSw_e!1<>b5IO_ z%asz`o#1ghmWkKq9qUyig;RYn`+ETG8x`ypRMK;mdeqw)F*GlY6u$e^D?x8Bx?uli z)EQHaeW@_SLa(zmvviHV>1OO7H*zV2KWLA>n|xHlll#R@hedg`@;qOq&38UN{gp3P znL(AytugTFR>$?MJza~l@8Jq9CjB25nMc*RO6}BsqwRr$&}K8gA=H!53HI5o5pRGzvETV1 zn$%9)^$zC;1ObpJ=80XRXwBk=dhUS_A&Jr4ELe-Qh4J5R^Xv-`4*u*38W!;y$6}{iPv3 zdOdvRTEnFXGa{*lkr635`;#eydzb9Al>>G-Jzs;~a5@@DEkz}b70nCFAss2#f^7<= zn}D1fn{$SoEv16f$B7M2G+Gi2}r+)1Y zj|=m)Zf~1CNK9a!p-{4z{ZA!ZaFbxKj-fIJ-uq~oWYK1IK`&8M2Kw^tKnup`aOai- zRDNf0`0ESqZ})?Pq%~~XXikM`^%xj0&;7M_5&Z@SVr-H&MsoRs1{CQ0Z4HP{R^Hmr zbbH_IsbYJ)C;l1pb;S)v;-wa7|Mo}!n9EE?RAjrty;E6IDbr%`qz9jU zp8G7QP!fq;VMzt{`chQ;HcQfUod=Kyd&U zX=^_ZKJ(_a^NZ&DM{nxKdZ*HnKQjv#&XU=CLxz6$Gnpgo0yl==?3olbK}9nQ&6)|h zVu>MuKZHN&A(`{qrNXVY zDAXBHA-$g)l`J|jHo!ZF4icrFZbV$>;X9pfm<`cC=HF%Pa3h!@}-O7ndF{ zsuFHbrB#&cwZQxG{)kfSMXiWlN}O}xoWW?X#zTD2DV9~QtJc%AefA_1k6gbc>r526 zGd+D&QS7Yv*g`~-#uD#qsYJu?cfUgLaA~9&LbTdKS7hA&DU~%7$2W|M3V}n4gv< zFAN__DwYeRiRqB59*)l3`(+N00#I29*}lIzIct&fKD~k=!8ngrl@Hiwm4-;D{wge~ z>Hrf#n|r&N_kFa9hMi(}?@T9)G`I2z6>&d!qaXulHku=a3>+}(6c3|9XQx1rxx;1b zBl^J5b_oYKBpEzx3(co~jf-EI5 z|F%ULb{WJm1TVliI}ao5hmyAWSQh2<)n)@H^t8?XAJ$)uxZt!{S-aTA}3Qe!d<9whhtt{LF0XJGPn+FiR{*2f#pS3<+QIz*rO2%FgEa=U*k*=VET{?XMPgauZMgRn z@n!4nu#PrA$A0hTWy3q(A8yeyM2{ZKLqDr^DO2eP+2DN>tiM zDNeq89ku&jFGW$#WO(tHH;H}O^oPG2kD*Wy3e8T5l;sd1*L)VB7z+gt(52RIJ^U4c z+)ld>eU^D+nN8%k&2;{~|Mpo^{i}zKLA&{T+uIhej&M39b53(v(X~Jkm*nvfk~cLw z1|m1E)JIBXW5x|vjwS2-00N^H_hd$W0Yr=WLlgDi`nHyNY#J4SdmSLT%k75i^Yg7V z+4HWmgUP+vWh!%J_#c(CfC!_CASKYl{uzs-P&ggPPX7W|9 zg1f-yn?>*LXzDiHC_pQ@mWF>v3q95IHUH|th-eD6(xr8QW>+OXFkF9r{BVR z36w_3OFs;ZPG17uNPeZMs_1}&vVPy_+kvNn{U2CV^2FKm_o$Lc8XYj$3ZbD#0O&7d zyX)v!w!FBU$iEAyL0iV4K>6}It^BlPZw+-Y0D~;aK=d7=Tm~mXh)K{3UoPS}UXi+V zc%Uo3h(O;i52R+dc2+ly7w+l_9GoJ885-zDM^f^*VTst^o09inXx;d*Y*dT#`TOVw z22+TPB(G0MkG_5kCw$%SveBk$4wG1BIbfl4EaLmbZHkplYIq`cj9z#Q*m18#_8dM*ueUxbuWBNKl*_cHuF_i|}dl=V-P-v3&en zj@8RV`0_h^ey4d1L%HJLcNdQ)7Yi$LO;3-JQUaUe4?8|}&iRB;1d99eWP(OUHT9Ls zlpA_-GN6;A=vl~W%_q1+jKff0xYl6oI>Ll}sgNPMG-nExW+VYeoxi`BexS_8&Q*k` zRtzrG?s-MUPjw{BPdX@}Z}wVPtDQ7_J2<@5N3>40*TjVOA?eHB;qpL`J^o;;+e4qb z2l2J*ePd8(vwcpj;mQaLrHSau zSZSV0uzwYONcH2U)h93y+)r}GW+NUPN`5aDES)!kISxt@u~jx`hZRXA19cFfhyI-P zyBamrZQ&)z5Af{fXf<<5fZ5Q;QIVS)g^6n*IA066Gosr-C>Ts~GGF*@)^uRlCwrU6 zIhso6GlVarEX{5x8c4$MCWgFM$wPo@HGCh;qr*JAbWR79Fp zf)wfX-Ef+Ox-Cq+6zPhijB`m}C;HrU#B1;a3!aY9_3`+D@Tot0%zA`E3VpLCpWWeq zn5P-EMt9yMWX}Gz;#_^RMat##z8K2$qfN1t{sd$_O_6ck$)`3pz;&3+_bG!b5*1GX7b@EQcmw7r56?y8arx z+OKNrQb7;943}=TtdQ~nDyKc$Oz(Y_UG^0Sx78U8NJflcGGW{hd%2#zzu!(&FO){$ zn#>BdbH`5hQ|4AmcJe428fUNr#i0(avBoLg}VYf%n#0HdQaCC-M#w{%DErO%xHI`=83F zMSflnw(fodG~LVrC+H1ZnJHd&$iDR9F@HAk=2gkj&f|9^DOJu4Rh4+rvOUkyPQ{f1 zZ9w@z@9RlWjpH)^ir0F8_d!Ye%qcBzjrB0SaB%@<5;>e+S$q0W3bc&V$q;aRI!cnnUxF>wi48@ZLeg}+i7&Uy&NlN^ zqVWqq-znA-oytTFM14#*JXf|Yp9fOCU9S~cvFeUVqmTOkCiXrg*6Ctr3U zQ5Gl@uj4#F_I)utK@IsQ57-(oK-F#xa^X+q;`Vc_|1=4U{)ApAzn-<(Muv<5FwBPX z2V}=3AEl_s0jr1F{l9MP%(woW{eGDUQg}c8k^7-v1~a$!3$7lgbDr#S7TT{Qw~%0h z^S#k*20$LLGSugBAenZ5wN>C7yHQLv_;IK(81$-Z4b>bnggu!5u@@7SM~U6!5+8Dq z<9d52pOM0#ibt2E(Gng!8CB=sK`;KQutAe*l;{FbEnBf;z7M?s1_fU2uQ2LlnLx=- z#tnRCtpQ9nWCb29-FiEGr?v}uL$6hj0#4&Up9Yl&wX2QEfrRX`Rff$I-~hjgHhWL# zG_U$hFPSxJUp|3L&7mbPeUAkXV`$P$cIq@KX?q>`IR9qwqE#J;4-Gm6Er=fVH8gk~ z+Z)Q=q<#4&9fs^37`0r1_%$V^%hf!%7b+7Z^i%kaoD{?~H3{O!aY1j6m8=#^a?mv) zzt3yi*rFu(rF2@@iGk~OU^$+Je^#kqO|s`4^zG?PA{K)jb#s`*!gV{I@MTd;cJkX( zckQOKq9UMP2Jak|fR)kbTr^B|_VsF9Lh7x_B=7+N$}=kt%U?0p+gRZts-sbe62%P} zLWkzd-C@YApgNpxt(A)C^LY=m;CB7Iv1Z-KMsk34zOACu)OQs5coItzN1l5iA4$<5 zupS#b5FQq$UXh@;=Ok5F)s#+|OVhLH%OdV}$@yljn2 zV1`WNp*+W_U#7+e)8mlK_m={mZY@P=L7}0Y!`pqE12q;> zw#(|p*#gc`oB7CcQSWO)A1*g*)QbUYPMoBlu=*;7=b2ETv-uaR} zgP2{JEJ}&(4zz#uG9Q)t^{Bv}-K_`dSm~oOYP89i_9u`tH<|4WB-`#y&sOMcSGAso za=0Z+;f%O}(23y~My60n{1jjlrEufg$hpjAN#6RP9SNe(6YzLl2ZL4WXuH6#KfGXZ-rc=T}d#u?mi zk*i;1L-A(m5p+E{4Hf%1!4LHEj(&S|!|HCC8TLk)e42I>u=zhU_LDd=n)^od;-Nv^ zhMrq3^(wskdr((guYsZME0hHEOIvii*y=67I%Q+C_P_FmlFeG-r{|kJu2ANkC8hRc zm$P=5EYE(W$(1CrL@8anG{{^XDsDyH6u%TeZ(ju-?~y`X?{VHK=$zyD@^K zM&ztbGk*I1VmmDu!p7!RRs3-Ga<{ItB$^dL$fmzDW=VYb_wV2DBJ?7T-@|BoFc+iV z!M*zoymI?i!2K0D}tz2tSR+0BSL>daGen!=RF z7bE@?D2aHi;y`ZU20yue0f7NH%~4JTPX$jn82Ohi6d1+tL{R~hCh2?s-(3J0U;dvF zG+!=;KxcM}G3%VfLYKVSzhZh@JPy_~Ugiq05{TgIfXS=|WCXmP6QWF3S(&oL_J)SAk(h7I+jCl+j1zeWgrRwi2qv+l<8y$$ zaS}zp5)P+RN*m_PqW!@J(Plx4io0e26yu|LVg|O^@U)=!K3>N`Z%_F#YEF|y5j0Tb z3CGEY0!Q57w3=1A`dK#DqK9n|e7^a9AX)AP^#scF1kKjU&ns3 z?Juu^HcW$w77zP0yi6%V%D{UwwWs7h!{|Wu$$G!rV-)qyVEAmQ?d_%w|Xs)Re_(r#P`-KyTQ#m5o>wi9?Cv5gyF~Oc?T5}65e`Ed_BPwg;{;M+kO+~CYSio{9mDg=6h;PT> zH&(%3;+~rwfI&6y*v$?v=sHYv%76Iz3*NK8%K8>2MNcyQycO4ct-4iT zoO=*{I=kFxyPvwt`3CRLMN~XB59n2|B+rrA^c+6Bf~ldjX#16gj~|83XaQe@EG|s5 zOYd*1gEq#ExccIRBAmfrdO13F&XX;gMZaR3qGF(8v(jut%)NK((94n`o>Yw2DMwYE znHbHNFv(07@i+{Jifs`DNoFJQ@W$a68$QF2HMY|(PAB?m8N(m{A5m|?R%I7%ZPO_N z(o)jh^&kz>-Q5imf^?&FcS%UAba!`$bhmUjEWXL!`#rutps?NS$w zZ#s37@?JUZwFT2;^TXR*B{K`Wkyw+#go^}&ULHZ*dwk5Z$N{)pzWyEH?`a~3T7SHi zJmIVM8Zdgd3zg67`ETMU{W*t_OKjM}u{f&0`CvqH9QRcM%<300HC*}Hpx`$maWq*< zqFFy}n##j6`6%4_qX*uT#zbWBY|UTo%V3PG)w5LX4(1*FJ`D@|uhvMU$#(PsQ;zgq z^@9;^CQ8u7)>vQd`R?y)d>>QW?WkdE+;;(x3EDRerxXSmI5ZvsZ6>P}IAtMh5Y)?0Iv zJq*ULkf_OgWnSlU9n3+3P09j;BjAwAIP zFr|+G2C`}m6B+wg&Os-{$j8%mLuAQ~^tRx1)S1X>CuUN^z^&>|LAtfZ=4q8i1 z8>@WVWHj;J#|VKiuU&sbk`(?MCxliPx7})PD1II9p&fsg^t7o4NTH?_yFLs)PEwcrFjnt*ENa}hAp?8S?E0BOb-7m_Ho-hd3 ztZyhk2AXT$4T1%nT|JV=BnhWg?7JHS&wVO>@EO||g(H_wUb57aZ{OpsRpi;rD~xq; z@hq!_ zs)EG{!Nmts8?1P1Tl}N04wORTM##e)%p<>etfwfwZx$?V-cA|DG+yU7NC1_j zj4k}`J^bf=RfW;)(~5FUyTbYT_gqtB95Dt9yv~V@+{ZcI$n-E&msGiy-8$~3JD%-F zUu3I~ImUkg#~t1g(voWT?c+SO{W9%(7y?VjI-gZEpWxNO3@Rayy_ya)(>ibQd*zTO zx6|lxp%m9|O{Sxk*4kLxuc-6waz*M7%BjEli%@vVq-;2`_Sk*nCpJqOd98OoW38W z5~0}+p^7BUp8L8e%F2m30kdjXzd}vDh=HHGGn|s=yf@ac5lom^(fuLFik?%FMXy20 zZm_T8P5PgL|0Kb_J60Js&k>nB6IXLYsrvhfnEMUjAYJAyQ5;{ET?vOqGylM_ek~AV zX-!O~54yijMQWS?r-&g5yqI626sLGRS`a&@?z+hdUFNM6hy$z{hX*q*wd%n>fNo9lOvYm}wJU410 zIWd-Gf&Iy#rdTFR?s#Q~m~V>Q#>4leSlI@Zb9cg-nz7JU%NNLiCtn_g52@WPCQ;Pr zRWN2EaM){^sHL+w_fy%Im~~w{)D}ed>s=uwqWj!UKzyoF7=Z&_y6$+GktJ62|{OMNv^9op{ ziR&Laa$wTQ2?Y2o$B1el#T@@be%7#VdmS~;!IHX_W|`HQ%4SMglw~*HL&I{lSMF#f z{Nepm&YP9%d)t&l`mHrnnzQ}B(SOP>?fY==KU@DF{zS%cS zg!VK_L6<9L3*rlU{|6l3o8s$xlpCVu8fz>PFkmFT*n`y|G%PHx($Q4OX09x%Hv*5r zuN7b|IAD4F&HknNi*=6%ro2JV+tROc7*gU!!j}*84PK-We#f4#QZ}<@Z7s<8e25j{ zG3EP6h|xK?XQ|{tFj@tABTbAK9tr zneBZcedpQ-Hzq#70sV=L4Cy;NOrw)oe-19%Stok?U9S*2e*ekz`RTb*pNPBjtJ{A# zUzv9q?$qaiMD+11(yI>!UhWWpx)P)ubZrz$ zzf`DdW&0w))yAyfM2lva`O(FjhDZXtVM_NNmqr|em7LXXk+TJ0Ft21>(iV#|=j2zV zc3&S;u6q&*B-T5V&5KjzQY(>Vt?DZ49~`ByP^d=7Z;Vj(&ZzG_cVc} zDWl_(j26aVeGhMLCdJTKTl~9|ExWeiGY`U=_hpo4Fk7nCdPjgVnUpi&$x2^#)wEO! z6$k!J<|Q@yKfWD{i zpqp`GPi~8M(>5>k14V!~#8ASCjebyYgL>^FL9PNC>Du?Z{w9?!)PS`c;;tX0e%;gh z_E68)-b^pn7gB>mpO+4yG%G0Z`yvD=>^R}Q0M^g)nn?jp41Ek9! zpEh-N7F?dDLI)%t(IeGQU_pgN7rgsm4@e}rR92Co+v>7Z4{s5{vpECJ7FGzO0UXDi zLze*4`_vKtop(m9y}Q}<&=sFK8v|m=EdBhf`e@f%0dMk9d9V;4~sw@-`VOY zE&%M|?|&^;XsWkx^-9GV0=61&tTadsA(g}-3fb4NvdQpJyD+J*uYt<{hTuBT0Kec2 zn{TA?b0M)%dsjh5K|v#(yZY|u=}^9O0;|I+Kd0T|Uxupf7~KzFra${SOczEi28KWr zwc!@x{99`mimbWI>@W67+$4nQy`ao-9+gRjJdZcnbh-1c@6th4`nyDab9Nd)gR`0W z9Rx_*kI#AB`r$7a7*Ee3rF@C-er`%bEzx(|=Fy5$KMLB3;Q1wBClzCO554K$V*+d$lMDAhW$%7CY$H&$lOC*2yP+;rrq1-R! z-)eN~*Fdc4*q<&^t~3!(YK}*|=5@0Q^2!umw~L8ee<39geDTwQ+sdfFlV#sFicChd zQ>n=Nm29yD>>7rIV?0CejVXsQZLIwMo+3`@@l>|%tAAS5*-z`=&=W0UmME0*^Mtj5 z2x-a4E1-bZhsLw)&(i+HEkATiaIDp-O&Qlg};T0*;%ul8TL)c;9ctlC?k7=#?f*kmTtYhIA9bzp82ZHW%U>D zo!r+upnLlIemi@Mm>9w6GhOVBXG>&U14v6V>FEb&Dl0cLGxITjd&OILu;AU&izM9E zG55K;N7f7=S_e5}K7Hf!3~WEW14uPruy28|o!0(Tfw~gh^9sN|bL2{51Rsj~k1s@o zRV@J~s-9TV569o&?_FmzgVG{R5G9+yO!1vxzOk#+I(0Z=U;pldHlVF?_n{e7X5eAu z@}|nAiRE4(c#%i*U~^1^?;YtPW-Sr33pBv$a2k<2m{d5d>?i4NK5ffDfDK~Vh0XI9 z2Uy=1rW$1_v!75rfNtUhD-5y^AW~HW^wXsPiN5NwM~ovfZu;io0Enjz!T&^==3Zgd z+U~j-QlT;{r&Faulkh9@J4Cnd@Gm3hc-bu@=g0@z4~<`x3)aw{SBe)*z8(WkaGGi0 zgtgIP^!-HQ!vIChoqosklTuEeQN81a=+Rocq4n0?O2SmGII3f}^-Y>PHm#DdH5va; zG8cV{4!d!yre~uS>ba!W-^OWns69qKr7e1%LB@R%;6}@X<44iMJ}Txa66SV4TmPEG zW4D;U%%A!-g~bRwp#DU<aq%<*SzJaMkv)>Ih9mtdlNd;y&qbmE-C<*tM1o%Ay%!Sgp1BEN0XIEw*2;wv)$o z_juC~V4*N4!+os|jV|M0$S!%h7JT?efg`MvCxNC}LLr;XXfubFZmkUvwpstn)A&K8 zUlWy{@Nx!Uu-Bfh(HGUuM^C5OD3gnz4#}6I@xKwxA!p}3WWAy8zxrkV({ZlKT=u#@ z(?+9Ir-0KYoyX<`zeAjZ`CH0e*C&y2t%fu{A4u`amPU%sp@JK-4ICK!e-x&@B7RB1 zv4JG;QhEC+_8Oa5zM92%taW5&sSDBthh~L6F+2iM*O68MiR@p ze9{eoEH=$d!9)xtVJNJRkF<#ntvHQdtJo>jKtX=j+-EpKUB-qo{Qk-~?;E!~6LyIx zozhRq7?Ppo%iRoz-ycxYlT$6nAscxCmH=q=Db_xt_7|IuQ=CKE9>j^rLbKUyCH3tr zj-I2S*7XeN;DXoWV%&qWrP1vO|w%A4l@#Ke7N>exTWG}=qCfHkg{{lbpGF+J1kDD|GA%i zpp}~R1EB!x0f4j*M6sxrO&BK^>e^4wHac5QI7}??=l8|D7qHyhY4EtLcMT&iR%w9R zeY$Jbb#FGXII`8cfwq!&W9?Y%gM4mya^>EG85H`Kp@n}f<4mtlf$#=Dr^fMSvU2Rs z6WEAKg>nQ+ru}x)leD@2=O+mrbBvJtU-uHk{e%0Y z9TJ3KjyOvGgCP7%NEi4-r>^#DB$?}(e;-QlJW#@6;nP~Xf0=bOb!H_^& z`TBAG%gCXK{uckHiTcpCPgkV5q_K$|A0|p&)sOgG-V;I!_;+J+RerJ74l#BtE|W5S zf*l1339SN-jbW7}di4%rDx2lIv*}%J&Hq>fGXrqH{xqVr4?@AgMtf*iA(X;mgl*Gm zcqW@>Q682gix+vn=r*p-7w2vOKHp*#@cH67m)b^l6(VH*qN8OasPyed#@pRLY6TWm z3s?w1MuaaZ_M_Fr2QZmHgC$(u+BEmx_}#xR`Dt~2)1L0vOD?pUA6QVR^T*RH-U=nx z`X_lUk3p9lQ-&Dq7GJcj`mm37-tY9R z9;(8Ascy9S&(7~y5r2;={27VTR5oer0oH%e6hN1iT^fqIhq5kAdPf8|domwY`*PHI zRwv*2Y)_m77~M0n69m$@M-WmTRijCT49p81fOkG*iff^N z7Pb;kCA)b&XQ+v+n>HIjn9rf2WVv=5)}Z%;4u**#x$lOepY?Q4NCd0w1qF+eVVViP zmBin?^=0uQtEzq%>r^-LgaQ6^J-U6JH0v&cnx$cDSNX8K_2zwpY)^@HwJ3TUYQ#>f2_3w%Z<)6BuPKJZW8AV`omPqZnLjx+Dmoo`_P4+a;hL4WS?mhy~8}% zwl@&P0&rA4j3E3Ed-D3v6ya3VIgTgps{2JW7WeNe*!{-%JJR8_3X6nT+W)Udf?W)# zH6V`+vqwA6CiylYh8>;c)(wrS7C2BDV@b##RqEB%8yEeZMJ&bHDv+8*ZGMb=D$p|F zZ!*v%I4xcJZp_2UmQVw$!#Tsgh#LWA>jQ-jlWc~F1M5qhc+&0fCWwTFvMK_#GLgUT z07Zdh*xGA2QY(Vb=$5f<5WoDb4#a0{@K90QRp0`YgmaA3HffW1&;{@B$Dt}r7vj=s z38+3p6Trhv6myaR%pg#B9#kV3gudMXZ|{u6R*b+6J_0%kQLY|}?qsBJOq2ST?PZTd z`$c*vR;}W^f_>LgChMvrH$tet1;`*dh~)5!C3l`@@H!VRc7-VCgXI-P>}qO2Dg5?6dzH{RZD-}4PKltBy!TNc&o?PG9DV@V(qDZaEy^_< zV6m8|OVn&KxvYB<+$2S-j4_zPY6?=g5d`qmuGF~P0-rb(7mj+ zUFvnOE-6a&B06WLy0mnP4pme;RWsF6(~YU&!ozjy1}S1PBDHKCGt?Qmf4ho&L+VL+ zi{~VFTLAHJWGV700UPo*W~WfBmF4GQ-6yIhZi-}R-V0#C8%^a)yPvk+Ci>&WN>Jp) z0T^*>!a*(r0lP#;#|PIz4~@&KRwRpEeCG6f?){|o1x7(>9zngz9<|lwld#&qc?|5M zsqZbK9y2ZrTTZ_Zx3f=?N@P5bVtdoCsTmkm7o*cKzH`zfoBEq$Km`}v)*yT7gZP)y z|MatbczUgf=`ig>T8yKR+ywqeKUh;{6mX#}$c{3ay>|l-#PojhzkO6W^dlCQUrkoj zG#wkwtMrCj>G4M)CfvBI6PUkR)jZ(wdBlC|O#t9NS1HN2o*^A`(&!F#h~AFzWLtajha4=rv4mW zjPluOw8b6H)sR*tgrnY^o&2Dq0c{mezpgyr4cGCJK*>Tc`}5T#e_M3y%`IE!(i)Pb zg5lXX5vgDH_3N#A?`8J+3Xpz=SI0|uAcNTf#Em>b@o<)k{v}TLv8e*Hh|!+|gwIDc z{(G{`PNkMejHDb-b`dj8|26Ln#e@HLOQ3o2XfYYyKZx*n6)lGe;9u9~9Xh>7$)ZUH zvy8adgBkkI_d}-MpK-C}5*q{*9kHJTM24T5Cd1p2Ja)$m-eB{5p3i}6@j7Wl1cBW_ zM}xNwF~TPMUG(dLnnd#rqZ>h2OQy!}arM)q|AjTd@=*ZVSdcOsT6q01e}lrP_GCC~ z=i;$6K3-IqzM=%4ahW7)Ffont3$KxRmcU6CCgrpf6hfe2N!SAGX{FP__!91Yj`zV$ zt`>VCxge=Fi8BdP#6dw%(F`&Ft1fb3m}%8(2x5ezbmI4~YmT6M0*xZb{%*b;(=7w6 zd*a{5(qF@f$=1n@B*YGWw-@hctg$1$gV^OV_4eOeXd%mi7sm(RvI}(eLda|Bi)nR^ zzx98J49_HtSl`1CzkF-ZnA#K1vUi^meK zZ_Jt{%UlS%w|!Q#sP#F%_|jRY?u>`c?QHa;l#fvBUg*1g=*Pt*7Rs>q_-Sb9t80@#G3egrH_0pc>Fmm7tuV z8N?drV6@PUV2z~FjxFT>%owAqs8sRs+xLl}sB+2fefrsR^q&_U{L+nK*W~oC=DNRh zDY9IZr*}y;rhlu>-;?n92*RJ14>VB7CB3Vp)ulziE7R*1LS*^VH0FLhOUpICzrTQ} z)*2YLpxiFX{P#~R>S0}$n`;_ff98grqTX4(WY3SaKp~8`mutQ?VjbQl<0^MAySV0g zk}-V?)s}1kEu*BQ{O(r-vkWWI$$l}TRWes5l3^0n@8{!(%ss9RjpEO^{5z7*gf&TK z3?|ITzxSN@(OAB*aLl!@n-oErLkbj}O;*wdRz#E6wkl}5INnmrc_f4dCfv1wuli29e#KcOV zzZ%VcK`5Ce!e`rjN3S7wo=HUrpPsRbuScJ$s6WKTkoCg{%C`Q8*rm;RL#wty0jP^L z=Sf!!>yGdVXV^G7nR6pE6yy-xbu-wvVTRH7;QfUB4`8L)<{qH3@?;4_ zmO$su_F#gf_(Zt>9un1DKW*_BJk+gh3h0NRp4U5o)R|iS4BL>(5tG;Br`C zcRb&wh+po`eCbA02bDn3dX4WwJT9y9tf%4rmS+=ITkdu247X))@Xg4p`MP9#`@L(W zfkbHkQScXZTFKZ}&-C-!SEgDAzuPy&X6&8;-H75X79U?)`D)8gzr<_Zsn#ct7EPZN zmLGRX)ML_lC3okZ&b4QDbesoBL0?YWe>kIuLdOMhmTql^jqhUir%flS%=vZd*^q_{8W(RH^AV)(_* z(g#UtR7P#043BB%uNOg!wT-q4#nlC6tw&uyBC_g+d0YW7Hzk>ObmqLF|RbwXUS9! z8i9iE-?7hPcJ9uv0+)A&`EBm~yewhLX8sDCxnx+%4@`V8E9OMLCWsd$pfC@Or&Xq8 z_xZ}8non+KrDUa)#-(aTEB!E66p9d}5-+iH0Lfl!J8jGKI`KI5+51wu*LW+?P2%_X z!)I38>)6MvH$2yATHd@sCh7Apeck&#DtAZ4`~KSpq4{@lYbx0-NNkxr4q5yc0~8k^ z?yor}aI4o2`G`5BkRS~M6Yx+$DWXoU)3dF4<>c6~RPTB1!|u3_;BmVmqtfV9$7%nA zYVSt^)%D>E0~#TO;dbpM?lbMX2T5Uk`eWyPGjMmq$0py=LQCKdRJzfgh_yRFznDY>D*@Vvhs)!btXiNWLP*f86c!%y`yWvWwnj5O!)ssdmRy*Qb8>r%b+F*EO zpJnKMNRrVc-RFH3B7K{fUa(Wrv=cdXd;fdoVlX_@qsw6_%^bU3YzP+zyS4QZU20TJ zHl4NoDJ~CcHwdsjbo zFGTIPhA!VLFQU?#XWlfMTsP;Yg;4i|_>BMlJllNeX@KM5Ht9-mwd-IpFl*RI@YX^K zk4zqWXj+lh_6F}6Vpmc%MJ+c7QY+adTMmoR? z%P%P=Cef+utfsh-d^hn5z0s;P2?96PCz-?n5B$D5I1lG3rjc`myX zc`tFqK3Yyx1UAHnH<|S&MS9K}<^-;)E(^6hmhRZLM7A?zD`Vw0*yS(JEt-AfOVodS z?{%||ylbu+&Xlw-wn!FIlIHNv@?MsVEws`VPFpg)|g zY(eR6qILM=w_-WARJAqj)_qnxm22R^Tc{(A6#_i=c{;7*7PFbY_JYIe!-5^U9r<*| zvPcKRkD(Va4h#%06L){&@hIy7;5h_9AXDrx*WCzjNWnsznrkxy#k^hKilxeSf`plAqPn3=IpCJ!ovYpkdHY6gJ{6~(u}?OvTe z#!I=TkP|Fm>7wH=@_5+hQfb1-2qX8z_|tp9SG)krIv*HTDkWNM9)J)B$1Oel_gU3S z>NBvK4Ghv`GZtkH?TMqK_u#BYkzS>@ z8BFqjrv#%+m4n6dwEleFPa2gySCC>rztZe>izS2J9|48(pV;10W6;9b-9KP{_b{p) z3zk0)iWym0WV$gvoh& zPEdzH7J4@2IPd=`W=W#@-tVMb0Cz9lcoB@}hR>(`_`Q!9$?Di=K*Ct8_5q=6I*|8I z00NfMVx1k~>81_D7$Y@1?Y=(r%9_jQZX_}iwmdWY{wn`j#YVU#MH_ima{m`iHqy61 z3AE(;jMNJ^EM$Abd&GW*sh?sy<=wSq$*J6S>2HYsQv$a+{p;s13B_;-q_dsEj0@in ztNK$wB0$EnI9pMb_JwhNT;n~=K7x&^Dv+AlBzR1&CpoKmiOHn)bE!oD4}&(K0=YPT zPi%l@2*^u}cOFqB0y^&jwZp9^|KTHCFghu-faetkwKg{8-9e^f7c#rY#dcueej_D0 ztI*|1sYMzN-IN%msoLSlPb`)ZqQdH*@x^Gh#jnpf+W%oYYycF-<{P&A20wro#$Hev zx0!n4M<7{;(#Hh)vX+Y>|J)Uiz04_KIW&U22dX*>tc(&Y6&!SlCwM4|^vTL?r|smj zx@h>^{0zETY1s>wUZU6&c^KY%o|n6$B$ihbpvsc@H6Tilv~x;YesMcajcb`ZD0J*| z@$W0atnJ_8IOQwHMRb=VK3qJpk9z^NNycB>2=wx$;`|qpvs-~rHCU{i2N$GB<$`qQ zeNAGQaDUXmc<+XR#i4_a|8YC}h?wPmOuV+L^9JjROIPvp`x$mByUjnicsf@`9?e*4 z5PR}ypPf^G%#cyq3&q2YML>=XNAav$;f}WgID{&*U4$@X_~SoD;4%69(yGk;;jx=0 z(%T9k;qMLYkOy0(b52JmjMTUVFAA{+f(7v(Z%ZF*jtoC3Ai zZ_lR?;YE56QlG0huqJDgxUWMu?xx%3ifO7w;Rj^$xur{E*JxLCJp`9z@;Ge|80FdA zfk(x68I9vida}gK$48;LV9T|>kEm<*b`qU{Mw!n>8GLs_i2`bf%7_8;EtmQ6l{O%A zIrxigHP7NSz*V4x)N-?EI|aP^*rmg!!!i!srPKD8)qG4D0!{_+gC{KP>0(;D`R>MI zaZu(toJW%X(Y1V=nMfv{Er}Va#?(bD7^%iPMMb3ACP&Dgy;ef$SGBA%(PuFg(~jAZWRp#N-@$@}#ac%?ns z!&17m!g1-yLo`c_AmqxUIw9(D2({7CpH!^lg283NQN-|4gI}g!Z=VVLD1lc$xbe7d zX(XwzaPMx_F<^IWI~BAFrLn|*U$Ph(=zTxH>N8y^?~*H@6BSFov1@5z5jMaH(b8pM z5c_B;b<;Bo8!Y2mCpQ!uY2Eg7)!&BbtthWYsoNOL<{ml`uEtP4SBX;A6NhBvpAtAQ+rk*gy`@}*4@ZpA#H-2^Bai4x5#KOT zEnJAW29Jykjonek&8?s&KM85S3{|-hev3+G;m2(WSv35N-QrDywce54&y_eQq|3;3 zI2%X7KpYQ6HAEK(J}A|ER6m}bM2(BrRHVmAGwsvVlJg1pG_e@ISh5tH!B~=Bs>f^aag_}W zjA^-l*y^wpujWV|9cnr3@3J9~mwlI9P2E2mpGU8mXxHpI{M*ns!KMAhI9#o~Pg$i# zD){qan;5sl8fX(_S*jb37VsNB%-(fTr&TUV<@{LKq*kt@eRJ|6sqs>$?Rz)#5;%IC z!4;P=hH8(T*X1t$@Dz~SYqk+{xSjX%{DJGYMm*|4Hm!V2DM8%lM86IEB)^SF69RJc zq!xgxtjblm?<8sFYcM?@C^NGBmfG&UwNRcQ^*-yAqD-NQdbmC$EXXd;_umuKQvch`~^ z3>xkv#6p$<=eDosCMs20%FdtX+WaUFI|7z3+fV1s>oi~L#5JBDnu%1J63tS6#d--Q zqV1GF<+(hEI0+!Ur`bv5r`9<0AevPd>zod8$Vm}32{Pqo8uF-Ntnhxsg+C2vD4VdQ ztV|a)mf5=8(4{4RO0!-lY1GtXe}A-4SKFJc@3q}Ls=G7Z`nX-*`k3!NNiMPfpHfXR zw{d3$XtC0l^X7i8EaM@JB z#(#)aI%)m2ZkE}1^Twm#F@tF0RV)r!E)IlWSiX)#wjY)?Uzjy|y$D^5xATVUH^%om zCw)%rROyXC))*oeXV16$mVowxcygnmcS)(!5q+TvnQVffHXL;cnkTjh6YF%3F%2Ob zY5%17?`Ga~2TVXb(bvCJ>S^;(dGC`LG>3qix&yrK*pd>)PMO(OFSlP{P8hIe$4WPz z05LR4J2Wvtx7k8@!b|cF%PCPzFFPi!iVm$S*W*Phz&&ut)>fEB$Q)p%W_#RfVBUwO z>pvaE^5gKwW-{?XR#_BBh3>-2j=dnsn!7%933$v&usG|D;5PbF$-DfG@ zDyQfOsBqFi(c5|5Y4dHS%R20_|?8{oXi`AF}ngOIQX2V%nu zoJ-7ch7#kq{t(mCkZQ}@5bJ2$kZP!(>^m17{l;xNMDlg^VYa8bTRk2hibvCZ8IeRc zY+-48VbdJUX#D%KI3A%dms16?fQfkE} z1>sJnDStJ%l`rs5j=&(y%pPqlj;5YE!s>PNIgL9ii8+_y&5ql_-$8F?rXTEUUZ<^% z-9wCZD_|JQ`xlh=pG5s`x+f>H8f1gey+y| zV4vd2{avnzBIFrkIs{ja)iT_AM{I@6!MiiJC_LUX2Q_*-8oj3`yJqD@W*e2Y?QS^G z;fq*iW&3oKm2qCFH*=(BdTlCkv^|dgmTalZfBSrI* zqHg(4{z~{NV*`MCP$@SAJA_It^?T>%Bj5=ICkFGWOy=@(K$iPm43WQ!t>^79>^wN9 z59Zi1^9)veMd_})+ucZ++X}?(YpVYj!gaB_XAj=4HBpqx^Fr+>xD}2AVgLx>hL;C)U`i1CLVdW>YgTKE519^jV>V$tv+`4SESXq7C^>{?NUQt7 zSSJA3o4jSV^ITB&rqb+soH|FL_x8Qee=v2=-y(1ftb*0uTt3oc?lJ-4f2nYMNf&3AS8hLA9}E7nDe4_ z5%Qb6KPm;`X~u+(RO|aF*yudyJjrzc%%uz_&}3{({o8q`H3XIhHe6}GZ6Ym3w1WTr zCuKgcAyd}GYvNaB`klrqI|mPLk4xa-a^8Bc4I{`*Z(%*cE;R6Z>{`Yh6eaCtl6J*8ytV0^@e^j?rrirUaZAR90!)G z5KQe~N|hwxin-4V7dzq_JBH=V>i2M} zv(E3xs>j-wJ`One*(VNWM`2>OoV$(1=8A`d%c?EqK;1^Vnd77M^wwCSWZ_j(ghJ~O ztzZ5y_d9v2@`{V1$NRl3ud$M9ywDe2M(qtlw;6TKL6A6Sc+K#b&!%glV7c3ztz(zz zZ)(Z|%_u6-&-;$%+Od?7Ab~YC^1P3RMS2bsZ@3^PlbO_4Kz!EWev;M~U9mwwwI_g| zbA#K%{vq##@~e;poK}MU-dN@oXk^CU!FT6o+yPiC9>*5Xnbv!i|M!NOU`qYkSmerZ zYO8hM@@U=Z>eC$H5%Qsn@Ov5=X)$lOiOJM;vixhm)Tmx}x{5n9i{$1`_B$xTbd%A0 zq-5Fm`2pGXFBN$aphK1!Z(0EAwoz?PFVi9a0kd4!jX>4f(4)VeE6LC4Lh)cjhv#Bk z(}(ffT9aWJUr{qbletuKJE6{1;t7iZUGvR}qvYiCe;BU<$GvFJucEB?Ab!BR$Wm7yfa|5#tNXOXpv|$m-r{d{32BcYo z7`~o0xL!D`r)sACp-@&A53&nHb`elID?ocA@K6bHSmXE8YB94{Jj5eq4sOA3J{)H~K8sQ1FDR$>wxwQYjn20`X>@eAl8GzcMNKsJ1 zskT4y^P47yKyo466(riOo8ff+!ga6kY2}kiGb#Pc_quKU^H0w0*!8ENmnru{WbN$n zO1tif@3lnUKVsJbOzM;L(Y3uaqw|X+va0pw1qXSo%$ zhji7uswWDwzXQu&{ulL5eJUcdOoZC>^cpRH196~=^-#rP3wne_l&{AJeGE2m{5|OA z3h*3@Dbe9%ri4Dcars84ouNLO(7E^X*rfFy*)erP=m#Rf`n5 z;Nk&+*aDUk!>q@U$D3`@uVLV9shiRYk+jlNK z-QQp7c&QO0v=CsNz%aTiflB$Au`7{T^`mtNHASx_h*NdVF@ZjgBz8vT)+-ec&@~Yy z9#ij{bVY@6a3)>cqxIM4uifc3y*qa%CaEAfmY~mq*1S@Kme5s`ap_Jj^>GwTA7u?; zaD803Im4hBad5iTbc!6#>%}vYNI_3*huv{ez<^KnWp}FHtRtV#?xL|hf6L%JXZL4+ z+GYE5BN>lg4Y$c?bA{(fIv2y7P0uBwTjEbK=ewpY_U`}&H{!UDbfyhL7r2V${5(Ga zCtleMZmDgcmJLLC&T3l@swxUlX`O?9SJP+~-MW!&4$xLDG8k6SlOYrd`i2FyD~tku zaSvSpeaSA73%d+NEN6+FNXH;1S@v-ilFW$ZO{rctJsX4&BSba&n-;|Xp)WV@g$ zsfU)oy0=7$Jzy$JgJd61EAWlhtbt{fHjX6y)@DxWKE!@UQS6|pJ|MM1vbC;-Ml23NLm`Fb^Q6E^WIj?MFHw9 z7_1jrjkTTt3XIqGsb`I2oZl3;(S9N)KzY-s+Xkto7aVj^@LxeSNf$kGkL_a3FXLk( zmHw`NK?=?qOlvk2asI{cc%I%Y+FLlb^u|g@p!LT58|kn?u8fw9L9T|(FpeM0UFk-9 zAECx8pRJ_vV0|z+<)FDG(hzLMXyG7Q zT5@yX2+_iCvybY#@B?$#5~^WBfh{h<&IIvlV@x2=zs0?Lt2h((?M&akcmMda-!Adr zX|q9@zkv@V6J{X*WvfQ_9(NldJFfz=h|O$?wf8*;T+)pTK9;;n+00GqWahi){5BoPdug$b6;e7*^@5Khd+e|35|>kv{m?zb6?bE<5}?AC$=U;3-)@y zbLttzGXGqD8>P;|gE<$he)b>#ztLt&;!<~JO8Ar960$LZ>^IkBTHi1j1B9N=zP+vn zPM}Hb_jviw_*>F(ED04*nPu|ddDSSRc4B4M_l+-`xYP>Spb<*ozi&U>-{4WWhGrfE zQ82?a?1_ajyXDg#tzy&JFpU^97W8hcrFsV*hr(Zr)XU4M5Rc|+hrPj7dmYF+QqRL; z+hJ9}uB{-te1rThvzOu7q*vbf5B?y*)-hfu<2(kZ7;{%5W}fJVo^tc(uNF)`lBpAu z#W(v?=Ogw`PjN0gkstR``JJ#X?0$GI`$fXXtb|Z6GhZevx*>)w`j&V|{w9Az%KoFD zwG`*kDb!iW2^?jFLaZeoZzMNELRhrPrCaXvD96({Bv-vo1vr+&lfZuFA;Al|Gd&#? zO*RA#6~`6>?X+Q9W@(9?N&;12D~g>)=h0BsLrk-b@uJMjOt_tkGv zeNp#-ASp;lND2(yD4o(>ibFSuG*UxKgD52_F*G6|Aq|3bH-dCCw6qT8_l)oRd7k%= z`0^VM!@XzDJ@=fo*Is+=487oJ$RCw!$D{PwT89@Qsn23k@h}IHhG5#OyuAg{4x>%o z&q_U_GuIys>Zf4ON!!Qf;s~h6ba**r_(_%qIb>XIlwY;R2OiLUkoLNDP7)IjwnQw^ zBG=-1Nrfr%x-Fx$Kpnd8+_lNX*{=WUxeOt?c#WOOi*2TZF?v$$%KNL^aRPnX40|h( z>bMsza}V0O!9lAA;>Hbt*FR`UA-l-1%*2XClI;=4uJz?uz^!^N-e5|7LfJzHJT#YF zW=?t+gGgGhv~*aKwE|I7L)yhoJdpizf^!K^tW{APqy2tbjd{|7U-x>Wpml1wkt2)b zl-NF0kn8=p?9#DI{YA2dU}wv{u|Rw)}l!(6j=qu8reebDlrw9zE`weyZb@7 zjITK**SP+%@Kbau9U6Q>-Zay^X+3fXnAk{CH|k z88FxBR;R2KR4y;<3uJ%Up9pau5k;ioQjg?zUpE#l5tDH?RFa#5JlpF?`>H@j=nNlM z&f`uT`ftj3%e=(_Y46(UVYC5f<2}0%`~4ZSQaE4T4Yj}wISa9l8A)x z(PDLn_$`b5?fh30sw8#1G$>WK#l4IxR9_(o1xXt!O*YfBHk=e~mU=$U6sOX&aj7~D#~=Hx zU*F>>85y`mw#OyTj{RymjkUJ1Nf+w+)IR3R7<0J+ao=M$n5B$exd(io4B5Z(m|AGa zMEcFzq*6I7g%U|Yp{|nfrTThvu;V)V*=f1;+Yy4y6irZ2Fe~!J@dKeVkEUBO5WKZ5V5MfQTGrCK+(Ji= zf_VC{F9F;kzswM6>6Wl@@sXuwDg^N*?*3psa5S_ zJ)>RPfHEL~5ZTJ zo#EdW%zR#{SxEQTkINO3ryw~q{*Lqdu-=q< zapBEhEF|+c;NfYRI?l7(xC4}#(E>sn+wO~`l4oHYA@045>3q66sA&f*%6(o8F=J}p z6P0Y<(yH~yVl!CZay*|suz@j9D*=%M^3HZb4Kpgfoo#`=Hk|%esS0YSR=1i9!ow#| z!8I_IWRhD}eC%Hm8#H^B8*qearIq-Qe*ZXAqQ}|@T(CgDE5#%)C7q+GfIPb@r}h~t{gT>^9pSt}s*05nKR4 z)fesn#q30iG%-Hmq>KU*Tb1L;wf)Gu__JzuM%3XIZ3*`G*}k)^NHGGgN|YdDE}w2xLju{wvwU>{F|0n|DajdJSyW+KxpfrF@?;qdSJyl&u8H{Qi^XCc-o}WRtCdo&yBCW)y6Xg@$KHyXZwk(|-nvBQ5hq@%wV?qVDQ$)Kpx}y^4 zZ*2R#KzM6mLYnZ_7`R`2ZBZ+jnDiitlc_NMrk)#u{l9Z2Ds?M4xc}4?&3& z@;!=}NqrkkX*#YR%Gac7A^>1YTO5jTBI$8ds@vey<=c6R4n}mBP@=MGHhcW79vCPa z5J^#x@!Tszagxf;7(=mUfDQyznC;K30Y2IC2KKzV<%YEj2Y}Clcq`PNfd#J&5)uOy z<``lv+I(==QI6#!9FDfL4Z9hB2k>z^wCAPz!zOUi@1wIK$AdmwlpHDRa)@wP3s(jIHJ&r`<2jjFU=ptrXMr#ZXi9cqN9)OnA< z1F%Us$ZijRv}reO!(#cwnk*BBOXItpYwO+vBt;r+V_=jxh@5_Quz{#ys5~(qY|`>N zD3*D&2wJ0s~NlFI@@YOz7WR6yo0U0~twgY*tahvZV zM=+E`tJ47Dhx4N3kZUR)0mRS9TGR2x^vSc27 z*CDq+QinbJkLe_RH%jd-W-8vxh>J|ZCH?B`+g#IoLg3|c9JefD zZGam1X!4)%djV8;G7#5m1rk07R0Xh{UCeo*`Z_MqJ@ZQg{h>UVGvm1%I>HovDf{O{ws*d$I7nG zK$a+tLBHMg%T<>V7tx9bd1*oW8(fN(Zefslp(lwj&^fB~=P_;h8Ma;?2%7P92s;w% z_w>FJw(223g1`@GqMZ4A4S;n!sLL|_s+H~R5G&zwUj z7F+W6KB@f|5E963XdRnkh;2tE5M)q9~x^x6xBqJFp??JkaBpXl)>_~Mlp zvpJ&8DhLPr{ce zSuB8L$GTF#b*!>V4N~EAX7oqR%4^Dr(V-#rsokzxx(Yg*EL#wTiATcf zkeI(R7;AX0LQ)HG)y9kYY>02cIK>|?pgjXfte-=Stvf{-?)k17tJIMwGo2}VeCi;! zz627oK9r3ObT6T%YTM#@jT1?U|KqHm_g8Lp{-L(6L>XOn*rCBkujL(a+c<+L6g`Kc zyKHB1_gb7s3)guC%`B|BwfW3tty9h`J44x=T-Pa8sjr6kJTF7t=t>`al>-7u3ocFX zC72zLE?JZ4>#a0{rYq^JF-!xk(RzzJ^oN5w;p`N&ZRDP#Q9EE1d(heauyO- z+7fD41d_IfCua|}F>p1}q$RG=vRdEZnp$&b|Ib%rMngAYm&~*60`$G~quRCZ>wOf}QMIW-6*=!Hr=UFv{FR>So>I7XwnOi{Z#m*hZyH9@X zJ-HkIBqoVD{t@FWz>-@11NMagi9|^SgWOyqZh(qd^hL(b)STf9NPHOPc7|tz5H7W5 z%s;sDKf6_M2M&s9hOYUVqR@Q&XUyw+GexfgYE8konCVfl!z{x-`XB5-jT9ZC&4&K` z1W%y=9o*=@9WUM=6U1t*LX#w%|EBlfUvjho>U{84`Obe6iUKQ0ob9fd4F^H)D)7&^ zGBT{D%rZbfIdle+)+x-uZ*Fk+ySEJ2~JBbJ=b+9l(& zPvCmdax&bo=t*)zlK*xcFl706Ko?FdSAq+$#(z`J;D!SlCK;Sm0GI3M%9)JzD&R`W zEEpg-zgb6FwtQ5#A*?S0xOcG`FiEB(5FCND*tP)>)qBl=8TzM2MIJVsf87}n#aE-c678p-?7 z29_T6m@EN37gAgW9)>Hih@rgLTV=`R;{!$kJ$c*4j#(4$AiKEUwV8C`8^>}qfkLUe zzc>sHccV$rv!l-t5)O!~6La_l6-A-MliA0NV`lQz_L^(XmaR0#Spb^{r^1H3QjUA8-#hGTqudyT-0WgS z-kZ6=OtVX`0f`|`8nr^zYuqtKX9$rR9l1d=8m{?(K6~1CTVji-X?E$$2&3Z5rDx^r zvs8-pq@mxN89U%8K4g`6B|O@$CyTLqAc5?Vvq3JCo$)}S;1%T;u2^fP*2ey1Mpev? zBY1P%@2m+pCR0x5wycnH0z;?6Q7C50A0b%triNE+l!WJz$00c_olDZ9f^}Y$@wRwp zmEUW*O-H_@=I_EZ9ol`It%Q?Q?^@_65`MJSXWnBJm|&vWIqm5@8gMs*u1Rd~eaf=; zb!t9+#DjESVBdHh?6N(V5-l9CM~xsDod4_4YC*?^5hJ-YpiDoQp)qK7qnx_xZX_J> zy~4wGYd(VCrUrxb0#;fqc}td%#V8;rzw(QW?&VRn%_E|vTi7Y^JK@6a_6QT+F8CbH=?oN4qftI~jsUq>jhz)69$ zRJ(!5vd|3nYFXtlJuhXrq_`9HPu~RWz`_gP8v9+BkM}D(YfUMHakSYQkV=1ierDXW z(<#nbAU-5m?tY(Vt}(VyfI~qgj7uRbY-vUeo2Q7yAIB5MINnj-=G$&)_#Pc;xw+5R z7fOf$iQt>+<&ZBhiyt#^?TjZnXQC&A`gMwnlb$>>ai-}Qn>8rJ!J5PqCiP^dc}t4e zPK$DR@N`!xQma)0Efbl@B3$y|K{GAsWrd&Lrld)`6O9ldT231ES$j;0KA*q`BQ=F&rGixYcbmy)sbsjsQ& zu|_Y(!@dS!Z_4^8-XE3JAyP=U3k`tPA}8SD;l3nrDZDCGx_u|(dYd7Nsp}jx5^~5Q zdrIl^0mtQtr#tGSboD*xyQ;oyJ|nId5HuV7N3`DXQ8zN^uGN)ccCo3yBQo<~y)}DZ zaN?$n(Pg6M!1BsF+d^|yR7S*(ZsfEns{^x4;5*jJd2jb#i#=>G?{q!yYlniC@zPS+ zPyCs~Q#BefNyNQQH>0~(cVq8rU!tMmK2(vHeHQ*S%zX#K=Dgse-?>B z%DuY=X?Y^LEY;tO?6F=KaZ&uKEsdDZxTmC~#VWU0-htSXfu6{x%kLW%Tt#fsAjS3& zTTbas>vM$jUgUL5)%&mAN03_7BB!p|lTMKMUCahJjPI1MzBBIZ&I0`6wu&J0=8fc5 zg^T-gvrx+wU7c!7GhC)$zP+00LOWUFQm3Nzg9p3#rmQ1e)@+sBPK1pALg%gaN}0k4 zZKQuzK&hmbh}EdP3{?hhoUzEU;@QCWepsbL0gM8w@+ru} z^$>HPa}BI;@9cc~4LXnCa%DyjcjpR)0xmfeB`RBC=Rq4gt-sTKbmPb9?JjS(E~gD{ z)aBC}golNXJGb`siJU%e*k4Key<3){G~`^|q)Sn`x{fT?Yq^v_z^DVio#wds<^H0e z7wzKYo{K_cX+-W}D<0Z?m^tl8b%_eBciulTB1sAj#$l!?Xil_V$oQb{Nr@BWe! zAj=a{^NkSfGwT_4viq0fbOXQsm^$a*9C=80nJD1FW-hAOw5Ph&TF59WTt}>iq!ihE z3d8!n%l+Hff`&8E^K0I_p>lPiWp66(=|d^WUX&J4kWUv;9acEayH2Wn z25na|o@sZmaS7uGaWTCqB#N0_v?xHts@m-a#?)Ad#D-Xv`&{_2wRyNM+&49g&e3dd z^j2gq*H%ie0qi_=d-KS8+&5E=Tp|3ktZ!RB5`Ln;=Eg`?NMD;M}}Pu zoxq5{hY>C_N$}rGx*;g2^qVt~axjQH>m(cesJ9(dR=5bGai9#e;!5Ap%5{tQJy#U8 zy2L_MaT|Bb4#=Ru=o*~}q}6t+g>D&~Ab{xNeEIg0Hx}En@%2o!1o0)Gcfd>5G%O=} z>RVT{xe|)MKfKN{WezDpo z`PTl*K)2Pg*_;J=_xD0lv|YpPQ=Y}G!RZ{tuE_CUl_?&i zE^F?z9p)yp5Yyj{B-yD5oX*436AN|zqP}XL^~Iz(WT{q8M>9y3q+OM|e}r{A#2yoJ zpB1Aa-!*70bjyaJveo!P>Rw$M8-GKnNV5DTJ{el^n3gM1Q=AnfpO<}yfM!GJ25*zq zNqoxw!%BFP3i*)8J;iOoZDDr!1Jk~8g{kTS)3fzM{~e;8IJuo=i?<{{`tz6bzOkiF z)-R+~ zYOt!{K(Pb7m2{!UI|Z(c-RU3fZ=98)4Q3N3 zy6oJ)EQWKHmq8x{6YU%eo_nmL-yc~$(as;-QKF0-j2sDVDT}}BmOd4{&U588duQ*@ zUKzbGy+y;ZlR>77iARl)i0>u7HM`Rk6x`b`pc-^(= zscTXGyeh+U+@t%wu&k@F);F#1t}s}=S+HC)RfzTNgYWbb)Wj|i@3~bXmSdT&(qE_3!ov|?9r8=1Fa-Ke^WcGa*00Ex6`722twbK) z3YyT0=b!sOc!>TYtGM1L(O18HQ(jJFzb5`ZC$oCA0NYQv{ z3-Fd)9DNWpD){sMjFh+zV+kj0vf_1igDLFK+Y0Ft%{Q8%_yeYu?Ng}+1;*y4Zv0g9 zu@fUHmMIA{jyy(p=S#Hj@)&a28<%ag(`V%!usMpdi)UpRU2kp@Dv2&4b8;HR(SnfL z^x}R4MZ+2{dzy{u8e4Tq$WpjoJCh_G^TMYKmcp+a7VBT=3E1)<3{EWy?URmZ?y*Y1 za?~d^GxIg4;7*9nuA*`P#eU%Cm>-9PPgq>h2kKZh&>w`(ySwrV;n;Vog`GA#2Th zQ5Z`8cD`qvhqpA6z3@fIQ5I%VQ-fMl%ZD#Vu*zNakA(#14@y1-{55faHJ0z(T zL<1wWrFobbEN?a=@ufa?XgpOyv>GsG6%Y5eDkw4rOyuojMGx0kULKoA-NhE86|)8V z1uZoZKa*m%8a7xvp@CAK`bWh?lJ9vv^U8cK z1}cU>l6dM&@CUxFaCFjo;5p-4t9FhJ{38_-)6mEzvIm_NMw?D^Fh8>eW8mfx@P+l> zzavKxZjghguCCRSh33)(wc4TN+p`*}@5KER_wuUeTXYTDKl, + # 'location', , + # 'status', , + # } + print("The unpacked input tensors:", input_dict["obs"]) + print() + print("Unbatched repeat dim", input_dict["obs"].unbatch_repeat_dim()) + print() + print("Fully unbatched", input_dict["obs"].unbatch_all()) + print() + return self.model.forward(input_dict, state, seq_lens) + + def value_function(self): + return self.model.value_function() + + +class CustomTFRPGModel(TFModelV2): + """Example of interpreting repeated observations.""" + + def __init__(self, obs_space, action_space, num_outputs, model_config, + name): + super().__init__(obs_space, action_space, num_outputs, model_config, + name) + self.model = TFFCNet(obs_space, action_space, num_outputs, + model_config, name) + self.register_variables(self.model.variables()) + + def forward(self, input_dict, state, seq_lens): + # The unpacked input tensors, where M=MAX_PLAYERS, N=MAX_ITEMS: + # { + # 'items', , + # 'location', , + # 'status', , + # } + print("The unpacked input tensors:", input_dict["obs"]) + print() + print("Unbatched repeat dim", input_dict["obs"].unbatch_repeat_dim()) + print() + if tf.executing_eagerly(): + print("Fully unbatched", input_dict["obs"].unbatch_all()) + print() + return self.model.forward(input_dict, state, seq_lens) + + def value_function(self): + return self.model.value_function() diff --git a/rllib/models/modelv2.py b/rllib/models/modelv2.py index ea77986b9..927baff9c 100644 --- a/rllib/models/modelv2.py +++ b/rllib/models/modelv2.py @@ -1,10 +1,13 @@ from collections import OrderedDict import gym -from ray.rllib.models.preprocessors import get_preprocessor +from ray.rllib.models.preprocessors import get_preprocessor, \ + RepeatedValuesPreprocessor +from ray.rllib.models.repeated_values import RepeatedValues from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.utils.annotations import DeveloperAPI, PublicAPI from ray.rllib.utils.framework import try_import_tf, try_import_torch +from ray.rllib.utils.spaces.repeated import Repeated tf = try_import_tf() torch, _ = try_import_torch() @@ -332,13 +335,17 @@ def _unpack_obs(obs, space, tensorlib=tf): """Unpack a flattened Dict or Tuple observation array/tensor. Arguments: - obs: The flattened observation tensor + obs: The flattened observation tensor, with last dimension equal to + the flat size and any number of batch dimensions. For example, for + Box(4,), the obs may have shape [B, 4], or [B, N, M, 4] in case + the Box was nested under two Repeated spaces. space: The original space prior to flattening tensorlib: The library used to unflatten (reshape) the array/tensor """ if (isinstance(space, gym.spaces.Dict) - or isinstance(space, gym.spaces.Tuple)): + or isinstance(space, gym.spaces.Tuple) + or isinstance(space, Repeated)): if id(space) in _cache: prep = _cache[id(space)] else: @@ -346,32 +353,55 @@ def _unpack_obs(obs, space, tensorlib=tf): # Make an attempt to cache the result, if enough space left. if len(_cache) < 999: _cache[id(space)] = prep - if len(obs.shape) != 2 or obs.shape[1] != prep.shape[0]: + if len(obs.shape) < 2 or obs.shape[-1] != prep.shape[0]: raise ValueError( - "Expected flattened obs shape of [None, {}], got {}".format( + "Expected flattened obs shape of [..., {}], got {}".format( prep.shape[0], obs.shape)) - assert len(prep.preprocessors) == len(space.spaces), \ - (len(prep.preprocessors) == len(space.spaces)) offset = 0 + if tensorlib == tf: + batch_dims = [v.value for v in obs.shape[:-1]] + batch_dims = [-1 if v is None else v for v in batch_dims] + else: + batch_dims = list(obs.shape[:-1]) if isinstance(space, gym.spaces.Tuple): + assert len(prep.preprocessors) == len(space.spaces), \ + (len(prep.preprocessors) == len(space.spaces)) u = [] for p, v in zip(prep.preprocessors, space.spaces): - obs_slice = obs[:, offset:offset + p.size] + obs_slice = obs[..., offset:offset + p.size] offset += p.size u.append( _unpack_obs( - tensorlib.reshape(obs_slice, [-1] + list(p.shape)), + tensorlib.reshape(obs_slice, + batch_dims + list(p.shape)), v, tensorlib=tensorlib)) - else: + elif isinstance(space, gym.spaces.Dict): + assert len(prep.preprocessors) == len(space.spaces), \ + (len(prep.preprocessors) == len(space.spaces)) u = OrderedDict() for p, (k, v) in zip(prep.preprocessors, space.spaces.items()): - obs_slice = obs[:, offset:offset + p.size] + obs_slice = obs[..., offset:offset + p.size] offset += p.size u[k] = _unpack_obs( - tensorlib.reshape(obs_slice, [-1] + list(p.shape)), + tensorlib.reshape(obs_slice, batch_dims + list(p.shape)), v, tensorlib=tensorlib) + elif isinstance(space, Repeated): + assert isinstance(prep, RepeatedValuesPreprocessor), prep + child_size = prep.child_preprocessor.size + # The list lengths are stored in the first slot of the flat obs. + lengths = obs[..., 0] + # [B, ..., 1 + max_len * child_sz] -> [B, ..., max_len, child_sz] + with_repeat_dim = tensorlib.reshape( + obs[..., 1:], batch_dims + [space.max_len, child_size]) + # Retry the unpack, dropping the List container space. + u = _unpack_obs( + with_repeat_dim, space.child_space, tensorlib=tensorlib) + return RepeatedValues( + u, lengths=lengths, max_len=prep._obs_space.max_len) + else: + assert False, space return u else: return obs diff --git a/rllib/models/preprocessors.py b/rllib/models/preprocessors.py index 162a81f56..0b1a30e36 100644 --- a/rllib/models/preprocessors.py +++ b/rllib/models/preprocessors.py @@ -5,6 +5,7 @@ import numpy as np import gym from ray.rllib.utils.annotations import override, PublicAPI +from ray.rllib.utils.spaces.repeated import Repeated ATARI_OBS_SHAPE = (210, 160, 3) ATARI_RAM_OBS_SHAPE = (128, ) @@ -77,7 +78,8 @@ class Preprocessor: # Stash the unwrapped space so that we can unwrap dict and tuple spaces # automatically in model.py if (isinstance(self, TupleFlatteningPreprocessor) - or isinstance(self, DictFlatteningPreprocessor)): + or isinstance(self, DictFlatteningPreprocessor) + or isinstance(self, RepeatedValuesPreprocessor)): obs_space.original_space = self._obs_space return obs_space @@ -243,6 +245,45 @@ class DictFlatteningPreprocessor(Preprocessor): offset += p.size +class RepeatedValuesPreprocessor(Preprocessor): + """Pads and batches the variable-length list value.""" + + @override(Preprocessor) + def _init_shape(self, obs_space, options): + assert isinstance(self._obs_space, Repeated) + child_space = obs_space.child_space + self.child_preprocessor = get_preprocessor(child_space)(child_space, + self._options) + # The first slot encodes the list length. + size = 1 + self.child_preprocessor.size * obs_space.max_len + return (size, ) + + @override(Preprocessor) + def transform(self, observation): + array = np.zeros(self.shape) + if isinstance(observation, list): + for elem in observation: + self.child_preprocessor.check_shape(elem) + else: + pass # ValueError will be raised in write() below. + self.write(observation, array, 0) + return array + + @override(Preprocessor) + def write(self, observation, array, offset): + if not isinstance(observation, list): + raise ValueError("Input for {} must be list type, got {}".format( + self, observation)) + elif len(observation) > self._obs_space.max_len: + raise ValueError("Input {} exceeds max len of space {}".format( + observation, self._obs_space.max_len)) + # The first slot encodes the list length. + array[offset] = len(observation) + for i, elem in enumerate(observation): + offset_i = offset + 1 + i * self.child_preprocessor.size + self.child_preprocessor.write(elem, array, offset_i) + + @PublicAPI def get_preprocessor(space): """Returns an appropriate preprocessor class for the given space.""" @@ -260,6 +301,8 @@ def get_preprocessor(space): preprocessor = TupleFlatteningPreprocessor elif isinstance(space, gym.spaces.Dict): preprocessor = DictFlatteningPreprocessor + elif isinstance(space, Repeated): + preprocessor = RepeatedValuesPreprocessor else: preprocessor = NoPreprocessor diff --git a/rllib/models/repeated_values.py b/rllib/models/repeated_values.py new file mode 100644 index 000000000..b8f885508 --- /dev/null +++ b/rllib/models/repeated_values.py @@ -0,0 +1,171 @@ +from typing import List + +from ray.rllib.utils.annotations import PublicAPI +from ray.rllib.utils.framework import TensorType + + +@PublicAPI +class RepeatedValues: + """Represents a variable-length list of items from spaces.Repeated. + + RepeatedValues are created when you use spaces.Repeated, and are + accessible as part of input_dict["obs"] in ModelV2 forward functions. + + Example: + Suppose the gym space definition was: + Repeated(Repeated(Box(K), N), M) + + Then in the model forward function, input_dict["obs"] is of type: + RepeatedValues(RepeatedValues()) + + The tensor is accessible via: + input_dict["obs"].values.values + + And the actual data lengths via: + # outer repetition, shape [B], range [0, M] + input_dict["obs"].lengths + -and- + # inner repetition, shape [B, M], range [0, N] + input_dict["obs"].values.lengths + + Attributes: + values (Tensor): The padded data tensor of shape [B, max_len, ..., sz], + where B is the batch dimension, max_len is the max length of this + list, followed by any number of sub list max lens, followed by the + actual data size. + lengths (List[int]): Tensor of shape [B, ...] that represents the + number of valid items in each list. When the list is nested within + other lists, there will be extra dimensions for the parent list + max lens. + max_len (int): The max number of items allowed in each list. + + TODO(ekl): support conversion to tf.RaggedTensor. + """ + + def __init__(self, values: TensorType, lengths: List[int], max_len: int): + self.values = values + self.lengths = lengths + self.max_len = max_len + self._unbatched_repr = None + + def unbatch_all(self): + """Unbatch both the repeat and batch dimensions into Python lists. + + This is only supported in PyTorch / TF eager mode. + + This lets you view the data unbatched in its original form, but is + not efficient for processing. + + Examples: + >>> batch = RepeatedValues() + >>> items = batch.unbatch_all() + >>> print(len(items) == B) + True + >>> print(max(len(x) for x in items) <= N) + True + >>> print(items) + ... [[, ..., ], + ... ... + ... [, ], + ... ... + ... [], + ... ... + ... [, ..., ]] + """ + + if self._unbatched_repr is None: + B = _get_batch_dim_helper(self.values) + if B is None: + raise ValueError( + "Cannot call unbatch_all() when batch_dim is unknown. " + "This is probably because you are using TF graph mode.") + else: + B = int(B) + slices = self.unbatch_repeat_dim() + result = [] + for i in range(B): + if hasattr(self.lengths[i], "item"): + dynamic_len = int(self.lengths[i].item()) + else: + dynamic_len = int(self.lengths[i].numpy()) + dynamic_slice = [] + for j in range(dynamic_len): + dynamic_slice.append(_batch_index_helper(slices, i, j)) + result.append(dynamic_slice) + self._unbatched_repr = result + + return self._unbatched_repr + + def unbatch_repeat_dim(self): + """Unbatches the repeat dimension (the one `max_len` in size). + + This removes the repeat dimension. The result will be a Python list of + with length `self.max_len`. Note that the data is still padded. + + Examples: + >>> batch = RepeatedValues() + >>> items = batch.unbatch() + >>> len(items) == batch.max_len + True + >>> print(items) + ... [, ..., ] + """ + return _unbatch_helper(self.values, self.max_len) + + def __repr__(self): + return "RepeatedValues(value={}, lengths={}, max_len={})".format( + repr(self.values), repr(self.lengths), self.max_len) + + def __str__(self): + return repr(self) + + +def _get_batch_dim_helper(v): + """Tries to find the batch dimension size of v, or None.""" + if isinstance(v, dict): + for u in v.values(): + return _get_batch_dim_helper(u) + elif isinstance(v, tuple): + return _get_batch_dim_helper(v[0]) + elif isinstance(v, RepeatedValues): + return _get_batch_dim_helper(v.values) + else: + B = v.shape[0] + if hasattr(B, "value"): + B = B.value # TensorFlow + return B + + +def _unbatch_helper(v, max_len): + """Recursively unpacks the repeat dimension (max_len).""" + if isinstance(v, dict): + return {k: _unbatch_helper(u, max_len) for (k, u) in v.items()} + elif isinstance(v, tuple): + return tuple(_unbatch_helper(u, max_len) for u in v) + elif isinstance(v, RepeatedValues): + unbatched = _unbatch_helper(v.values, max_len) + return [ + RepeatedValues(u, v.lengths[:, i, ...], v.max_len) + for i, u in enumerate(unbatched) + ] + else: + return [v[:, i, ...] for i in range(max_len)] + + +def _batch_index_helper(v, i, j): + """Selects the item at the ith batch index and jth repetition.""" + if isinstance(v, dict): + return {k: _batch_index_helper(u, i, j) for (k, u) in v.items()} + elif isinstance(v, tuple): + return tuple(_batch_index_helper(u, i, j) for u in v) + elif isinstance(v, list): + # This is the output of unbatch_repeat_dim(). Unfortunately we have to + # process it here instead of in unbatch_all(), since it may be buried + # under a dict / tuple. + return _batch_index_helper(v[j], i, j) + elif isinstance(v, RepeatedValues): + unbatched = v.unbatch_all() + # Don't need to select j here; that's already done in unbatch_all. + return unbatched[i] + else: + return v[i, ...] diff --git a/rllib/tests/test_nested_observation_spaces.py b/rllib/tests/test_nested_observation_spaces.py index 18d7a9bda..51eb11fd8 100644 --- a/rllib/tests/test_nested_observation_spaces.py +++ b/rllib/tests/test_nested_observation_spaces.py @@ -1,6 +1,7 @@ from gym import spaces from gym.envs.registration import EnvSpec import gym +import numpy as np import pickle import unittest @@ -19,6 +20,7 @@ from ray.rllib.rollout import rollout from ray.rllib.tests.test_external_env import SimpleServing from ray.tune.registry import register_env from ray.rllib.utils import try_import_tf, try_import_torch +from ray.rllib.utils.spaces.repeated import Repeated tf = try_import_tf() _, nn = try_import_torch() @@ -49,9 +51,23 @@ TUPLE_SPACE = spaces.Tuple([ spaces.Box(low=0, high=1, shape=(10, 10, 3)))), spaces.Discrete(5), ]) - TUPLE_SAMPLES = [TUPLE_SPACE.sample() for _ in range(10)] +# Constraints on the Repeated space. +MAX_PLAYERS = 4 +MAX_ITEMS = 7 +MAX_EFFECTS = 2 +ITEM_SPACE = spaces.Box(-5, 5, shape=(1, )) +EFFECT_SPACE = spaces.Box(9000, 9999, shape=(4, )) +PLAYER_SPACE = spaces.Dict({ + "location": spaces.Box(-100, 100, shape=(2, )), + "items": Repeated(ITEM_SPACE, max_len=MAX_ITEMS), + "effects": Repeated(EFFECT_SPACE, max_len=MAX_EFFECTS), + "status": spaces.Box(-1, 1, shape=(10, )), +}) +REPEATED_SPACE = Repeated(PLAYER_SPACE, max_len=MAX_PLAYERS) +REPEATED_SAMPLES = [REPEATED_SPACE.sample() for _ in range(10)] + def one_hot(i, n): out = [0.0] * n @@ -91,6 +107,22 @@ class NestedTupleEnv(gym.Env): return TUPLE_SAMPLES[self.steps], 1, self.steps >= 5, {} +class RepeatedSpaceEnv(gym.Env): + def __init__(self): + self.action_space = spaces.Discrete(2) + self.observation_space = REPEATED_SPACE + self._spec = EnvSpec("RepeatedSpaceEnv-v0") + self.steps = 0 + + def reset(self): + self.steps = 0 + return REPEATED_SAMPLES[0] + + def step(self, action): + self.steps += 1 + return REPEATED_SAMPLES[self.steps], 1, self.steps >= 5, {} + + class NestedMultiAgentEnv(MultiAgentEnv): def __init__(self): self.steps = 0 @@ -158,6 +190,45 @@ class TorchSpyModel(TorchModelV2, nn.Module): return self.fc.value_function() +class TorchRepeatedSpyModel(TorchModelV2, nn.Module): + capture_index = 0 + + def __init__(self, obs_space, action_space, num_outputs, model_config, + name): + TorchModelV2.__init__(self, obs_space, action_space, num_outputs, + model_config, name) + nn.Module.__init__(self) + self.fc = FullyConnectedNetwork( + obs_space.original_space.child_space["location"], action_space, + num_outputs, model_config, name) + + def forward(self, input_dict, state, seq_lens): + ray.experimental.internal_kv._internal_kv_put( + "torch_rspy_in_{}".format(TorchRepeatedSpyModel.capture_index), + pickle.dumps(input_dict["obs"].unbatch_all()), + overwrite=True) + TorchRepeatedSpyModel.capture_index += 1 + return self.fc({ + "obs": input_dict["obs"].values["location"][:, 0] + }, state, seq_lens) + + def value_function(self): + return self.fc.value_function() + + +def to_list(value): + if isinstance(value, list): + return [to_list(x) for x in value] + elif isinstance(value, dict): + return {k: to_list(v) for k, v in value.items()} + elif isinstance(value, np.ndarray): + return value.tolist() + elif isinstance(value, int): + return value + else: + return value.numpy().tolist() + + class DictSpyModel(TFModelV2): capture_index = 0 @@ -435,6 +506,31 @@ class NestedSpacesTest(unittest.TestCase): self.assertEqual(seen[1][0].tolist(), cam_i) self.assertEqual(seen[2][0].tolist(), task_i) + # TODO(ekl) should probably also add a test for TF/eager + def test_torch_repeated(self): + ModelCatalog.register_custom_model("r1", TorchRepeatedSpyModel) + register_env("repeat", lambda _: RepeatedSpaceEnv()) + a2c = A2CTrainer( + env="repeat", + config={ + "num_workers": 0, + "rollout_fragment_length": 5, + "train_batch_size": 5, + "model": { + "custom_model": "r1", + }, + "framework": "torch", + }) + + a2c.train() + + # Check that the model sees the correct reconstructed observations + for i in range(4): + seen = pickle.loads( + ray.experimental.internal_kv._internal_kv_get( + "torch_rspy_in_{}".format(i))) + self.assertEqual(to_list(seen), [to_list(REPEATED_SAMPLES[i])]) + if __name__ == "__main__": import pytest diff --git a/rllib/utils/spaces/repeated.py b/rllib/utils/spaces/repeated.py new file mode 100644 index 000000000..4ba367ef3 --- /dev/null +++ b/rllib/utils/spaces/repeated.py @@ -0,0 +1,37 @@ +import numpy as np +import gym + +from ray.rllib.utils.annotations import PublicAPI + + +@PublicAPI +class Repeated(gym.Space): + """Represents a variable-length list of child spaces. + + Example: + self.observation_space = spaces.Repeated(spaces.Box(4,), max_len=10) + --> from 0 to 10 boxes of shape (4,) + + See also: documentation for rllib.models.RepeatedValues, which shows how + the lists are represented as batched input for ModelV2 classes. + """ + + def __init__(self, child_space: gym.Space, max_len: int): + self.np_random = np.random.RandomState() + self.child_space = child_space + self.max_len = max_len + super().__init__() + + def seed(self, seed=None): + self.np_random = np.random.RandomState() + self.np_random.seed(seed) + + def sample(self): + return [ + self.child_space.sample() + for _ in range(self.np_random.randint(1, self.max_len + 1)) + ] + + def contains(self, x): + return (isinstance(x, list) and len(x) <= self.max_len + and all(self.child_space.contains(c) for c in x)) diff --git a/rllib/utils/spaces/simplex.py b/rllib/utils/spaces/simplex.py index 44ed9adcf..94daed091 100644 --- a/rllib/utils/spaces/simplex.py +++ b/rllib/utils/spaces/simplex.py @@ -1,7 +1,10 @@ import numpy as np import gym +from ray.rllib.utils.annotations import PublicAPI + +@PublicAPI class Simplex(gym.Space): """Represents a d - 1 dimensional Simplex in R^d. @@ -32,7 +35,7 @@ class Simplex(gym.Space): super().__init__(shape, dtype) self.np_random = np.random.RandomState() - def seed(self, seed): + def seed(self, seed=None): self.np_random.seed(seed) def sample(self):