From fa294cd53298d4d19f16591a858a9e3f255c7970 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Mon, 7 May 2018 21:24:52 -0600 Subject: [PATCH 1/4] Fix import --- deep_rl/network/network_utils.py | 1 + deep_rl/utils/plot.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deep_rl/network/network_utils.py b/deep_rl/network/network_utils.py index 79d6fb2..3a54b7b 100644 --- a/deep_rl/network/network_utils.py +++ b/deep_rl/network/network_utils.py @@ -7,6 +7,7 @@ import torch import torch.nn as nn import torch.nn.functional as F +import numpy as np class BaseNet: def set_gpu(self, gpu): diff --git a/deep_rl/utils/plot.py b/deep_rl/utils/plot.py index 0869174..690c0c0 100644 --- a/deep_rl/utils/plot.py +++ b/deep_rl/utils/plot.py @@ -1,7 +1,7 @@ # Adapted from https://github.com/openai/baselines/blob/master/baselines/results_plotter.py +from ..component.bench import load_monitor_log import numpy as np -from ..component import * import os import re @@ -44,7 +44,7 @@ class Plotter: def load_results(self, dirs, max_timesteps=1e8, x_axis=X_TIMESTEPS, episode_window=100): tslist = [] for dir in dirs: - ts = component.load_monitor_log(dir) + ts = load_monitor_log(dir) ts = ts[ts.l.cumsum() <= max_timesteps] tslist.append(ts) xy_list = [self.ts2xy(ts, x_axis) for ts in tslist] From 06876d1269b0bd45a6c166a3db0af8abf0bbaafd Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Mon, 7 May 2018 22:01:21 -0600 Subject: [PATCH 2/4] Add Option-Critic --- README.md | 18 +++ deep_rl/agent/OptionCritic_agent.py | 115 ++++++++++++++++++ deep_rl/agent/__init__.py | 1 + deep_rl/network/network_heads.py | 20 +++ deep_rl/utils/config.py | 1 + examples.py | 46 ++++++- ...ption_critic_pixel_atari-180417-092617.png | Bin 0 -> 25879 bytes 7 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 deep_rl/agent/OptionCritic_agent.py create mode 100644 images/option_critic_pixel_atari-180417-092617.png diff --git a/README.md b/README.md index 312222f..94415bc 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Implemented algorithms: * Synchronous N-Step Q-Learning * Deep Deterministic Policy Gradient (DDPG) * (Continuous/Discrete) Synchronous Proximal Policy Optimization (PPO) +* The Option-Critic Architecture (OC) * Action Conditional Video Prediction Asynchronous algorithms below are removed in current version but can be found in [v0.1](https://github.com/ShangtongZhang/DeepRL/releases/tag/v0.1). @@ -46,6 +47,10 @@ Support for PyTorch v0.3.x can be found in [v0.2](https://github.com/ShangtongZh ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/ppo_continuous-180408-002056.png) ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/ppo_pixel_atari-180410-235529.png) +## OC +![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/option_critic_pixel_atari-180417-092617.png) +This is my synchronous option-critic implementation, not the original one. + ## Action Conditional Video Prediction ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/ACVP.png) @@ -64,6 +69,18 @@ Prediction is sampled after 110K iterations, and I only implemented one-step tra ```examples.py``` contains examples for all the implemented algorithms +Please use this bibtex if you want to cite this repo +``` +@misc{deeprl, + author = {Shangtong, Zhang}, + title = {Modularized Implementation of Deep RL Algorithms in PyTorch}, + year = {2018}, + publisher = {GitHub}, + journal = {GitHub Repository}, + howpublished = {\url{https://github.com/ShangtongZhang/DeepRL}}, +} +``` + # References * [Human Level Control through Deep Reinforcement Learning](https://www.nature.com/nature/journal/v518/n7540/full/nature14236.html) * [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/abs/1602.01783) @@ -81,4 +98,5 @@ Prediction is sampled after 110K iterations, and I only implemented one-step tra * [Action-Conditional Video Prediction using Deep Networks in Atari Games](https://arxiv.org/abs/1507.08750) * [A Distributional Perspective on Reinforcement Learning](https://arxiv.org/abs/1707.06887) * [Distributional Reinforcement Learning with Quantile Regression](https://arxiv.org/abs/1710.10044) +* [The Option-Critic Architecture](https://arxiv.org/abs/1609.05140) * Some hyper-parameters are from [DeepMind Control Suite](https://arxiv.org/abs/1801.00690), [OpenAI Baselines](https://github.com/openai/baselines) and [Ilya Kostrikov](https://github.com/ikostrikov/pytorch-a2c-ppo-acktr) \ No newline at end of file diff --git a/deep_rl/agent/OptionCritic_agent.py b/deep_rl/agent/OptionCritic_agent.py new file mode 100644 index 0000000..a0602e7 --- /dev/null +++ b/deep_rl/agent/OptionCritic_agent.py @@ -0,0 +1,115 @@ +####################################################################### +# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) # +# Permission given to modify the code as long as you keep this # +# declaration at the top # +####################################################################### + +from ..network import * +from .BaseAgent import * + +class OptionCriticAgent(BaseAgent): + def __init__(self, config): + BaseAgent.__init__(self, config) + self.config = config + self.task = config.task_fn() + self.network = config.network_fn(self.task.state_dim, self.task.action_dim) + self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim) + self.optimizer = config.optimizer_fn(self.network.parameters()) + self.target_network.load_state_dict(self.network.state_dict()) + self.policy = config.policy_fn() + + self.episode_rewards = np.zeros(config.num_workers) + self.last_episode_rewards = np.zeros(config.num_workers) + + self.total_steps = 0 + states = self.config.state_normalizer(self.task.reset()) + self.q_options, self.betas, self.log_pi = self.network.predict(states) + self.options = np.asarray([self.policy.sample(q) for q in self.q_options.detach().cpu().numpy()]) + self.is_initial_betas = np.ones(self.config.num_workers) + self.prev_options = np.copy(self.options) + + def iteration(self): + config = self.config + rollout = [] + + q_options, betas, options, log_pi = self.q_options, self.betas, self.options, self.log_pi + for _ in range(config.rollout_length): + var_options = self.network.tensor(options).long() + worker_index = self.network.tensor(np.arange(config.num_workers)).long() + intra_log_pi = log_pi[worker_index, var_options, :] + dist = torch.distributions.Categorical(intra_log_pi.exp()) + actions = dist.sample() + next_states, rewards, terminals, _ = self.task.step(actions.cpu().detach().numpy().flatten()) + next_states = config.state_normalizer(next_states) + self.episode_rewards += rewards + rewards = config.reward_normalizer(rewards) + q_options_next, betas_next, log_pi_next = self.network.predict(next_states) + rollout.append([q_options, betas, options, self.prev_options, rewards, 1 - terminals, np.copy(self.is_initial_betas), intra_log_pi, actions]) + self.is_initial_betas = np.asarray(terminals, dtype=np.float32) + + np_q_options_next = q_options_next.cpu().detach().numpy() + np_betas_next = betas_next.gather(1, var_options.unsqueeze(1)).cpu().detach().numpy().flatten() + options_next = np.copy(options) + dice = np.random.rand(len(options_next)) + for j in range(len(dice)): + if dice[j] < np_betas_next[j]: + options_next[j] = self.policy.sample(np_q_options_next[j]) + for i, terminal in enumerate(terminals): + if terminals[i]: + self.last_episode_rewards[i] = self.episode_rewards[i] + self.episode_rewards[i] = 0 + self.prev_options = options + options = options_next + q_options = q_options_next + betas = betas_next + log_pi = log_pi_next + + self.policy.update_epsilon() + self.total_steps += config.num_workers + if self.total_steps / config.num_workers % config.target_network_update_freq == 0: + self.target_network.load_state_dict(self.network.state_dict()) + + self.options = options + self.q_options = q_options + self.betas = betas + self.log_pi = log_pi + + target_q_options, _, _ = self.target_network.predict(next_states) + prev_options = self.network.tensor(self.prev_options).long().unsqueeze(1) + betas_prev_options = betas.gather(1, prev_options) + + returns = (1 - betas_prev_options) * target_q_options.gather(1, prev_options) +\ + betas_prev_options * torch.max(target_q_options, dim=1, keepdim=True)[0] + returns = returns.detach() + + processed_rollout = [None] * (len(rollout)) + for i in reversed(range(len(rollout))): + q_options, betas, options, prev_options, rewards, terminals, is_initial_betas, log_pi, actions = rollout[i] + options = self.network.tensor(options).unsqueeze(1).long() + prev_options = self.network.tensor(prev_options).unsqueeze(1).long() + terminals = self.network.tensor(terminals).unsqueeze(1) + rewards = self.network.tensor(rewards).unsqueeze(1) + is_initial_betas = self.network.tensor(is_initial_betas).unsqueeze(1) + returns = rewards + config.discount * terminals * returns + + q_omg = q_options.gather(1, options) + log_action_prob = log_pi.gather(1, actions.unsqueeze(1)) + entropy_loss = (log_pi.exp() * log_pi).sum(-1).unsqueeze(1) + + q_prev_omg = q_options.gather(1, prev_options) + v_prev_omg = torch.max(q_options, dim=1, keepdim=True)[0] + advantage_omg = q_prev_omg - v_prev_omg + advantage_omg.add_(config.termination_regularizer) + betas = betas.gather(1, prev_options) + betas = betas * (1 - is_initial_betas) + processed_rollout[i] = [q_omg, returns, betas, advantage_omg.detach(), log_action_prob, entropy_loss] + + q_omg, returns, beta_omg, advantage_omg, log_action_prob, entropy_loss = map(lambda x: torch.cat(x, dim=0), zip(*processed_rollout)) + pi_loss = -log_action_prob * (returns - q_omg.detach()) + config.entropy_weight * entropy_loss + pi_loss = pi_loss.mean() + q_loss = 0.5 * (q_omg - returns).pow(2).mean() + beta_loss = (advantage_omg * beta_omg).mean() + self.optimizer.zero_grad() + (pi_loss + q_loss + beta_loss).backward() + nn.utils.clip_grad_norm_(self.network.parameters(), config.gradient_clip) + self.optimizer.step() \ No newline at end of file diff --git a/deep_rl/agent/__init__.py b/deep_rl/agent/__init__.py index 15f8b8f..2175789 100644 --- a/deep_rl/agent/__init__.py +++ b/deep_rl/agent/__init__.py @@ -5,3 +5,4 @@ from .CategoricalDQN_agent import * from .NStepDQN_agent import * from .QuantileRegressionDQN_agent import * from .PPO_agent import * +from .OptionCritic_agent import * \ No newline at end of file diff --git a/deep_rl/network/network_heads.py b/deep_rl/network/network_heads.py index 9e509e6..33e9f42 100644 --- a/deep_rl/network/network_heads.py +++ b/deep_rl/network/network_heads.py @@ -89,6 +89,26 @@ class QuantileNet(nn.Module, BaseNet): quantiles = quantiles.cpu().detach().numpy() return quantiles +class OptionCriticNet(nn.Module, BaseNet): + def __init__(self, body, action_dim, num_options, gpu=-1): + super(OptionCriticNet, self).__init__() + self.fc_q = layer_init(nn.Linear(body.feature_dim, num_options)) + self.fc_pi = layer_init(nn.Linear(body.feature_dim, num_options * action_dim)) + self.fc_beta = layer_init(nn.Linear(body.feature_dim, num_options)) + self.num_options = num_options + self.action_dim = action_dim + self.body = body + self.set_gpu(gpu) + + def predict(self, x): + phi = self.body(self.tensor(x)) + q = self.fc_q(phi) + beta = F.sigmoid(self.fc_beta(phi)) + pi = self.fc_pi(phi) + pi = pi.view(-1, self.num_options, self.action_dim) + log_pi = F.log_softmax(pi, dim=-1) + return q, beta, log_pi + class GaussianActorNet(nn.Module, BaseNet): def __init__(self, action_dim, body, gpu=-1): super(GaussianActorNet, self).__init__() diff --git a/deep_rl/utils/config.py b/deep_rl/utils/config.py index 4e2feef..590a576 100644 --- a/deep_rl/utils/config.py +++ b/deep_rl/utils/config.py @@ -60,6 +60,7 @@ class Config: self.test_interval = 0 self.test_repetitions = 10 self.evaluation_env = None + self.termination_regularizer = 0 def add_argument(self, *args, **kwargs): self.parser.add_argument(*args, **kwargs) diff --git a/examples.py b/examples.py index 9870b6a..81a5ca8 100644 --- a/examples.py +++ b/examples.py @@ -117,6 +117,24 @@ def ppo_cart_pole(): config.iteration_log_interval = 1 run_iterations(PPOAgent(config)) +def option_critic_cart_pole(): + config = Config() + game = 'CartPole-v0' + task_fn = lambda log_dir: ClassicalControl(game, max_steps=200, log_dir=log_dir) + config.num_workers = 5 + config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers) + config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001) + config.network_fn = lambda state_dim, action_dim: OptionCriticNet( + FCBody(state_dim), action_dim, num_options=2) + config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1) + config.discount = 0.99 + config.target_network_update_freq = 200 + config.rollout_length = 5 + config.termination_regularizer = 0.01 + config.entropy_weight = 0.01 + config.logger = Logger('./log', logger) + run_iterations(OptionCriticAgent(config)) + ## Atari games def dqn_pixel_atari(name): @@ -247,6 +265,28 @@ def ppo_pixel_atari(name): config.iteration_log_interval = 1 run_iterations(PPOAgent(config)) +def option_ciritc_pixel_atari(name): + config = Config() + config.history_length = 4 + task_fn = lambda log_dir: PixelAtari(name, frame_skip=4, history_length=config.history_length, log_dir=log_dir) + config.num_workers = 16 + config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, + log_dir=get_default_log_dir(config.tag)) + config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=1e-4, alpha=0.99, eps=1e-5) + config.network_fn = lambda state_dim, action_dim: OptionCriticNet(NatureConvBody(), action_dim, num_options=4, gpu=0) + config.policy_fn = lambda: GreedyPolicy(epsilon=0.1, final_step=1000000, min_epsilon=0.1) + config.state_normalizer = ImageNormalizer() + config.reward_normalizer = SignNormalizer() + config.discount = 0.99 + config.target_network_update_freq = 10000 + config.rollout_length = 5 + config.gradient_clip = 5 + config.max_steps = 1e8 + config.entropy_weight = 0.01 + config.termination_regularizer = 0.01 + config.logger = Logger('./log', logger) + run_iterations(OptionCriticAgent(config)) + def dqn_ram_atari(name): config = Config() config.task_fn = lambda: RamAtari(name, no_op=30, frame_skip=4, @@ -331,7 +371,7 @@ def ddpg_continuous(): def plot(): import matplotlib.pyplot as plt plotter = Plotter() - names = plotter.load_log_dirs('') + names = plotter.load_log_dirs(pattern='.*') data = plotter.load_results(names) for i, name in enumerate(names): @@ -371,6 +411,7 @@ if __name__ == '__main__': # quantile_regression_dqn_cart_pole() # n_step_dqn_cart_pole() # ppo_cart_pole() + # option_critic_cart_pole() # dqn_pixel_atari('BreakoutNoFrameskip-v4') # a2c_pixel_atari('BreakoutNoFrameskip-v4') @@ -378,6 +419,7 @@ if __name__ == '__main__': # quantile_regression_dqn_pixel_atari('BreakoutNoFrameskip-v4') # n_step_dqn_pixel_atari('BreakoutNoFrameskip-v4') # ppo_pixel_atari('BreakoutNoFrameskip-v4') + # option_ciritc_pixel_atari('BreakoutNoFrameskip-v4') # dqn_ram_atari('Breakout-ramNoFrameskip-v4') # ddpg_continuous() @@ -385,5 +427,5 @@ if __name__ == '__main__': # action_conditional_video_prediction() - # plot() + plot() diff --git a/images/option_critic_pixel_atari-180417-092617.png b/images/option_critic_pixel_atari-180417-092617.png new file mode 100644 index 0000000000000000000000000000000000000000..f2e193c6c409f986a6ccbff30cecdd4b958b65b0 GIT binary patch literal 25879 zcmd?RbyQaE*Dm_dB1m_GAPv$XrGm5wl2XzQ0us^|9nyjbilmgFbSjNf(wz#3boZId z@B8h2#yyCNPdCfVmYlYprt9a!y&1Dn{bwyc8UJHdn4?>~P zW-npGUqt#R7vTq{`z>YdOYqO{l4U6Tj{8W-z#WAmG(-MJ%ahHugFi|p$|b}r6Lf_(5F50mZV$B!iW`Tyqyd@gP_{J|N6cqkMT zN?HDpad5plX$VRUOJ)7dM`M32G0`A3iN@YOMa#OlTIWl( zo_*@?xCcq+lUhGSF!mhyszQQ8bDrh>JveB$OS*v<@nbT@OlD&wVx<0H@`e|~9jEjj z=_Ls+mA`H0Y_E|{H6^Zt1K)mec7s%Sn!ig0FAZmz18=mo7q~O0q7VQ{iPO) zj#D*5!*iVLO;0giOFa=>pRBfcT&!F4;9kBqm4sVXf3>vlzFfAJ+1c?PTYyHlgZf$B z<~KaTJ>OTqrfNL!61jAqsbqi4QGeq|PY~Pe{uy=zQ|+xJdAY+#FspJ^z?ozcuVHX* zy3ge|VlJK^)2lFW2p)DNa8ijsdN($m&!(EhBq~aw^HIBW**AtpEo zH|9bEPS(Wow7CpJo`#XKvv6~NA2uUOq*g25rG&r1(;OFt#@LU_88?(va-QSC2W#;C_o1UJg zh-#i5ZJsGvXpgbzjAMH(Y)5iUJ=JpcQD_&Y4EnXQ+rPdihg4RIO-{C4d@yeCqvlGy z%ounoAan9pXt2nTGCe(A+~W@!is{A;TE>7w=08IPI2sxn+9f9R8x1FXWuHG=9T?X< z=9KjK^P;Iq&fCX_g_RW(kAjCoSM7T;f5@9Rv~5wexlXlS8w}F{Cr-#0;hwgmpO|IN z4=t49vg+@Uo|IXAk6h2!%D?_(+Y;5Z*co4bP;6XVu`lo!hF3l^@=*t`q;C6i5|6>N zgoFeZ3mKnX4xim+w7|f?!{g&j_oW{3$IJIh159tj(%#OH@&Dwps{7*Q%dg+RwIfO_q4@GvGFB5yyq}^2LD~n)AN%JRFKE&@O$O>Yn$8K zx4tLwx}F}oe(+6`^tjyL-w#eE2(EolZvWCW@Jz>3-PyT#f8%*_G9{PJ#}}~lW^luo z=h3*hxOw`eERtU9ktYWm7}psvs@#|Et}X22l3rut;JEawD-jvtX0Pl8*^z|j#x(sE z8sY0tiggNfZ3l9qG*9w0KWt9A_cBXK)peYZYHoePdDTIXm7BZpyZg;6R$MzfyTk48 z{IMx1VacWe2A)$tvsGgg6JLr=dt<=L!Z77--bC{`Sda2Q*`JFNo0eN>i*lWB!EI@6 zo%=iXxok@dwnxs^mIJQ2A|@v0I9*5W(r|R`^5x4!q@)(24XqKB7Neicyf^9(uKOQ9 zMrGm@dTmUP*87TK6EZ}?C@dx_oT$W{>CTUq7~!so%R$SNZq|;Mky} zw2O^kbSE2)F<>dh9VSij>>8QIwVs>c|7tfH@VvadcpiMYL@)VR&d*PZA@J07Y!9B( z+Wkh%$_Wh(?U8H$S>Sg5{oKCBz(9OjQMD5TjHc70o&LHf?rTfO0~)es%YE-*MQ(=> zBu)u&a^hlO<2QXWYrg)l3H2jI2nUXKOWl*5>#p3}iaja&)f21srlrY}SkSX%V z^o2D6H~Q{Q6wT}FX6Sx?exIznX?EAAXlccqAt4s9#$YspCu8X5qt6NoxXmprT-U}${9uWe$IH1% zNJ;Vhof-qqHpd-G=9c?Ec>gVJ|FTBOZ_<(xcy70QV7>QeNZf7ll5Vk4SXNfnEen=R zl2@S})=7NETxi#iw{~|EU#*A-sLq)gQx#R!_$iClaPqi_h=^TS=I`T_d`28- zul6@)N}Or~FB&xQV`F2r-b#8{I*B+8CuC-3vMR@Af!!Ht>xw$+Z%f5OmNIQmmi$H#=b(s`sB$I zWlm1cI9BDK>r=ICsGl_+t6xSARt6NeH#cAL8CAV=QdL%tJv}|mV$bl?n#g%ll+So3 z{!zDx3+nVqiDs3D_#Z%t7yocebH1qK0ess&mXdf=*cM~ zV**#jgMIVXR@^Sy|3OxgZ&T}Zji9cblt6)gZ;IM zxR8(#;eb~&UTuC*u#Wyb&8w4jPwt-##_8k-k=d~|Xd|6&)rx31=Yj@pC+`0ufE%^~Um#o*)z7)J6mZ+n{Tr0A8HOj(|Q z5u=65+Mq(Cq7)n*3p>Ss{`^Si&N&~cpj*F0W%W*TsWHBsrJ2>9vp1xn>1fTPy}7Qy>f z&Qmqx44t_d0W<&t^lLqNk+lKi`!+PBBZ_zOoKQv%uo(dF_v&0C(rI8yRenc;_QQpC zlx+5)bi~L)B_=21$n9p44P$6kIvXRa3M<05`~4AkfsUS@HgK&xt1o{B$`_VsWvhVs zOC#_UV#VKyN)27#66@~N(Q0@0_33)RZfgRm~ z=x))Usc?r>l1r}`AG~3FJa$i-1O_D~nt`x<6z5>>L-)VB`~ z6th(lKY4AKjCt%V^@{n(s%JXbvk~3Db~i0_a?%K5zvA%vpFiQS=X0>*afz|j0Mr5X zzXT_9cydCyDjjbA$GzVFL^O%t)H(LVL2A3>YG`O^(=#l*FVbem&XZN|r$#Q^(dEDC zz`}kglS9p}tc0n*5q+-^L9smcnVp1$B%Hi=``34?7;nUTefc81mf;J#*ALmqS}&=F z6tk~C#QR6Ty2xSHnCu$q^6;zWl1)!q7=)0nfE=|HbQA(SM4cz~e>+?!F(CB?`%Yko*bf92;j!1nHfg#x`HR;Xc*WJVK1me9jO31M!_Dk$jUOpO}~zpJCr(2 zh<&4&+}V7smN&jD&QL$vFEe;Y^s&Yk(nnSkew zjk3tqh!;3Jm}v*py|w%y#{6V|?GO(8b)Q{ZrxCLWe*ZQ)QN$bv!SM!W(MT<=v zqY?jpytk^PriO_Uw(Ua!qEyh=r{dDi3z9uQ#CRj>^bPQp`t(S9J5J;IF}I}W-%yCQ zS7l@vn3IvEli?p6e1+D0bM|ZNu`wi=VQ#;==H7ZuKZVrg zcz=!ihRa8Gt$~Sy!-W{`{n3oPA7v|amMrWxAMqoTlamJljM{yDhNX4$=2KSX8Z9os zG$Kzj($eVcz|(2j+U6~;tZ?e&-Me`i4=*0}oFzLq*P-O(XelEb+<`JUBQ5YdSrCU+ zX97-&G&@Pp;<#x-2pNZy?)4`~SNk3m3tRu7%2!F`imR=y)rOdA2U+ZRmB{)`BmWf! zDSfSip*RR*yQe$7s$e#EtR4c)gN*f>4un%TSJ&89E3Rt*{9t6t5J3Lxa{#2-u=<@N zBYN9gTd}Z69Q5Lk#|$}*IzXu8FMx;EMDFL_ zO$1oMmdI~vvc12bL?hOansOm!;2TWsn6u z4=3mTHZnqqK)BCaaq;oQfUM*Eq<_TRuqUGiFjIX}=(r5D($SAf=%R z2M1;!Gd20Ru?wQkCcs&9IE(iRhNuM0F~OK3z<4o;xQXNy6qv=usc;DBnji#ic5#<9 zO-<1=`tQchH{u~Bo;W%;icQFaOYH`((cSR;v25hygG7T!SW&xVa!L2!_^^7B;7K!K zmqu$mxDm@uNFa0ibp&qQ9D-(*>%yH<%G>NPTQ_-vqf1mo--&ecwXi(?4BosrKZ{LE z!$&D8Dak<^>zJPf`0_pcFYdj`gqnq#gYEd2e8;i(U?P^5ma{ODu5%Wo26frJL$N7F z_FNn(d%(Gw2g7(RXmuHWgurRT%g@gTSRi+_{mZHZ_wwai_V!#TCJ_;xq*J>dhn|%9 zw{Lk3Rr24xLm?m%l9y^G37GD4$l@UNQ-Hj2c7FbozghPDE8KV1Man4K;X-O8H5@E7 zAaxiiF8?$BO30>XuvkGE1{1voX+oZEQEE~1C)^M<)D^HS2FRwHj&_!i&0kzxoCP~5 zGih2KDFL@ag(9M(iv%xm+mo=H$E;fO7NrNx(0T^x@qJVVKv!kwS_E@DPKDf&v>n-QV$YJTMVd5P&RJcJ`}MQgkRH28L*A zA!{aH-YWohQ9+OrfctLj!|IckmY!Q#u^U|9*yu&fuj8xu&L%I6d*;a!JvdD~$kOCh zR4&V$Zr-k{shMx3G=1@)s10^Q!1J#@CMG5l2Cgbb%X@o^1Mr4_-=ffr|#09^0L%VRZ7sEwAWFWE}>)& zYLNFgFf_9CIuw2Dbq zH$P}~Br)PUwckbks7bZ?gm{OWIPMErqHn#j7|p?9{%6y5v|^DZPLrcmy!?cZ7AmK2 zS9^RPCVrJ&iUXEG_pSd!&V=m6WYyZf(?Uh}8(!KNA&G!V$w$3QC=;dMu&%ZIk!tPGBsy6yjpYvb4uFWdsp^-50Wx&_>a$< zao&jw8Zp7ZzEZaKB8h_6@nG6g0?Lk|9mUEdeV^H{dweWj`=e6|IkF?S-tb#)y}OmB z)q?j*GuID`Dhnxd%cp9vc#JRe7-DB-aU3+{OgkL8&1M89L_Fn*L2A@vZzn3B>J$X| z9t7oa%#N5ooiv&{60&L0a&iA1x*+Kx?{;vyBd7mbiDGpKW7=0k#$mKSF3Ei<$R-0z z)ERTQc&OaQhI{ASz(8*3W7`@A4tjSgCn<&j&eCo07E>w0xQBnZA1>c}coDF$V=wZ_ zTi}y_t@!={j?9Ug*ZFh8#|rQF7?X!TT9v=z<>DtioVf@Jy)^q3W44{~7kKzWMhN?8 z7Hu69M#fK1{YnE@lMWrPtn$WLJS6-#`jQ5c3E0m_{Dqydv8K(r8Vgz3odd0Wd1edL zXc?B0Mx{|&(*D7WT!C&-ZS8WW-)7UUN8mZ9k77~h^&<=@BQCb{@d1<@j(T4B99a>G zG(D48M#>}M+Zk(Bt_RRJ3+i za{h3f@6+CEs9RahRsVdK&QcoL*-JtCzg4qIWGi@tpn}wCdVetMQZr`%poNuTJnKSg z@J1tl#TDArfbK}@;iJzrPwodyHq;;Db8BDsKl?kD6!Ww&0+|LHO44(1&Fc?q1t;fx z>Eh8&MOc27&mPw?&DpKKl$7C!UuI6@mXkT0cBFKUJ|fZ0Ay2;%yj{Vj z+ESs#W;_vlb=yI0$ddJ>75rdFr?fXy(*4>;H%BULVwQ0WACMnz-qsA!z&+VdaW2tO zHJRzXrx_E3c4VP)7ALs>ZF6fQl6rf=y1PYO>b7b!`b>aBwRpgValq-(lHA#``Aj>` z;UcLf_B9^0fT>X8q==`5h*ieMM?_)WiQzj<&0@7OKRR|;sHk>Va7#$BOJhYdw+ip* z$lRbOVd-i`btPzWayEximX@}SESChAIlXb7ncVhD3bh#IINBO5Wm(bAUkr4a4Cmkx zb4h<)XW?uu#EGm=Z2ZYD%7u#7h^UD|kp=CH++632L)R_aowr_(!!(paw?4GLz&u*L z{DJypq~3SMqFnf_{b%5*Sxtk9TI47JHH+z7%L7+3Ha-IHBdb!=`kJFvR^?ryVtr$R1<02s)HB@^wWn*Lrpzm)iRLw(b#b{P`k4U`Zi@Vcnn4RQK9@UR-s~ z#LjpHUb*|}0W?*QabL^MRPA;%1}4Y^KSQ6*PJZ0(Qm47=kSa<(u{zRi`a!n9dsgmY zGj=Q)-vdl}MefQn+?DHyvnh(8^iU0?9{zscd~)*b4ReQ&(`k3IRm`m-H9vqG@jX09 zmDrgxb-0qcIP$$ykwDKJD!#9j^j8K8 zaq(%d#eMztU6q18kpb~n?38qc6D7uml`}OWk7k3`^}{gl4cxaZhKJ2Gn7iH5T81hJRzUc<{X?ET~fKlXaB1rx*XZ!}Dj66T>Aq>ArU(y&}U& zEY`27b4{P^CVL1SP>tVP9j@_rk9P=H8SL&pzqX3>>=xx@PVlKD2L%^KwC=Zxmqk#v zmR2icrM_qx$9)*k;GE>5DIJt;rr#hRcsiDUf7sX59shg%>pz`B?VFlUd}@@=hf63s zi-H4O49#}qtIy9O4pOnNJU;nteMqtArPvtRi^CW?9&c%koOrqytl;88scUi=@)HNU z61Nw+64gzU)A=BrL~K16j*H8j;CH-ZXGdP)_uxV5-gIRZl6IFnJ8ze!9=B%hCD*wN z>^L>LvGwH<-}?~kjekDnmFYj*G%tOQ)pEg9RTB2UiSC$xY+DqB{ToRFfml{UjnoWJ@;rfP^ z=4Zc=63Y6Zbyei zyCNgb3?j`3sU-YoClNKlyoNNU3%}TsW_@4FhGhPsIX+Hx9*G=GSZ>yNs_$a9wJejU?4v&*L&RdbmQ9a9cpTGzgdfzPKR|nlBW6MEc2f2Uu~;ou(>|kY|5;jsJ@~%;&+va zEpYt(TVF7GdBr9uDn>NAgb+rojucshQ8^$xY`KI@NWm>jC&}v42ot+!T!ms~$KbDH z(%?YN{xuLN(E99`nr~juU(ZT{<8E%=a)uT=O1*!$GLnh0XhBXZCK_n!PsxuCrMmqB z9qwrdm4yiYmZi*IZI}DXNeV(=Jh-nrBpvNkx?mXBEJxqI#c*E#qo2qBU~j~ngzriB&v2NAx1@cv|0au~IZo=UjrU#nqqYvL7Vh*<;E)A6nhoTzKax3Yd z^X^$0u^u>BjL%%kGLi0gsIZPY#XJ0p(Z3+%U&FBxo8eLy&G(MB$&{+Q%;M;%jZFcc zk^fJOj+%O6#tkn5{W5vh5@u$(M6S#Ol|Y>cu^Ar8^q(K~HBIcZY&3`!Z$9;V*4Df$ z^H$=jPt~piK~EZI++F{xcb4}v{C<{MTo)0J(YlhI6?$u&CW#}cNQ=7U2MGgThPZaz zadfIdN4D~}&N`Cedyp~uCMRKIVu!ox9Sc3cArLl+!{abP&od>l{G;9fA?tkx%A}Dm zqw~pZ(6Wd9v=Lnl9j@cVHI=t%GES#59)GBM?^fhQP%7LyK0le)&nB_B_^3nbz~VCX zF`}NsrWYM>hK!imGbKJ+5t5jjXb<=7r=F;){SmPGs~_89ZQT_@WSev{VB7yqaKq-% zfVwAdz3=_u!S){c%daV@2RI1Y|NA;Wj+HUMb~2}t0Fp5Cf(s6cz9By$?KI4Lfga%w zBa2Jcy~ArG`4*?+67G*rCMv>;a@yX^?bZ8I(nqij6<_h&jdks=EzoP-uRjP6EXlN< zuc-=6mw^WI%AaL}2XaBnD+3oof~{KICBn`;3`eOybE1yMI&!o>vR3ThdsE6|ju>Nm zz_D_`Wc6o`O$YHLeo@`Z5>C@h>WgfRKwhJ$d5Y7a(K3bej!*C8cG2&qJ4%}I7G8Q3 zMYC`%q(vI=T#i}}|D zJ+%etAP=&{z``>Bp)VIiRbs=z+x^%)$CPXFJHJA2#x=X2yTaY+yn&0kx)JH@XG2uz z=#NQ-!%q%f&U|7&}t&+Da!w0TahTj(&`sm9IQ&j@qhWF=8e* zBYoSynp(oPEBZfpIBZrp*aR3CZFNOGZ&t3Wr#qCb3fYuKWKLFjgieB-E_29^#rxZS9riZw#1` znF_GYb8f66RCO}Gl1E|%s40I8YL*=?uf zIrciUV>YcHwjwB1O^hU2ZcVSMCABIx$;)RaBwf2ERJX0NRovS)(4V+` zs7}9q`&c^7rYF>?+O1~aeyD}VqT52@Y|HvM-A5&$@FKKCxWG(5H9vFgSooK+iey-D z2Ll2XxU%EfJx)@|2`{_vSkEoR$F)@4!s@?4MXqzI_~95gA)t49#54jIZ)JtWY;{6U zDp-VPMJu6lhQhFW<1Nk?Uf5^rU$#HdV3Z_ccbc9DiTqSSyF?YM#3e zGvqnW#sb0pBYGy!+R}B30SId=CvO&+^4m{2jk88Wl_5G5sj{f#n=9-r;V0h^;;|#9 zu@9ivZT8Svdm6w=DW_(_NgyV+Ww2cG*1clDm`!D3|5F+ds zVnaI`ps7zwXwwx*UE|+vVJ;Whh=(^o?=nMxPtV2ZWvwA(E~2Hi*dDzteqa_w%N{OK zY5U$`weMy?lt=_8sR2@w$kY-KG;wqo!8Y+x9x0X$P>Yj)LU|^t{?sz6*tqo%^>Jqw zi9<_LLcq_JvJF39 z+dMmHs&>zmS253APmPfw!Ajpt@g@6cdbQUiNnyZ7+)yvy!_ldGhViwOLE+KrIswoC z;sO~N4HfU~K0@57bXJwPaY=^tH@SKrbFI+x7NX`ob3D~ll^T*y@YzbV)Yr#Gbc!bvt)g zEWQL^oM}X2MMhepD@BLR%vfdNJc2$N!a$Su`BK3PL@faU0i#Y=vw#cu*+bM-odP{rzRW>#(|GjgEw2X`xcmz@< z1$vx*gP#Oa)CKwz=NUIq?DMK8_Xh({wiJ&99Z;L=wQVD`Jp}bnNUPiwe(bZcl(B`} zMM3AeAILdKgNvL?yRRY_SW3XO`U*|A@Vjp%nR$5dp}X7q6a(kca%1c$bOhUh&-2Oc zmkN+B=%jsin}CIK-DO$|#xVLv50L5pv~Sbz?o6Uj4QD5PML;Bk@? zk3GB19t3L@8XkTNYVVe7z_se=?oKQhvmd-9Vn5gfy+bsB=?wA%IqJk@WLfpspr#3R zTycZJTW{qT69P{AANyM8tnB}FoI0OxO^OK#T{e^W`=CkobQ1$wBCd-a#2?)MPc`hd zn>TMNsjEYKBICab*l};)Mmg4BWZGC-Cj9YYVrkFhJVHmky@65d8FnV>q-JVbkmQ{4 z>sDw-y0|6bEEHTfhYelkp!*$?{ETfB8g;nk4~ow6o%M+#Lg{)4WkWmhWl!%NdSGGI zoNYAXc&=l*xdqjFsmq>Y$xl_8H3#uGe$lUzm#3I}h5PW{QtG`P0D_kN!^9~;J>QgjL>nrHZw9XKze?DUdG7cMsOSxxHq!O zhO{Iku={#@3DxntekDVN8Mpz16@&~@R2J>g3UYF&)3Y-Hw?!r3R(elaHXwDaBdahs5XS>Cox-?>8UqL4!kw1aP z9W6{o1UBTMEAZ9cL(>vzfCCXCYdNntIw61?9rnQ?w?BlCZVsIOY!5-Cl%tFuQVAY0YX0%I~lIpl<3V%zH4jdqlz8 z5>AVT*81ihvvQ3N|8xND;i1L%Ru%cKg@elHz#U={Tv}9?6S~EEgBB=L83+5=Wo1Dh z6ct<;Q9sgo!*7ffIUJ`StEQK}5+Y8-vf3xUPMk>p*&5AhB3!^nK_bAYGH@ryIk1<3 ziC3L9kS13n!l~Xz?yY2Iu0~*42xqEdGc-sdA&8F^{ZO0jp=T1M5PyWClB!xBnQSbX zFs;>fjG;r5^u)xsZ*;)I{aX+{X^>F;gusUm$&c_5EYZey8uMqP7_PiDVauZwNR5X2?lhZianSSL> zjH6N&tS^qZKi=qVQ$ckb4PrsrjSPS5^T~3D-@VycubufFhuW^t{^0*LoExK$?)(kY z^>jev_r`7cMI~_1vA4s*1q0Vd1qRkW+kB5vj7DmZ+J`6k+D$)F%%3R%YcK-FJdKPw z#BDc^YNXPmK;J@)8?VZXQ0eZlblKhkPVg0>vtvRTR9Er{{mG%eEDYJVSu#l8QZM}e zpI?pS^k|_vH-J7;<)bs=c_Gu6jrZx%`%@1I3G?!pXXvGul}`4JMv_C*kT!isv90|< zbya8>6I7oal9#K6tmbY#*~US=5k>PqLI2g&IU&-S9udJoB^}scWx>wkMg}#R4(t7O z@)+m6sVe3Qa&B~b$xz9lw~}ZDdXyIp>ZzfnzNwwGsOz@(@|)RMxP*{-oAa4sPy16U zsgQ{5k5}T@4=0#@SV1e)R-Sg&|CoI}^;?gCK;|R&7>1_G8JWkU(DI2^iDT^?zA4Vd zcTHVe@YvzcpITqa&AmiZQVQ(TquljWJj#|hr`E3*ABi>14$rz7U@I_EYXSq$vK@r{ zgpDeToz+Z32-?cgmwpgW=oS@{un=pee1olZTz^g|&;M{Br$$s(c44*H2rY>R4XX8% zyY^I$6WEcoI7T&=J|aXcEO7|d&~@ENUoMAY1)YrzWkCJh{nyXX7aX{>QNj4c9;egZ zx4-BVaSg9y1wANlDKvwYEgfPNRcUkrIUDl|aDo+_8rvE9$E3CHHU^R*pQCj~LH zWM-BU2@@Mk9`9{0gm6~SQQD;FLo1C6*_8R?1F=Bo!$+llhPR%rwAU5rqTvvrZihVJ zmtkSCX@{w54eFVOC5k`zLezeCf-w`wzi-g+GE@oSTz6!;xx<6ZHz%3-2s%1YCV^8v z0?=*vH>^Bvy~ih%e4&Zb!lZH({GtsFZdS8P8Hr(0p`n=3Xsnqb6?UXZ6{dfnZ=1J8Xf96B|&&;1&dAt9h-o0IL3+B29P~1%`iUy5VEF71f1_&tUsjDMV6QYiNYsuU=c|#PqF`=P_ zp}0)3q=J=;<5RUS8`poUm%bGz_TEHwd?(F*2;Dg<=?7lvp`m|i_0XB!ZUuiYB_BE3 zF*nR7nf;aRuraE6N<8`FQ0}D&p6#%byvsB1S z_qO9S{`6hK=Q-WmTQ!tHX2F208cU2cwAN5NOK)U0R&f80vxO_19Ck2LV?g9Fki7%9 zkU?&-_!Xmc$h58dUhduBy-_q)W8QESL=P{$?g+EA3Ue~UeJ69K`0b5Ygvm|l2l|1P zr?njI0N68ctiqjLpK5yJf^KIg*PA{Z0fwA_B%XI~1)?juk~X~}dN?h$+HzZ7zM>fp z4*RPFR@uACrRsFh8fSzpN_3UIz!$b>by*!~7cGT-LyTOzt^P3c^l(7qsoO6{QeZNS z;lMDhMxSCdot z9WKYqvaq1TT=IV8>vli(m48v@3XeI1)z)qU8Apx(jEvO+6AKH{M9RY#NZY&#L%U2W zmnoQa3u|ZTcds5F0W6dY%)0AU8r{wk_?&8SSfB>jQLg6nfCq)Y-o&C+)r?Ehz;%@2 z6fecS*69#h6|lL;$jCZ+fc>J}c`vk!PC1S-L&~m!4j(#sVyz@ipzC;exH*@nolo-Y z`SbT$`Cpc_xya$!T3tO!jAD+qZ;%I_bM9aA23js~L9n&%-*^2xDgdBY`y~GLYyC`L zUj0ufVBI3zD*!tP>L@a-wC}aCvSPk=4IA*#i;@!l%nu)2&yGFDr2j7*l#cJ;KX+`X zSmGTE1#L{uT{UH45!5Kf+yrKq_j>gbk6VfT&__fZ0wj+nAen~&AfE#=S6gR1I~wYL zlM$Tk4>iaDnM_ktv!k=~D}eS-en)ORh82aeK``Ba-vEgJ)CIal-#+)s;Gx9-Z>cJnSXgK%3O=LIt%Y{TZvX{t zat!-g|A$n79A4d{`p|HP6|LFwk(LI*i;vLNAZ=_Ur^C=MF_HG${E7pz52932r^~>_ z7(3GZa7TL3e`jY0$#W)wIy*Nv_g@SbKA?tX*5|+lAg%}&MUw)W3w-UP`Za9acCuEp4bM2UzM>DdwZC`BtDZD}Y{Meqq51 z_FN~7vUFo4fmVJ5^RB+mfGer;~X0480sF(c4?2M4uLa`yHgd5#9sM$&|A zK#Bmo8I`bkRxOpiKijYxUigTWrD;Zvfq zZyAB;pKUN)8n0yKMpS&$MPp`R`I)PPzP!F3U0BFd(4Yv46OaNC{|{D-sxKetF*-Xt zFA);rpa8@#w1kl=sj3c`d1f;jmOZ+BvdD`1jQ_YaRb;9tpiuMP=;`im z1-;8w;Ke~8D1OqkK%gd(%n#Y_Kb}fB;&6&8rv=AkV93yZ&ro@JQ~NhC&)-$i~sLS9O2UF z0ickvjF)3oMmS_Z$YtjKsV4v1%970g%70(O7KBJYc+egL6CMcn2%lBheo!g@dMMok zR}wjh>A)7!28~Ju!8c8-|Fd5KP|yWP^PRY=&e9SFpD5$xO?k`4!?U7UDmgtpy{znP z>hcT7*&lg$=y2h)gp^wj zshap34$)EXwDLDWYtXFcl6`!7>IXd8WS}30oBdMx=Si$y4v{uhRf&VJ>e1Y{mu2ns z3=9l;Aa4X`XuAfZH_29M>V9J`%ST{4!vKAd3IvB(l=+~0ul!^~$w4Mw3WQ2vj(;6@ zQP#e8g^@t*4YqdC{TZ)7P9ehj^S!Ps`=r?P^C(g?g!C6R%-K3$TelN)NO8Biev( z&CQ9hP0SvvITJ3!KM-G|IrdJUa2EXF3kGSQ*}?q#5KEhvG6JK4p*jcXC^RNU57~Ax z^A|-|0bwZS>JvT$-dr@JY;;|nlrW^Vh{ga2)dUPup`Sio&;G{b=6xJ52i!P@U>4~h zkmexlZI-K7OS7uD$4O(A=>%1wT$!yz_zq~rz?%#PzsWWo4POYo{C@Q6EU>;8fVQks zPp_Hsv!ZBpIL~YXAVi+yJ&?UXNi7Um+O$CBL#WPO)cBV6ZTV61)@w8T6-Ivmy7AnE zgZPrt1ci#E5mA}Q(aHy6Dj*V&K_y;(+3|jv|Be4PYhP~6nmMFUWysY)1N!GZwx_(H zpz8xI2l?#x`sn(ft<|%q!}HYoO|cfUlN~8>h@tKfu8z)}8M4z%SabG@XrdL6mK6?U z6FBqV0!smPp`A)Hm$^^XMmW6@tw2(fE&{-Po_yEuIRY`w2{5C!Cr2u3y4oEF*%Cw? zw>r#=k~24FL1Y1J%}j0_DJkaPMWDjnp+)uv>$LIWj1UB8Pl5l4kkOA*JSym~BhXb06_L?InM(&j#B7D#b6ua5_-Bn7K7Tmj?CBv6p(}820IlQu zR}gq42Aku~#$Hvt4wBIi`yX`*Sb(4U@0uL-8h;m;|C(NeIqtjMTy{`AMUMA^vJv=_ z+=y9^mgCuu-t+pq>QJkX?2jf78azV`3Ps@{JwTb$&n_-nf_RAmc%k477m3b~aF%~! zDXIJ{S5LF81^@f+JEx2B8m~aY37S~|BPb9DDT2^vc4;XDxTDDVxeayJ0O;e(MbT7* zfo`FTq176#V8|XJ(*rHP`QrRQ*3#NK6VW65%#P4vCX&EUR0%je1xgmO0oPhdTrvKXCf`?~Lzm%B(T%ZcQ$7=Zu&J;d5Dpx5L7M zzRJ?Gu=psQp6fm$=>Tf=D;slG-+%xa5DpY;Q^^yA!~}qa{Uj^EK+2l<^pa#@VPWGH zFJ54RLNqP=Vd#%MgMndUa1jPs`sjyil`||r8wQv10&D?z@clI&4uD(dfZ3g4+Km1m zE`ToLEw&G4Dx!m6=~-GC(11ux4Z-^>qOSuG2B8*0(f2*x0{2q(zwn-E9^`vbUj-J-4=Y1$ghm4#W7|x`p=y;rs4?5{}%T zV{;;hQ-ZM72PJ#9dmq8W2+w@H!U+$BX#Ws51DYyxSZbA*w=DJ6r;rpCuWl3l)hiTm zH+c-qKYs#y#J^uifQU*2th&hRw!65?_Tk7`Y%g{eVkq5p7!x$sU~I=AAvRzH=}Q1W#|U;|R0uTWbC{xxXL2oaIe zh~@v9Rz(fd-MxGFjNgH2?ejD!sS2R}r2F@`XpU zO=vu(4RjzgZ_ZMVSGo93LO!rIQRDyLSyp^3*-y~e*nun%QQ1O#WCh**3UI;`KvBV7 zzc*}{UE>ABElfEWMKYXYP=#EruB}xC73YA7I~W~q(BsY7n;?j_gNGn0;e^!GkM-7| zR*7qDY)p9j*0}IMT|*-orL?lLk{zCrHgbLz#~R+Kq_r_!5gi=bA+dR47)Q2>Cgr=& z1NvJ6lzyc%{dlEI45$#y#>%V?x7rz1JvM+cjgo_sJ&FmoqPn^o@kjfq{~JXC=d<1P zT7^4zATDBnAc{vvO1?FF2BXS0c5NQQ>{8|^&qL)ni^Lt(q0n5b8&sh8nR2=E>V6Hljm_lO|FE+4_9#;^HF=TIyF(}PCMOG6<+ zK?X@@7P(}U=Q=(n&V(oh>?%X}9;yOb86ZS#2c-uQBO}rXLp6cwIg+F_E;KCcE5xan zAi4ai^*&X^{kK})ko}qln0C>5x^{)#wWr#750UuXQO%Kr3fLdB0Xb4jDDl-zc#FUX zlX@Ri5FF|^M$B2_cOt2tBBDejS1Gq2`mzQY7Ls1mpK&Ik5IxiEMU4TZUQyOq4$;rM zAGd-e0hH#(C++}gL&D!E-5$WO?K4n^D60dg*m<}x#gXC@6hPeP{``56>8l*~0%Dmt zB+#RuZ3yj0ig|A^{*N^5|DibHf3|5LNn#66%)5@@a~8vUJZQr0-@g~Ml)w@A3XyUX zME7v8G;UaH6zMikOwihw(|`u0ZM4*an1~1kuVaX$lOTb#8{TAa1@+Fb()kiD?pGkV zUAO6>%gV_y2T3z5u34O7_*ku;BjbDq8ZmMIZ73+P8m#_?5`s8b8gl+GlXJmaIa2EP z0T;D4lvJO9Mvx3%t%1pO`DWwU{)B$Lj|h@*Qd5V4*5fhwGPlQ4FLsuN_`HZ+bpw+Xq{*A(J!gD{+B+c=!>BF?G5RvuagqLO39ya8SMC% z#_ni7#(xMA2ULBbU|tv~5NK?IWR>zHmRr*x)AZtWkn^1);Y2Gq#vX;zAFH6PSq1^7){>*qT0F4gA$iPj(<3&N3oQl34qZYNHFEG>Yv#_pSCHITBz@%4mY6I zGTLLj*;6sSa1eD6!=Vth?Q63Abp?f_>cy9DEvgB}_}_T^$&Lyh{sNLh#JNHh1$kWx zLi;W@sSgK3w*b-dYiVCmL>?|H-y;`~D!U{MDl%5Vkei03SMXvBl*PoqaLIptzL?G9V#6lM9$Kz1+WSNFeHyzU;I5Ti4LAd6ygV* zcyV~Cg62ts{|QuT>5+9$d4@i)cA=vHxtdla;86e+(IBP61L0q=O7<;K0X1Bl`_}FB z+yJYtg0y?=$J#{B#D7nM8$-zK8zz(}-+`zPk*BBoZ7ajOSLQ*t_!Vjz$N;UeuLw@{ zK+_;JAtz4wyDJCebT~`B8H(1{*T5!cS62%bCk(a!SFqQUqPp8~NV@()R{=NK|Y*4NWKSDFkUm{C$gsxk5k7! zSCi4f0%`Oo%7vzNZZuN*M}>s!Aq87O;Kp>QoS#oTK4H&T-Zw1Qx&}^}#orH8&5~^( zC&hVVl3Ga>k;QY4%L-V8KW9A(DDCfKCht=u_(ZLFph-=B;|*z9aQnpf^|H}D zW@d5>ZHg;5SRy!>wU}{nSzjMb$>yk0m3^}M-T7lD;O$P@lcX7;owrMB+kQP8W0!)0 zVkW&ZzBnWg>7Y<@D9pOwNrpxx1_nQqO;ubPc@-243`tQa5;UUC)P;8A60d2_y)_3X zUKA=V8g8xO5OI zXgC6qGJKS6(WM|c zyf+d9f8ZTowhYL{E{#qeSTO1s)Fl+^p%5m33e+29hTAh2|Mdwh$(MHu5r5h>8W~}r zMY!;_n;-5B#RMMbdm!VdLuZ<0i;=eecOPT{JI@XbjbSoTSK&U_0~^Tb96iY48OUc} z``+v7!KMr#BN?kG3XtOYQ6q^$iPNGp_1*G0C}wn-nf}ioX0c*p|8q5rOLQR_?NeT! z61Z67Dp{=ee?H%J7ziVF>u;o@);j z6cjjxgzmH7O?y;H4Fx$sF`3|eXa!^6TfpFF9CHbwmoRzH{jMg3rQ zgoEFPiQT#pfK}rGZap_zDwJ*aXyAz*;X`+b&mVsN)PibU)lzFq3kN8EK{*_kH9v3u zfAn?b@l@@5d(ohi3`HqKop4VHIYq{hi|j%fGi+{&B2$LUaXOWbl2THHTvKK_rcB$R zC@HfY+BQjSbLQFmtaCr_`#JZ%pZEPk`s}^eT6?YE@An;^=lMO})YOzz$BW@Dm=T=7 zW>4qI^2?5n#%Ip-%2vkd-&E73=x5VLPg)80@(_)S8?1rt=v?eEW~XC7p>vlEp!Hx(0u zdJPC8b?@IZssXZG_R`H&XI5|xe0aw43k#hj*P+$zV9sDjZ6cC1Q*9nEuY_uWV^iR=jXE!*n4uKV|k;Q-K*(0vRxpOi7VdbKbvXw`RSq{E0&>P;UHTJsl|kSNLF zuRu>H=k)22>heT&w6b@$ew<1gR9+5%5(WY`92&73GIr=zacIx{)}^|T!Yro}C*8Hg7II=0&LQnOs^TEPh+s)DcqMR3bgCpvteUO<@M~`AU=VWLN3I=euAU7j6*WJ-yeOdxf1?{$YS%*P9ud>XwA z5NkKE?{OOV787Ir_`%*cwM*GM2qIIRxRKS`CRd(mtK+~jWVW#r!6uu6b0gPr4Anl2 zu*zARb&Zg1=21I=+|s&%2$s3B{g9L0VB_NVz@;STt|ZCq+-ywTOm&_;?50{b^F1!E zOb^7L^q*}tP!>j$X~-LdM;DX8!OQk;=YENrp#ylZv4czM>Yb%U3;i|oNm~fQLY#dM zdvocyeED6?3W6xLHM<>*y^(q{%D&CsWWX2PN^EuG7R?qNXWCelnwGF6SQSf2GPEQP zUMw1RjmnO)SJwi*pQk@m4GqoA;2^VP1@C%(Tb1#dT3ZG#d7HQ$T-8e@h>!O)D<}Lf zw->uCE;s4KO~sV*t|tiPeMD7GmbDba$F_RGw&sLPSIBDn_4%x!p&A4&DXYOQ z3%vBVXR;e;CZ`+c1IDH}2=%E=!4sC2g{|SK$-31cC!sefNU#wig6&^s++ZrTt%G*Y zYJ!cZTzJmGdySPGm1Otcs?Wip9ib_hwwI_HUmgge7^Suw@CaX`A`s2*W8bq}H^yiV zT8SDcn{d1Ah=XmeaPP4w|L2bmpfLliF0W}E|&sUKgJq1 z#8qTrbW8UTRoiOAPd<1MCZl}i_-4*2XH~TKKM4F%nM$Ep4J^g(S}k<-W-PwhSta_f znQSmK+U3^m(Ex|BiX8G3n8RgttVbe&#?flD@BJ$8))53Vk%QOzyV_cxQmc^gXPi`<`g8 z!VJHoPWKmQJi=`J(p(*Jm_Y4pNbo@Y(?np#H$y^@!1RFL{9ck|+E*(Y!xN;XHG^D7 z%=1`sRC;&&>cBy-eVWx$0QM(2CFKAL0XsW8(qh0(blRR7W zDl%~8d|@1FX6>uRqqdwY;?u)6Gsdt)(_nJpLeh?dXaC{l&{;uU9FlZkDDB^?G1R2F z_=VH#5VS{_O|0>tC~jdl#@UWuH?H2^>{O?bu7S;-`l-fd1A-Jv(gx0?k&(sOR2~hD zY{{!@D@>twkLBXw6@4ye1z!|MG`lW_kF#@DdyY-c#W>#@RT<=UsBsTSOBd$;%$lJm z|G+2sk+$kbA1C~`SNqhgLPBi(KeUbYXwWBWgt9I3MwsJ{L9FC#&9`)G zNiHD?xN;a6*RZTBvR=WySgA)=GhL|)On$_O80Z%2eLP$C)0@}vs9+HBDQnwdLIF-O zD-+c`&^AQFAJFTA9Q_iqGne!h)W4g=3RO9B#of5^|J9I{qOb-w1h-tt|ETx>t?tH3 z-NJ@2T4}#27}t>3B*;T{S*mRwJ0I(Jn}Pv&6Nz@g!^@H8JP?wwm5@1ZXO}DnZ@_n7 zC^&8*Ju}nz)Tz5|c4x0&WV2bH+L8vVHEW>FC`2#4oXdLg;7n@s$@=4Hnpo83eRj*$xM!qafIYy~zThgs^}UDi<>$#H9nu^KV_ zaT$tvPadqSsdI0BNlZ_tKmZ>NmleeVCQ3{f+R9a~-AcwVRx(71DdIP{SIVa1^ZUq+ zDL6b8MPzl~920DcjLFVo0Po4rTR1}3v=Kf@4LWD3u>9$?H+o6@?r1(!9jYJ6D58Iw+$5$_rr zTnj%yx9tE`RiCUYm<@@;hg)KjJE~qet-*wD-#cMN_t!cm9XjUbPaj;s*J-;Tv^rNE zGsrgmva{17%Z6r+2sGvkU4Px%x2LaMxso`{FCgHbAC**JUoX*>=pc}{BF+M!InpbB<3;Nru8V%-QXUsA$cK-uv6fr( zSY}Cy(to!mmnX}jaaU80;VIO&vj)1lP4F3;?hO$g`f!x_sD;(~T;o?y@BucDoM1pa zD|z66C!A;Rr00c$nEDW_Y=$LM(H*oFC@wDU@b>=R&fIgqwk0pfVKG6h#Bi(TDj7=8 zh<2)7yXM2!5Y6_H1d!2RjIlI+-N=H91uj)n(;;fhe^k{G;}l)pGZadwzhC2XjZUC$ z)`2r7<=<U@b=1h z{#+gF*SSz?ZK+ppy}BqT7ncb{J%4saD!3oW-h8Hij+ItbWh9o11x~*En_i(#i}Z!1 z__XY7ecV6g#EA*hr}F~v%4&dQXMWU%u)Ry39YTvNt5=3-ebk1PmREaCOl@wS*8zHW zb3WbunxcS;>K+&2TkrvUX2_3Z(D=fq2CDe?|`sF&(!p>&Jt~PCMN|qD{ zyPf0lRL5#ZN5@zp)`LBCjrP^IxONziuIV#Ozt;D*&VH<``x!XDOYTQGx|J<_9`)d_ zcnZ!@4a4001Eni#^R;Tx@1;xYq4bJDKl)^^=B+2-<}8fL;x9Xuo}8RlouY|Wd_7dN z(`HTyg-@l@vvvQD@cbW{=6{mY{7u|1q9<}|>yRPgWB$KlYFT>_h?ssi1;H(lFj}X7 zS_6iaPtm-!MjRNo6{l)p55fTwT-3Tt_+N| zlvUc=j|BS!Co(iiuvPcwmhcnSSG$~xSkO7a+cQy+YK=k!wB`#${?O1+L$Gmhd1JiC zz-G+P`^>owHtuz1l$h`wn|~f&p)m}6ta0enlS3L%M>d0<_#EOgIXS7~$Hc$ zAd=$ZT9_XtNG&E*Y8D!ODpYii%!+c-1*@&C?G0m73hR2%{V0gRPQ4s+f94J&b}e%w(9&Ho8V`hbdx z6{wX~@Er|{uL!|txW!_NV}V=m@jwt9B?EE(M}Qt6WoI4N%pXv{7tu^^$Vs+*i9=H_YfYs&+Ia*mIn{wVNd8Ip=O zL2Q6?Dem1Ew(GEV#O2zTaP5||Dn1Q#sScZ7;i7A7X!y@KRewkDM(J5uCKz+vZazL% z;M`@ZtE=m`wlXINcf!$6FEv3Q197`A*8lbEAmVB(5VN-5FB+iSVu6)2OVy7(0^Oi@ zIYd5fc=-IHqB+df*&;#N^Eco~oOb{IeF-&z;8bjHl016U4Bp*ls652zdzrA5-;Y;h z$|kPsEFp6ZLh;s&)#$ACDr*NDuX+Zb&b1J6{4s*lvPi5n#7~5nx6*(4Ll$ziF%w}E30QHJ;h?X?)8E4rQ3$9rU3Igi%d$6AdKz0E^vV`ElAJE zVEi%m2#FS~P=jrETJTS=sxBbe08G595zqOl?WJukzp!v7-j+nzvo*IZ|3-R_?e_kh z00zz`a!>$;=CUcchT41-h@NpRtVfCBmly6&c~~(J3eq$1X~dw@7G5Z={h_so zut>&Wh2J@9QN>ce_g;qf*;c{pg*ozm#PoZcRYTE$iC^M;zWlJ#FB5cyF7_@l8<42v z!GotTJzOW^Fat&6X>bjnn@3re59?#9Z-3;)6{|`YSZN|X$-_cdFTzU01;08msxaA1 zhR%Q`K%2jLY`$mRXvgn5==h8PQ^MT-%lBh2y*z#V_;r`kQF+MwK;I_UYg0msvHA?3zOVx_GIWrN+js9Cz0%!;W$gJ*JnAO^ z!n7sEcq^)aFkwZ*h-mWWD(1g%&Y8?b9D#MBt|2lD0@$ONm<(v>tf30IQL!|p>^1V~ z3>j81w7u4~EKi)?0D8eojeR`28wra92qvCDV;+q!v+o_eBn5T9p*pSEY3uIcp@)v4 zRtW{|QIw)0Q+auL^;B<0e_VD}mK>hsVT42pF|pTTVq#CgMm~i*UENOGddMj1?V_mm zRaI4|u-fbAzxREB9W!IN;fZWD_Pq*)RG-7C?#k9E#OfagssAt{;ykD@^24X>?NiJP z93zpe|12W|rkiw94C0TGi4vO*D+3n#qsHb`y9U_RPm-PAn_c__yT!S43Aea+r^A!4 zI7eh7AqgDiFD-;-<)1AR&tG5r4_@Q{&3emfZ@=hebu30{;5>w`mj2(fkN@NPe*rcz Bsrdi^ literal 0 HcmV?d00001 From 692a9098ffcc5a80f5ff09fab6c0a59a15198bb3 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Mon, 7 May 2018 22:01:43 -0600 Subject: [PATCH 3/4] Minor update --- examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples.py b/examples.py index 81a5ca8..331fb2b 100644 --- a/examples.py +++ b/examples.py @@ -427,5 +427,5 @@ if __name__ == '__main__': # action_conditional_video_prediction() - plot() + # plot() From 3c3a998c49a3a4abe8d96da07f505fdf2fb308cc Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Mon, 7 May 2018 22:03:11 -0600 Subject: [PATCH 4/4] Update README --- README.md | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 94415bc..8021b6d 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,29 @@ Asynchronous algorithms below are removed in current version but can be found in Support for PyTorch v0.3.x can be found in [v0.2](https://github.com/ShangtongZhang/DeepRL/releases/tag/v0.2). Note all the figures are generated via this version. After the upgrade to PyTorch v0.4.0, I have only tested the classical control tasks. +# Dependency +* MacOS 10.12 or Ubuntu 16.04 +* PyTorch v0.4.0 +* Python 3.6, 3.5 or 2.7 (deprecated) +* Core dependencies: `pip install -e .` +* Optional: [Roboschool](https://github.com/openai/roboschool), [DeepMind Control Suite](https://github.com/deepmind/dm_control)+[DMControl2Gym](dm_control2gym) + +# Usage + +```examples.py``` contains examples for all the implemented algorithms + +Please use this bibtex if you want to cite this repo +``` +@misc{deeprl, + author = {Shangtong, Zhang}, + title = {Modularized Implementation of Deep RL Algorithms in PyTorch}, + year = {2018}, + publisher = {GitHub}, + journal = {GitHub Repository}, + howpublished = {\url{https://github.com/ShangtongZhang/DeepRL}}, +} +``` + # Curves > Curves for CartPole are trivial so I didn't place it here, and there isn't any fixed random seed. The curves are generated in the same manner as OpenAI baselines (one run and smoothed by recent 100 episodes) ## DQN @@ -49,6 +72,7 @@ Support for PyTorch v0.3.x can be found in [v0.2](https://github.com/ShangtongZh ## OC ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/option_critic_pixel_atari-180417-092617.png) + This is my synchronous option-critic implementation, not the original one. ## Action Conditional Video Prediction @@ -58,29 +82,6 @@ This is my synchronous option-critic implementation, not the original one. Prediction is sampled after 110K iterations, and I only implemented one-step training -# Dependency -* MacOS 10.12 or Ubuntu 16.04 -* PyTorch v0.4.0 -* Python 3.6, 3.5 or 2.7 (deprecated) -* Core dependencies: `pip install -e .` -* Optional: [Roboschool](https://github.com/openai/roboschool), [DeepMind Control Suite](https://github.com/deepmind/dm_control)+[DMControl2Gym](dm_control2gym) - -# Usage - -```examples.py``` contains examples for all the implemented algorithms - -Please use this bibtex if you want to cite this repo -``` -@misc{deeprl, - author = {Shangtong, Zhang}, - title = {Modularized Implementation of Deep RL Algorithms in PyTorch}, - year = {2018}, - publisher = {GitHub}, - journal = {GitHub Repository}, - howpublished = {\url{https://github.com/ShangtongZhang/DeepRL}}, -} -``` - # References * [Human Level Control through Deep Reinforcement Learning](https://www.nature.com/nature/journal/v518/n7540/full/nature14236.html) * [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/abs/1602.01783)