From 774768640d5e48eb8d4c6e30668fd5f8c9e967b4 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Tue, 30 May 2017 23:39:26 -0600 Subject: [PATCH] Update async agents --- async_agent.py | 16 +++++----- bootstrap.py | 2 +- main.py | 29 +++++++++--------- network.py | 80 +++++++++++++++++++++++++------------------------- 4 files changed, 65 insertions(+), 62 deletions(-) diff --git a/async_agent.py b/async_agent.py index 4b83c7d..be0fd7a 100644 --- a/async_agent.py +++ b/async_agent.py @@ -60,7 +60,7 @@ class AsyncAgent: steps = 0 terminal = False buffer = [state] * self.history_length - while not terminal and steps < self.step_limit: + while not terminal and (not self.step_limit or steps < self.step_limit): state = task.normalize_state(np.vstack(buffer)) action_values = network.predict(np.reshape(state, (1, ) + state.shape)) steps += 1 @@ -95,8 +95,8 @@ class AsyncAgent: batch_states, batch_actions, batch_rewards = [], [], [] if terminal: if id == 0: - self.logger.info('episode %d, epsilon %f, return %f, avg return %f, total steps %d' % ( - episode, policy.epsilon, episode_return, np.mean(episode_returns[-100: ]), + self.logger.info('episode %d, return %f, avg return %f, total steps %d' % ( + episode, episode_return, np.mean(episode_returns[-100: ]), self.total_steps.value)) episode_steps = 0 episode_returns.append(episode_return) @@ -106,7 +106,7 @@ class AsyncAgent: state = task.reset() buffer = [state] * self.history_length state = task.normalize_state(np.vstack(buffer)) - value = worker_network.predict(np.reshape(state, (1, ) + state.shape)) + value = worker_network.predict(np.stack([state])) action = policy.sample(value.flatten()) while not terminal and len(batch_states) < self.batch_size: episode_steps += 1 @@ -120,18 +120,20 @@ class AsyncAgent: buffer.pop(0) buffer.append(state) state = task.normalize_state(np.vstack(buffer)) - value = worker_network.predict(np.reshape(state, (1, ) + state.shape)) + value = worker_network.predict(np.stack([state])) action = policy.sample(value.flatten()) policy.update_epsilon() batch_rewards = self.bootstrap_fn(batch_states, batch_actions, batch_rewards, state, action, terminal, self) - if episode_steps > self.step_limit: + if self.step_limit and episode_steps > self.step_limit: terminal = True worker_network.zero_grad() - worker_network.gradient(np.asarray(batch_states), batch_actions, batch_rewards) + worker_network.gradient(np.asarray(batch_states), + worker_network.to_torch_variable(batch_actions, 'int64').unsqueeze(1), + worker_network.to_torch_variable(batch_rewards).unsqueeze(1)) self.async_update(worker_network, optimizer) worker_network.load_state_dict(self.learning_network.state_dict()) diff --git a/bootstrap.py b/bootstrap.py index 910a788..0f52a2e 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -51,7 +51,7 @@ def AdvantageActorCritic(batch_states, batch_actions, batch_rewards, reward = 0 else: with agent.network_lock: - reward = np.asscalar(agent.learning_network.critic(tailing_state)) + reward = np.asscalar(agent.learning_network.critic(np.stack([tailing_state]))) rewards = [] for r in reversed(batch_rewards): reward = r + agent.discount * reward diff --git a/main.py b/main.py index 7d2840b..49c4cdf 100644 --- a/main.py +++ b/main.py @@ -5,15 +5,15 @@ import logging def async_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() - config['optimizer_fn'] = lambda params: torch.optim.SGD(params, 0.001) - config['network_fn'] = lambda: FullyConnectedNet([8, 50, 200, 2]) + config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001) + config['network_fn'] = lambda gpu=True: FullyConnectedNet([8, 50, 200, 2], gpu=gpu) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=5000, min_epsilon=0.1) config['bootstrap_fn'] = OneStepQLearning # config['bootstrap_fn'] = NStepQLearning # config['bootstrap_fn'] = OneStepSarsa config['discount'] = 0.99 config['target_network_update_freq'] = 200 - config['step_limit'] = 300 + config['step_limit'] = 0 config['n_workers'] = 8 config['batch_size'] = 5 config['test_interval'] = 4000 @@ -27,7 +27,7 @@ def async_lunar_lander(): config = dict() config['task_fn'] = lambda: LunarLander() config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001) - config['network_fn'] = lambda: FullyConnectedNet([8, 50, 200, 4]) + config['network_fn'] = lambda gpu=True: FullyConnectedNet([8, 50, 200, 4], gpu=gpu) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=40000, min_epsilon=0.05) config['bootstrap_fn'] = OneStepQLearning config['discount'] = 0.99 @@ -61,17 +61,18 @@ def dqn_cart_pole(): def actor_critic_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() - config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001) - config['network_fn'] = lambda: ActorCriticNet([4, 200, 2]) + config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001) + config['network_fn'] = lambda gpu=True: FCActorCriticNet([8, 200, 2], gpu=gpu) config['policy_fn'] = SamplePolicy config['bootstrap_fn'] = AdvantageActorCritic config['discount'] = 0.99 config['target_network_update_freq'] = 200 - config['step_limit'] = 200 - config['n_workers'] = 10 - config['batch_size'] = 5 - config['test_interval'] = 50000 - config['test_repetitions'] = 5 + config['step_limit'] = 0 + config['n_workers'] = 16 + config['batch_size'] = 6 + config['test_interval'] = 4000 + config['history_length'] = 2 + config['test_repetitions'] = 50 config['logger'] = gym.logger agent = AsyncAgent(**config) agent.run() @@ -125,8 +126,8 @@ if __name__ == '__main__': benchmark = gym.benchmark_spec('Atari40M') # async_cart_pole() - # async_lunar_lander() # actor_critic_cart_pole() - # dqn_cart_pole() - dqn_pixel_atari('BreakoutNoFrameskip-v3') + # async_lunar_lander() + dqn_cart_pole() + # dqn_pixel_atari('BreakoutNoFrameskip-v3') # async_pixel_atari('BreakoutNoFrameskip-v3') diff --git a/network.py b/network.py index 9da3fdc..236fa06 100644 --- a/network.py +++ b/network.py @@ -26,6 +26,7 @@ class BasicNet: x = x.cuda() return Variable(x) +class VanillaNet(BasicNet): def predict(self, x, to_numpy=True): y = self.forward(x) if to_numpy: @@ -38,7 +39,31 @@ class BasicNet: loss = self.criterion(y, targets) loss.backward() -class FullyConnectedNet(nn.Module, BasicNet): +class ActorCriticNet(BasicNet): + def predict(self, x): + phi = self.forward(x) + return F.softmax(self.fc_actor(phi)).cpu().data.numpy() + + def gradient(self, x, actions, rewards): + phi = self.forward(x) + logit = self.fc_actor(phi) + prob = F.softmax(logit) + log_prob_ = F.log_softmax(logit) + state_value = self.fc_critic(phi) + log_prob = log_prob_.gather(1, actions) + advantage = (rewards - state_value).detach() + policy_loss = -torch.sum(log_prob * advantage) + value_loss = 0.5 * torch.sum(torch.pow(state_value - rewards, 2)) + entropy = -torch.sum(torch.mul(prob, log_prob_)) + (policy_loss + value_loss - self.xentropy_weight * entropy).backward() + nn.utils.clip_grad_norm(self.parameters(), self.grad_threshold) + + def critic(self, x): + phi = self.forward(x) + return self.fc_critic(phi).cpu().data.numpy() + + +class FullyConnectedNet(nn.Module, VanillaNet): def __init__(self, dims, optimizer_fn=None, gpu=True): super(FullyConnectedNet, self).__init__() self.fc1 = nn.Linear(dims[0], dims[1]) @@ -55,52 +80,27 @@ class FullyConnectedNet(nn.Module, BasicNet): y = self.fc3(y) return y -class ActorCriticNet(nn.Module): - def __init__(self, dims, gpu=True): - super(ActorCriticNet, self).__init__() +class FCActorCriticNet(nn.Module, ActorCriticNet): + def __init__(self, + dims, + xentropy_weight=0.01, + grad_threshold=40, + gpu=True): + super(FCActorCriticNet, self).__init__() self.fc1 = nn.Linear(dims[0], dims[1]) self.fc_actor = nn.Linear(dims[1], dims[2]) self.fc_critic = nn.Linear(dims[1], 1) - self.gpu = gpu and torch.cuda.is_available() - if self.gpu: - print 'Transferring network to GPU...' - self.cuda() - print 'Network transferred.' - - def to_torch_variable(self, x, dtype='float32'): - x = torch.from_numpy(np.asarray(x, dtype=dtype)) - if self.gpu: - x = x.cuda() - return Variable(x) + self.xentropy_weight = xentropy_weight + self.grad_threshold = grad_threshold + BasicNet.__init__(self, optimizer_fn=None, gpu=gpu) def forward(self, x): - phi = self.fc1(self.to_torch_variable(x)) + x = self.to_torch_variable(x) + x = x.view(x.size(0), -1) + phi = self.fc1(x) return phi - def predict(self, x): - phi = self.forward(x) - return F.softmax(self.fc_actor(phi)).cpu().data.numpy() - - def gradient(self, x, actions, rewards): - phi = self.forward(x) - logit = self.fc_actor(phi) - prob = F.softmax(logit) - log_prob_ = F.log_softmax(logit) - state_value = self.fc_critic(phi) - log_prob = log_prob_.gather(1, self.to_torch_variable(np.asarray([actions]).reshape([-1, 1]), 'int64')) - advantage = np.asarray([rewards]).reshape([-1, 1]) - state_value.cpu().data.numpy() - policy_loss = -torch.sum(log_prob * self.to_torch_variable(advantage)) - value_loss = 0.5 * torch.sum( - torch.pow(state_value - Variable(torch.from_numpy(np.asarray(rewards, dtype='float32'))), 2)) - entropy = -torch.sum(torch.mul(prob, log_prob_)) - (policy_loss + value_loss - 0.01 * entropy).backward() - nn.utils.clip_grad_norm(self.parameters(), 40) - - def critic(self, x): - phi = self.forward(x) - return self.fc_critic(phi).cpu().data.numpy() - -class ConvNet(nn.Module, BasicNet): +class ConvNet(nn.Module, VanillaNet): def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): super(ConvNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)