From 662c0834f6638ef8d915c5d26f71c8bc4325fc42 Mon Sep 17 00:00:00 2001 From: wassname Date: Wed, 1 Nov 2017 18:58:03 +0800 Subject: [PATCH 1/2] This helps me avoid NaN when I have small rewards --- agent/DDPG_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 2c71050..600c334 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -61,7 +61,7 @@ class DDPGAgent: done = (done or (config.max_episode_length and steps >= config.max_episode_length)) next_state = self.state_normalizer(next_state) total_reward += reward - # reward = self.reward_normalizer(reward) + reward = self.reward_normalizer(reward) if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)]) From e16a560abc9b2df6277069f74b5c39c6978839af Mon Sep 17 00:00:00 2001 From: wassname Date: Wed, 15 Nov 2017 09:40:58 +0800 Subject: [PATCH 2/2] let ppo work with gpu --- async_worker/ppo.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/async_worker/ppo.py b/async_worker/ppo.py index 58f1ea8..db59c05 100644 --- a/async_worker/ppo.py +++ b/async_worker/ppo.py @@ -65,7 +65,7 @@ class ProximalPolicyOptimization: for i in range(config.rollout_length): mean, std, log_std = actor_net.predict(np.stack([state])) value = critic_net.predict(np.stack([state])) - action = self.policy.sample(mean.data.numpy().flatten(), std.data.numpy().flatten(), deterministic) + action = self.policy.sample(mean.data.cpu().numpy().flatten(), std.data.cpu().numpy().flatten(), deterministic) action = self.config.action_shift_fn(action) states.append(state) actions.append(action) @@ -92,12 +92,15 @@ class ProximalPolicyOptimization: if not done: R = critic_net.predict(np.stack([state])).data - values.append(Variable(R)) - A = Variable(torch.zeros((1, 1))) + + values.append(actor_net.to_torch_variable(R)) + A = actor_net.to_torch_variable(torch.zeros((1, 1))) + discount = actor_net.to_torch_variable([self.config.discount]) + gae_tau = actor_net.to_torch_variable([self.config.gae_tau]) for i in reversed(range(len(rewards))): - R = Variable(torch.FloatTensor([[rewards[i]]])) - ret = R + self.config.discount * values[i + 1] - A = ret - values[i] + self.config.discount * self.config.gae_tau * A + R = actor_net.to_torch_variable([[rewards[i]]]) + ret = R + discount * values[i + 1] + A = ret - values[i] + discount * gae_tau * A advantages.append(A.detach()) returns.append(ret.detach()) advantages = list(reversed(advantages))