From 212ff0bc70650377463b48433c3431ef574c15f6 Mon Sep 17 00:00:00 2001 From: Mike Clark Date: Wed, 1 Nov 2017 10:52:23 +0800 Subject: [PATCH 1/2] moving epsilon outside softplus I think you intended this to be outside the softplus. The reason is that it should be applied just before the log to avoid `log(0)=inf`.e.g. - `log(softplus(-1000+1e-5))=log(0)=inf`. - `log(softplus(-1000)+1e-5)=log(1e-5)!=inf`. Also this fixes a NaN I had. --- network/continuous_action_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index e322b5f..a77281c 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -145,7 +145,7 @@ class GaussianActorNet(nn.Module, BasicNet): log_std = self.action_log_std.expand_as(mean) std = log_std.exp() else: - std = F.softplus(self.action_std(phi) + 1e-5) + std = F.softplus(self.action_std(phi)) + 1e-5 log_std = std.log() return mean, std, log_std From 662c0834f6638ef8d915c5d26f71c8bc4325fc42 Mon Sep 17 00:00:00 2001 From: wassname Date: Wed, 1 Nov 2017 18:58:03 +0800 Subject: [PATCH 2/2] This helps me avoid NaN when I have small rewards --- agent/DDPG_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 2c71050..600c334 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -61,7 +61,7 @@ class DDPGAgent: done = (done or (config.max_episode_length and steps >= config.max_episode_length)) next_state = self.state_normalizer(next_state) total_reward += reward - # reward = self.reward_normalizer(reward) + reward = self.reward_normalizer(reward) if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)])