From 79311f02e60e5489a3d0f396f9f373354b16cbc3 Mon Sep 17 00:00:00 2001 From: wassname Date: Sat, 4 Nov 2017 17:05:13 +0800 Subject: [PATCH] found NaN's for small std --- async_worker/ppo.py | 5 +++++ network/continuous_action_network.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/async_worker/ppo.py b/async_worker/ppo.py index 200a1a9..a8e2e8b 100644 --- a/async_worker/ppo.py +++ b/async_worker/ppo.py @@ -155,6 +155,11 @@ class ProximalPolicyOptimization: probs_old = actor_net.log_density(actions, mean_old, log_std_old, std_old) mean, std, log_std = actor_net.predict(states) probs = actor_net.log_density(actions, mean, log_std, std) + + # avoid NaNs with small std I going to clamp this - mike + probs_old = probs_old.clamp(-10,20) + probs = probs.clamp(-10,20) + ratio = (probs - probs_old).exp() obj = ratio * advantages obj_clipped = ratio.clamp(1.0 - self.config.ppo_ratio_clip, 1.0 + self.config.ppo_ratio_clip) * advantages diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index a77281c..d900094 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -154,7 +154,7 @@ class GaussianActorNet(nn.Module, BasicNet): def log_density(self, x, mean, log_std, std): var = std.pow(2) - log_density = -(x - mean).pow(2) / (2 * var) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std + log_density = -(x - mean).pow(2) / (2 * var + 1e-5) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std return log_density.sum(1) def entropy(self, std):