diff --git a/async_worker/ppo.py b/async_worker/ppo.py index a576b2b..8cf8405 100644 --- a/async_worker/ppo.py +++ b/async_worker/ppo.py @@ -122,6 +122,20 @@ class ProximalPolicyOptimization: returns = list(reversed(returns)) assert np.isfinite([a.data.numpy() for a in advantages]).all() assert np.isfinite([a.data.numpy() for a in returns]).all() + + # 40x faster + # values = torch.cat(values).squeeze() + # rewards = critic_net.to_torch_variable(rewards) + # returns = rewards + self.config.discount * values[1:] + # deltas = returns - values[:-1] + # advs = [] + # cum_adv = 0 + # multiplier = self.config.discount * self.config.gae_tau + # for delta in reversed(list(deltas)): + # cum_adv = cum_adv * multiplier + delta + # advs.append(cum_adv) + # advantages = advs[::-1] + # returns = list(returns) replay.feed([states, actions, returns, advantages]) batched_rewards /= batched_episode diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index 013e0b4..32c8f6b 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -165,7 +165,7 @@ class GaussianActorNet(nn.Module, BasicNet): def entropy(self, std): # 8.18 http://www.biopsychology.org/norwich/isp/chap8.pdf - sq_stddev = torch.max(std.pow(2), 1e-5) + sq_stddev = std.pow(2).clamp(1e-5) # return 0.5 * (1 + (2 * sq_stddev * np.pi).log()).sum(1).mean() return 0.5 * (2 * sq_stddev * np.pi * np.e).log().sum(1).mean() # return std.log() + 0.5 * log(2.0 * Variable(torch.FloatTensor([math.pi*math.e])))