misc

async_worker/ppo.py

@@ -122,6 +122,20 @@ class ProximalPolicyOptimization:
             returns = list(reversed(returns))
             assert np.isfinite([a.data.numpy() for a in advantages]).all()
             assert np.isfinite([a.data.numpy() for a in returns]).all()
+
+            # 40x faster
+            # values = torch.cat(values).squeeze()
+            # rewards = critic_net.to_torch_variable(rewards)
+            # returns = rewards + self.config.discount * values[1:]
+            # deltas = returns - values[:-1]
+            # advs = []
+            # cum_adv = 0
+            # multiplier = self.config.discount * self.config.gae_tau
+            # for delta in reversed(list(deltas)):
+            #     cum_adv = cum_adv * multiplier + delta
+            #     advs.append(cum_adv)
+            # advantages = advs[::-1]
+            # returns = list(returns)
             replay.feed([states, actions, returns, advantages])
 
         batched_rewards /= batched_episode

network/continuous_action_network.py

@@ -165,7 +165,7 @@ class GaussianActorNet(nn.Module, BasicNet):
 
     def entropy(self, std):
         # 8.18 http://www.biopsychology.org/norwich/isp/chap8.pdf
-        sq_stddev = torch.max(std.pow(2), 1e-5)
+        sq_stddev = std.pow(2).clamp(1e-5)
         # return 0.5 * (1 + (2 * sq_stddev * np.pi).log()).sum(1).mean()
         return 0.5 * (2 * sq_stddev * np.pi * np.e).log().sum(1).mean()
         # return std.log() + 0.5 * log(2.0 * Variable(torch.FloatTensor([math.pi*math.e])))