Training without GAE

agent/A2C_agent.py

@@ -103,8 +103,11 @@ class A2CAgent:
             actions = self.network.tensor(actions, torch.LongTensor).unsqueeze(1)
             next_value = rollout[i + 1][2]
             returns = rewards + config.discount * terminals * returns
-            td_error = rewards + config.discount * terminals * next_value.data - value.data
-            advantages = advantages * config.gae_tau * config.discount * terminals + td_error
+            if config.no_gae:
+                advantages = returns - value.data
+            else:
+                td_error = rewards + config.discount * terminals * next_value.data - value.data
+                advantages = advantages * config.gae_tau * config.discount * terminals + td_error
             processed_rollout[i] = [prob, log_prob, value, actions, returns, advantages]
 
         prob, log_prob, value, actions, returns, advantages = map(lambda x: torch.cat(x, dim=0), zip(*processed_rollout))

main.py

@@ -176,6 +176,7 @@ def a2c_pixel_atari(name):
     config.reward_shift_fn = lambda r: np.sign(r)
     config.policy_fn = SamplePolicy
     config.discount = 0.99
+    config.no_gae = True
     config.gae_tau = 0.97
     config.entropy_weight = 0.01
     config.rollout_length = 5
@@ -321,7 +322,7 @@ if __name__ == '__main__':
     # logger.setLevel(logging.DEBUG)
     logger.setLevel(logging.INFO)
 
-    dqn_cart_pole()
+    # dqn_cart_pole()
     # async_cart_pole()
     # a3c_cart_pole()
     # a2c_cart_pole()
@@ -333,7 +334,7 @@ if __name__ == '__main__':
     # dqn_pixel_atari('PongNoFrameskip-v4')
     # async_pixel_atari('PongNoFrameskip-v4')
     # a3c_pixel_atari('PongNoFrameskip-v4')
-    # a2c_pixel_atari('PongNoFrameskip-v4')
+    a2c_pixel_atari('PongNoFrameskip-v4')
 
     # dqn_pixel_atari('BreakoutNoFrameskip-v4')
     # async_pixel_atari('BreakoutNoFrameskip-v4')

utils/config.py

@@ -33,6 +33,7 @@ class Config:
         self.update_interval = 1
         self.gradient_clip = 40
         self.entropy_weight = 0.01
+        self.no_gae = False
         self.gae_tau = 1.0
         self.noise_decay_interval = 0
         self.target_network_mix = 0.001