From b6436186ea559b2f1dcb3e6f40dd96e0fb4d3292 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Sat, 3 Feb 2018 16:46:53 -0700 Subject: [PATCH] Training without GAE --- agent/A2C_agent.py | 7 +++++-- main.py | 5 +++-- utils/config.py | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/agent/A2C_agent.py b/agent/A2C_agent.py index b4adb90..41fca9f 100644 --- a/agent/A2C_agent.py +++ b/agent/A2C_agent.py @@ -103,8 +103,11 @@ class A2CAgent: actions = self.network.tensor(actions, torch.LongTensor).unsqueeze(1) next_value = rollout[i + 1][2] returns = rewards + config.discount * terminals * returns - td_error = rewards + config.discount * terminals * next_value.data - value.data - advantages = advantages * config.gae_tau * config.discount * terminals + td_error + if config.no_gae: + advantages = returns - value.data + else: + td_error = rewards + config.discount * terminals * next_value.data - value.data + advantages = advantages * config.gae_tau * config.discount * terminals + td_error processed_rollout[i] = [prob, log_prob, value, actions, returns, advantages] prob, log_prob, value, actions, returns, advantages = map(lambda x: torch.cat(x, dim=0), zip(*processed_rollout)) diff --git a/main.py b/main.py index 5e6c710..7074730 100644 --- a/main.py +++ b/main.py @@ -176,6 +176,7 @@ def a2c_pixel_atari(name): config.reward_shift_fn = lambda r: np.sign(r) config.policy_fn = SamplePolicy config.discount = 0.99 + config.no_gae = True config.gae_tau = 0.97 config.entropy_weight = 0.01 config.rollout_length = 5 @@ -321,7 +322,7 @@ if __name__ == '__main__': # logger.setLevel(logging.DEBUG) logger.setLevel(logging.INFO) - dqn_cart_pole() + # dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() # a2c_cart_pole() @@ -333,7 +334,7 @@ if __name__ == '__main__': # dqn_pixel_atari('PongNoFrameskip-v4') # async_pixel_atari('PongNoFrameskip-v4') # a3c_pixel_atari('PongNoFrameskip-v4') - # a2c_pixel_atari('PongNoFrameskip-v4') + a2c_pixel_atari('PongNoFrameskip-v4') # dqn_pixel_atari('BreakoutNoFrameskip-v4') # async_pixel_atari('BreakoutNoFrameskip-v4') diff --git a/utils/config.py b/utils/config.py index cb1324f..182f704 100644 --- a/utils/config.py +++ b/utils/config.py @@ -33,6 +33,7 @@ class Config: self.update_interval = 1 self.gradient_clip = 40 self.entropy_weight = 0.01 + self.no_gae = False self.gae_tau = 1.0 self.noise_decay_interval = 0 self.target_network_mix = 0.001