From 0eaba783e87ba70477f94965132c6b4667c6b208 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Sat, 24 Mar 2018 23:21:58 -0600 Subject: [PATCH] Minor update --- agent/A2C_agent.py | 2 +- main.py | 6 +++--- network/shallow_network.py | 2 +- utils/config.py | 2 +- utils/tf_logger.py | 2 ++ 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/agent/A2C_agent.py b/agent/A2C_agent.py index 77506cd..b87c9ed 100644 --- a/agent/A2C_agent.py +++ b/agent/A2C_agent.py @@ -81,7 +81,7 @@ class A2CAgent: actions = self.network.tensor(actions, torch.LongTensor).unsqueeze(1) next_value = rollout[i + 1][2] returns = rewards + config.discount * terminals * returns - if config.use_gae: + if not config.use_gae: advantages = returns - value.data else: td_error = rewards + config.discount * terminals * next_value.data - value.data diff --git a/main.py b/main.py index 0caff62..4095200 100644 --- a/main.py +++ b/main.py @@ -197,7 +197,7 @@ def a2c_pixel_atari(name): config.reward_shift_fn = lambda r: np.sign(r) config.policy_fn = SamplePolicy config.discount = 0.99 - config.use_gae = True + config.use_gae = False config.gae_tau = 0.97 config.entropy_weight = 0.01 config.rollout_length = 5 @@ -463,7 +463,7 @@ if __name__ == '__main__': # quantile_regression_dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() - # a2c_cart_pole() + a2c_cart_pole() # a3c_continuous() # p3o_continuous() # d3pg_continuous() @@ -472,7 +472,7 @@ if __name__ == '__main__': # dqn_pixel_atari('PongNoFrameskip-v4') # categorical_dqn_pixel_atari('PongNoFrameskip-v4') - quantile_regression_dqn_pixel_atari('PongNoFrameskip-v4') + # quantile_regression_dqn_pixel_atari('PongNoFrameskip-v4') # n_step_dqn_pixel_atari('PongNoFrameskip-v4') # async_pixel_atari('PongNoFrameskip-v4') # a3c_pixel_atari('PongNoFrameskip-v4') diff --git a/network/shallow_network.py b/network/shallow_network.py index 65dbb8e..294a2f9 100644 --- a/network/shallow_network.py +++ b/network/shallow_network.py @@ -53,7 +53,7 @@ class ActorCriticFCNet(nn.Module, ActorCriticNet): def forward(self, x, update_LSTM=True): x = self.variable(x) x = F.relu(self.fc1(x)) - phi = self.fc2(x) + phi = F.relu(self.fc2(x)) return phi class CategoricalFCNet(nn.Module, CategoricalNet): diff --git a/utils/config.py b/utils/config.py index 169597c..183bf7e 100644 --- a/utils/config.py +++ b/utils/config.py @@ -33,7 +33,7 @@ class Config: self.update_interval = 1 self.gradient_clip = 40 self.entropy_weight = 0.01 - self.use_gae = True + self.use_gae = False self.gae_tau = 1.0 self.noise_decay_interval = 0 self.target_network_mix = 0.001 diff --git a/utils/tf_logger.py b/utils/tf_logger.py index 6b532e8..927c0aa 100644 --- a/utils/tf_logger.py +++ b/utils/tf_logger.py @@ -12,6 +12,8 @@ class Logger(object): def __init__(self, log_dir, vanilla_logger, skip=False): try: for f in os.listdir(log_dir): + if not f.startswith('events'): + continue os.remove('%s/%s' % (log_dir, f)) except IOError: os.mkdir(log_dir)