Minor update

agent/A2C_agent.py

@@ -81,7 +81,7 @@ class A2CAgent:
             actions = self.network.tensor(actions, torch.LongTensor).unsqueeze(1)
             next_value = rollout[i + 1][2]
             returns = rewards + config.discount * terminals * returns
-            if config.use_gae:
+            if not config.use_gae:
                 advantages = returns - value.data
             else:
                 td_error = rewards + config.discount * terminals * next_value.data - value.data

main.py

@@ -197,7 +197,7 @@ def a2c_pixel_atari(name):
     config.reward_shift_fn = lambda r: np.sign(r)
     config.policy_fn = SamplePolicy
     config.discount = 0.99
-    config.use_gae = True
+    config.use_gae = False
     config.gae_tau = 0.97
     config.entropy_weight = 0.01
     config.rollout_length = 5
@@ -463,7 +463,7 @@ if __name__ == '__main__':
     # quantile_regression_dqn_cart_pole()
     # async_cart_pole()
     # a3c_cart_pole()
-    # a2c_cart_pole()
+    a2c_cart_pole()
     # a3c_continuous()
     # p3o_continuous()
     # d3pg_continuous()
@@ -472,7 +472,7 @@ if __name__ == '__main__':
 
     # dqn_pixel_atari('PongNoFrameskip-v4')
     # categorical_dqn_pixel_atari('PongNoFrameskip-v4')
-    quantile_regression_dqn_pixel_atari('PongNoFrameskip-v4')
+    # quantile_regression_dqn_pixel_atari('PongNoFrameskip-v4')
     # n_step_dqn_pixel_atari('PongNoFrameskip-v4')
     # async_pixel_atari('PongNoFrameskip-v4')
     # a3c_pixel_atari('PongNoFrameskip-v4')

network/shallow_network.py

@@ -53,7 +53,7 @@ class ActorCriticFCNet(nn.Module, ActorCriticNet):
     def forward(self, x, update_LSTM=True):
         x = self.variable(x)
         x = F.relu(self.fc1(x))
-        phi = self.fc2(x)
+        phi = F.relu(self.fc2(x))
         return phi
 
 class CategoricalFCNet(nn.Module, CategoricalNet):

utils/config.py

@@ -33,7 +33,7 @@ class Config:
         self.update_interval = 1
         self.gradient_clip = 40
         self.entropy_weight = 0.01
-        self.use_gae = True
+        self.use_gae = False
         self.gae_tau = 1.0
         self.noise_decay_interval = 0
         self.target_network_mix = 0.001

utils/tf_logger.py

@@ -12,6 +12,8 @@ class Logger(object):
     def __init__(self, log_dir, vanilla_logger, skip=False):
         try:
             for f in os.listdir(log_dir):
+                if not f.startswith('events'):
+                    continue
                 os.remove('%s/%s' % (log_dir, f))
         except IOError:
             os.mkdir(log_dir)