Update DDPG parameters

agent/DDPG_agent.py

@@ -30,7 +30,6 @@ class DDPGAgent(BaseAgent):
         self.critic_opt = config.critic_optimizer_fn(self.critic.parameters())
         self.replay = config.replay_fn()
         self.random_process = config.random_process_fn(self.task.action_dim)
-        self.criterion = nn.MSELoss()
         self.total_steps = 0
 
     def soft_update(self, target, src):
@@ -89,7 +88,7 @@ class DDPGAgent(BaseAgent):
                 q_next.add_(rewards)
                 q_next = q_next.detach()
                 q = critic.predict(states, actions)
-                critic_loss = self.criterion(q, q_next)
+                critic_loss = (q - q_next).pow(2).mul(0.5).sum(-1).mean()
 
                 self.critic_opt.zero_grad()
                 critic_loss.backward()
@@ -99,7 +98,6 @@ class DDPGAgent(BaseAgent):
 
                 self.actor_opt.zero_grad()
                 policy_loss.backward()
-                torch.nn.utils.clip_grad_value_(actor.parameters(), config.gradient_clip)
                 self.actor_opt.step()
 
                 self.soft_update(self.target_network, self.network)

main.py

@@ -322,7 +322,7 @@ def ddpg_continuous():
     config.critic_network_fn = lambda state_dim, action_dim: DeterministicCriticNet(
         TwoLayerFCBodyWithAction(state_dim, action_dim, [400, 300]))
     config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
-    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
+    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
     config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=64)
     config.discount = 0.99
     config.state_normalizer = RunningStatsNormalizer()
@@ -331,7 +331,6 @@ def ddpg_continuous():
                                          n_steps_annealing=1000000)
     config.min_memory_size = 64
     config.target_network_mix = 1e-3
-    config.gradient_clip = 1.0
     config.logger = Logger('./log', logger)
     run_episodes(DDPGAgent(config))
 
@@ -390,7 +389,7 @@ if __name__ == '__main__':
     # ppo_pixel_atari('BreakoutNoFrameskip-v4')
     # dqn_ram_atari('Breakout-ramNoFrameskip-v4')
 
-    # ddpg_continuous()
+    ddpg_continuous()
     # ppo_continuous()
 
     # action_conditional_video_prediction()