Tune parameters for DDPG

agent/DDPG_agent.py

@@ -29,8 +29,8 @@ class DDPGAgent(BaseAgent):
         self.criterion = nn.MSELoss()
         self.total_steps = 0
 
-        self.state_normalizer = Normalizer(self.task.state_dim)
-        self.reward_normalizer = Normalizer(1)
+        # self.state_normalizer = Normalizer(self.task.state_dim)
+        # self.reward_normalizer = Normalizer(1)
 
     def soft_update(self, target, src):
         for target_param, param in zip(target.parameters(), src.parameters()):
@@ -40,7 +40,7 @@ class DDPGAgent(BaseAgent):
     def episode(self, deterministic=False, video_recorder=None):
         self.random_process.reset_states()
         state = self.task.reset()
-        state = self.state_normalizer(state)
+        # state = self.state_normalizer(state)
 
         config = self.config
         actor = self.worker_network.actor
@@ -59,9 +59,9 @@ class DDPGAgent(BaseAgent):
             next_state, reward, done, info = self.task.step(action)
             if video_recorder is not None:
                 video_recorder.capture_frame()
-            next_state = self.state_normalizer(next_state)
+            # next_state = self.state_normalizer(next_state)
             total_reward += reward
-            reward = self.reward_normalizer(reward)
+            # reward = self.reward_normalizer(reward)
 
             if not deterministic:
                 self.replay.feed([state, action, reward, next_state, int(done)])
@@ -99,6 +99,8 @@ class DDPGAgent(BaseAgent):
                 actor.zero_grad()
                 self.actor_opt.zero_grad()
                 actions.backward(-var_actions.grad.data)
+                for param in actor.parameters():
+                    param.grad.data.clamp(-config.gradient_clip, config.gradient_clip)
                 self.actor_opt.step()
 
                 self.soft_update(self.target_network, self.worker_network)

main.py

@@ -240,15 +240,15 @@ def ddpg_continuous():
         DisjointActorCriticNet(state_dim, action_dim, actor_network_fn, critic_network_fn)
     config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
     config.critic_optimizer_fn =\
-        lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
+        lambda params: torch.optim.Adam(params, lr=1e-4)
     config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
     config.discount = 0.99
     config.random_process_fn = \
-        lambda action_dim: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2,
+        lambda action_dim: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.3,
                                          n_steps_annealing=100000)
-    config.min_memory_size = 50
-    config.target_network_mix = 0.001
-    config.render_episode_freq = 0
+    config.min_memory_size = 64
+    config.target_network_mix = 1e-3
+    config.gradient_clip = 1.0
     config.logger = Logger('./log', logger)
     run_episodes(DDPGAgent(config))
 

network/continuous_action_network.py

@@ -13,12 +13,11 @@ class DeterministicActorNet(nn.Module, BasicNet):
                  action_gate=F.tanh,
                  action_scale=1,
                  gpu=-1,
-                 non_linear=F.tanh,
-                 hidden_size=64):
+                 non_linear=F.tanh):
         super(DeterministicActorNet, self).__init__()
-        self.layer1 = nn.Linear(state_dim, hidden_size)
-        self.layer2 = nn.Linear(hidden_size, hidden_size)
-        self.layer3 = nn.Linear(hidden_size, action_dim)
+        self.layer1 = nn.Linear(state_dim, 300)
+        self.layer2 = nn.Linear(300, 200)
+        self.layer3 = nn.Linear(200, action_dim)
         self.action_gate = action_gate
         self.action_scale = action_scale
         self.non_linear = non_linear
@@ -54,12 +53,11 @@ class DeterministicCriticNet(nn.Module, BasicNet):
                  state_dim,
                  action_dim,
                  gpu=-1,
-                 non_linear=F.tanh,
-                 hidden_size=64):
+                 non_linear=F.tanh):
         super(DeterministicCriticNet, self).__init__()
-        self.layer1 = nn.Linear(state_dim, hidden_size)
-        self.layer2 = nn.Linear(hidden_size + action_dim, hidden_size)
-        self.layer3 = nn.Linear(hidden_size, 1)
+        self.layer1 = nn.Linear(state_dim, 400)
+        self.layer2 = nn.Linear(400 + action_dim, 300)
+        self.layer3 = nn.Linear(300, 1)
         self.non_linear = non_linear
         self.init_weights()
         BasicNet.__init__(self, gpu)
@@ -122,14 +120,6 @@ class GaussianActorNet(nn.Module, BasicNet):
     def predict(self, x):
         return self.forward(x)
 
-    # def log_density(self, x, mean, log_std, std):
-    #     var = std.pow(2)
-    #     log_density = -(x - mean).pow(2) / (2 * var + 1e-5) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std
-    #     return log_density.sum(1)
-    #
-    # def entropy(self, std):
-    #     return 0.5 * (1 + (2 * std.pow(2) * np.pi + 1e-5).log()).sum(1).mean()
-
 class GaussianCriticNet(nn.Module, BasicNet):
     def __init__(self,
                  state_dim,