Update DDPG

agent/DDPG_agent.py

@@ -24,8 +24,6 @@ class DDPGAgent:
         self.random_process = config.random_process_fn()
         self.criterion = nn.MSELoss()
         self.total_steps = 0
-        self.epsilon = 1.0
-        self.d_epsilon = 1.0 / config.noise_decay_interval
 
         self.state_normalizer = Normalizer(self.task.state_dim)
         self.reward_normalizer = Normalizer(1)
@@ -55,8 +53,7 @@ class DDPGAgent:
                 if self.total_steps < config.exploration_steps:
                     action = self.task.random_action()
                 else:
-                    action += max(self.epsilon, config.min_epsilon) * self.random_process.sample()
-                    self.epsilon -= self.d_epsilon
+                    action += self.random_process.sample()
             next_state, reward, done, info = self.task.step(action)
             done = (done or (config.max_episode_length and steps >= config.max_episode_length))
             next_state = self.state_normalizer(next_state)

component/random_process.py

@@ -46,4 +46,4 @@ class OrnsteinUhlenbeckProcess(AnnealedGaussianProcess):
         return x
 
     def reset_states(self):
-        self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size)
+        self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size)

main.py

@@ -233,7 +233,9 @@ def p3o_continuous():
 
 def ddpg_continuous():
     config = Config()
-    config.task_fn = lambda: Pendulum()
+    # config.task_fn = lambda: Pendulum()
+    # config.task_fn = lambda: BipedalWalker()
+    config.task_fn = lambda: ContinuousLunarLander()
     # config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1')
     # config.task_fn = lambda: Roboschool('RoboschoolReacher-v1')
     task = config.task_fn()
@@ -250,10 +252,9 @@ def ddpg_continuous():
     config.max_episode_length = task.max_episode_steps
     config.target_network_mix = 0.001
     config.exploration_steps = 100
-    config.noise_decay_interval = 10000
-    config.min_epsilon = 0.1
     config.random_process_fn = \
-        lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
+        lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2,
+                                         n_steps_annealing=10000)
     config.test_interval = 0
     config.test_repetitions = 10
     config.save_interval = 50
@@ -269,12 +270,12 @@ if __name__ == '__main__':
     # a3c_cart_pole()
     # a3c_continuous()
     # p3o_continuous()
-    # ddpg_continuous()
+    ddpg_continuous()
 
     # dqn_fruit()
     # hrdqn_fruit()
 
-    dqn_pixel_atari('PongNoFrameskip-v4')
+    # dqn_pixel_atari('PongNoFrameskip-v4')
     # async_pixel_atari('PongNoFrameskip-v4')
     # a3c_pixel_atari('PongNoFrameskip-v4')