diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 2c71050..07dc46e 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -24,8 +24,6 @@ class DDPGAgent: self.random_process = config.random_process_fn() self.criterion = nn.MSELoss() self.total_steps = 0 - self.epsilon = 1.0 - self.d_epsilon = 1.0 / config.noise_decay_interval self.state_normalizer = Normalizer(self.task.state_dim) self.reward_normalizer = Normalizer(1) @@ -55,8 +53,7 @@ class DDPGAgent: if self.total_steps < config.exploration_steps: action = self.task.random_action() else: - action += max(self.epsilon, config.min_epsilon) * self.random_process.sample() - self.epsilon -= self.d_epsilon + action += self.random_process.sample() next_state, reward, done, info = self.task.step(action) done = (done or (config.max_episode_length and steps >= config.max_episode_length)) next_state = self.state_normalizer(next_state) diff --git a/component/random_process.py b/component/random_process.py index 70caf4a..6e8d252 100644 --- a/component/random_process.py +++ b/component/random_process.py @@ -46,4 +46,4 @@ class OrnsteinUhlenbeckProcess(AnnealedGaussianProcess): return x def reset_states(self): - self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size) \ No newline at end of file + self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size) diff --git a/main.py b/main.py index 6a437a6..87c6ce6 100644 --- a/main.py +++ b/main.py @@ -233,7 +233,9 @@ def p3o_continuous(): def ddpg_continuous(): config = Config() - config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: BipedalWalker() + config.task_fn = lambda: ContinuousLunarLander() # config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1') # config.task_fn = lambda: Roboschool('RoboschoolReacher-v1') task = config.task_fn() @@ -250,10 +252,9 @@ def ddpg_continuous(): config.max_episode_length = task.max_episode_steps config.target_network_mix = 0.001 config.exploration_steps = 100 - config.noise_decay_interval = 10000 - config.min_epsilon = 0.1 config.random_process_fn = \ - lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) + lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2, + n_steps_annealing=10000) config.test_interval = 0 config.test_repetitions = 10 config.save_interval = 50 @@ -269,12 +270,12 @@ if __name__ == '__main__': # a3c_cart_pole() # a3c_continuous() # p3o_continuous() - # ddpg_continuous() + ddpg_continuous() # dqn_fruit() # hrdqn_fruit() - dqn_pixel_atari('PongNoFrameskip-v4') + # dqn_pixel_atari('PongNoFrameskip-v4') # async_pixel_atari('PongNoFrameskip-v4') # a3c_pixel_atari('PongNoFrameskip-v4')