From 66f5c37407b53b9b5e430028f40a35ad2527f4d3 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Wed, 30 May 2018 11:24:23 -0600 Subject: [PATCH] Update DDPG params and evaluation protocol --- deep_rl/agent/BaseAgent.py | 2 +- deep_rl/agent/DDPG_agent.py | 1 - deep_rl/component/random_process.py | 24 +++++++++++++++++++++--- examples.py | 17 ++++++++++------- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/deep_rl/agent/BaseAgent.py b/deep_rl/agent/BaseAgent.py index c5a607d..dd67998 100644 --- a/deep_rl/agent/BaseAgent.py +++ b/deep_rl/agent/BaseAgent.py @@ -46,7 +46,7 @@ class BaseAgent: if done: break total_rewards += reward - return + self.config.logger.info('evaluation episode return: %f' % (total_rewards)) def evaluation_episodes(self): interval = self.config.evaluation_episodes_interval diff --git a/deep_rl/agent/DDPG_agent.py b/deep_rl/agent/DDPG_agent.py index 8269a46..e5c3751 100644 --- a/deep_rl/agent/DDPG_agent.py +++ b/deep_rl/agent/DDPG_agent.py @@ -51,7 +51,6 @@ class DDPGAgent(BaseAgent): if not deterministic: action += self.random_process.sample() next_state, reward, done, info = self.task.step(action) - # torchvision.utils.save_image(torch.tensor(np.asarray(next_state)).unsqueeze(1), 'data/image/%s.png' % get_time_str()) next_state = self.config.state_normalizer(next_state) total_reward += reward reward = self.config.reward_normalizer(reward) diff --git a/deep_rl/component/random_process.py b/deep_rl/component/random_process.py index 5853e69..786c6fb 100644 --- a/deep_rl/component/random_process.py +++ b/deep_rl/component/random_process.py @@ -5,9 +5,27 @@ class RandomProcess(object): pass class GaussianProcess(RandomProcess): - def __init__(self, size, std_schedule): + def __init__(self, size, std): self.size = size - self.std_schedule = std_schedule + self.std = std def sample(self): - return np.random.randn(self.size) * self.std_schedule() + return np.random.randn(*self.size) * self.std() + +class OrnsteinUhlenbeckProcess(RandomProcess): + def __init__(self, size, std, theta=.15, dt=1e-2, x0=None): + self.theta = theta + self.mu = 0 + self.std = std + self.dt = dt + self.x0 = x0 + self.size = size + self.reset_states() + + def sample(self): + x = self.x_prev + self.theta * (self.mu - self.x_prev) * self.dt + self.std() * np.sqrt(self.dt) * np.random.randn(*self.size) + self.x_prev = x + return x + + def reset_states(self): + self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size) diff --git a/examples.py b/examples.py index 81413a5..85aacdb 100644 --- a/examples.py +++ b/examples.py @@ -335,9 +335,12 @@ def ddpg_low_dim_state(): config = Config() log_dir = get_default_log_dir(ddpg_low_dim_state.__name__) # config.task_fn = lambda **kwargs: Pendulum(log_dir=log_dir) - config.task_fn = lambda **kwargs: Bullet('AntBulletEnv-v0', **kwargs) - # config.task_fn = lambda **kwargs: Roboschool('RoboschoolAnt-v1', **kwargs) + # config.task_fn = lambda **kwargs: Bullet('AntBulletEnv-v0', **kwargs) + config.task_fn = lambda **kwargs: Roboschool('RoboschoolHopper-v1', **kwargs) config.evaluation_env = config.task_fn(log_dir=log_dir) + config.max_steps = int(1e6) + config.evaluation_episodes_interval = int(1e4) + config.evaluation_episodes = 20 config.network_fn = lambda state_dim, action_dim: DeterministicActorCriticNet( state_dim, action_dim, @@ -348,8 +351,8 @@ def ddpg_low_dim_state(): config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=64) config.discount = 0.99 - config.state_normalizer = RunningStatsNormalizer() - config.random_process_fn = lambda action_dim: GaussianProcess(action_dim, LinearSchedule(0.3, 0, 1e6)) + config.random_process_fn = lambda action_dim: OrnsteinUhlenbeckProcess( + size=(action_dim, ), std=LinearSchedule(0.2)) config.min_memory_size = 64 config.target_network_mix = 1e-3 config.logger = get_logger() @@ -374,8 +377,8 @@ def ddpg_pixel(): config.discount = 0.99 config.state_normalizer = ImageNormalizer() config.max_steps = 1e7 - config.random_process_fn = lambda action_dim: GaussianProcess( - action_dim, LinearSchedule(0.3, 0, config.max_steps)) + config.random_process_fn = lambda action_dim: OrnsteinUhlenbeckProcess( + size=(action_dim, ), std=LinearSchedule(0.2)) config.min_memory_size = 64 config.target_network_mix = 1e-3 config.logger = get_logger(file_name=ddpg_pixel.__name__) @@ -435,7 +438,7 @@ if __name__ == '__main__': # ddpg_low_dim_state() # ddpg_pixel() - # ppo_continuous() + ppo_continuous() # action_conditional_video_prediction()