From 20f34935e58c163588c953a78a8a1ce4658e124d Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Fri, 18 May 2018 15:29:17 -0600 Subject: [PATCH 1/2] Update pixel ddpg params --- deep_rl/network/network_bodies.py | 25 +++++++++++++++++++++++++ examples.py | 11 ++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/deep_rl/network/network_bodies.py b/deep_rl/network/network_bodies.py index 7b706ad..c936475 100644 --- a/deep_rl/network/network_bodies.py +++ b/deep_rl/network/network_bodies.py @@ -23,6 +23,19 @@ class NatureConvBody(nn.Module): y = F.relu(self.fc4(y)) return y +class DDPGConvBody(nn.Module): + def __init__(self, in_channels=4): + super(DDPGConvBody, self).__init__() + self.feature_dim = 39 * 39 * 32 + self.conv1 = layer_init(nn.Conv2d(in_channels, 32, kernel_size=3, stride=2)) + self.conv2 = layer_init(nn.Conv2d(32, 32, kernel_size=3)) + + def forward(self, x): + y = F.elu(self.conv1(x)) + y = F.elu(self.conv2(y)) + y = y.view(y.size(0), -1) + return y + class FCBody(nn.Module): def __init__(self, state_dim, hidden_units=(64, 64), gate=F.relu): super(FCBody, self).__init__() @@ -50,6 +63,18 @@ class TwoLayerFCBodyWithAction(nn.Module): phi = self.gate(self.fc2(torch.cat([x, action], dim=1))) return phi +class OneLayerFCBodyWithAction(nn.Module): + def __init__(self, state_dim, action_dim, hidden_units, gate=F.relu): + super(OneLayerFCBodyWithAction, self).__init__() + self.fc_s = layer_init(nn.Linear(state_dim, hidden_units)) + self.fc_a = layer_init(nn.Linear(action_dim, hidden_units)) + self.gate = gate + self.feature_dim = hidden_units * 2 + + def forward(self, x, action): + phi = self.gate(torch.cat([self.fc_s(x), self.fc_a(action)], dim=1)) + return phi + class DummyBody(nn.Module): def __init__(self, state_dim): super(DummyBody, self).__init__() diff --git a/examples.py b/examples.py index 2d23f3b..4c28f6c 100644 --- a/examples.py +++ b/examples.py @@ -363,7 +363,8 @@ def ddpg_low_dim_state(): def ddpg_pixel(): config = Config() log_dir = get_default_log_dir(ddpg_pixel.__name__) - task_fn = lambda **kwargs: PixelBullet('AntBulletEnv-v0', frame_skip=4, **kwargs) + task_fn = lambda **kwargs: PixelBullet('AntBulletEnv-v0', frame_skip=1, + history_length=4, **kwargs) # each bullet environment should be started in a new process, it is a workaround # to the issue of self-collision @@ -371,15 +372,15 @@ def ddpg_pixel(): config.task_fn = lambda: ProcessTask(task_fn) config.evaluation_env = ProcessTask(task_fn, log_dir=log_dir) - phi_body=NatureConvBody() + phi_body=DDPGConvBody() config.network_fn = lambda state_dim, action_dim: DeterministicActorCriticNet( state_dim, action_dim, phi_body=phi_body, - actor_body=FCBody(phi_body.feature_dim, (200, 200), gate=F.relu), - critic_body=TwoLayerFCBodyWithAction(phi_body.feature_dim, action_dim, (200, 200), gate=F.relu), + actor_body=FCBody(phi_body.feature_dim, (50, ), gate=F.tanh), + critic_body=OneLayerFCBodyWithAction(phi_body.feature_dim, action_dim, 50, gate=F.tanh), actor_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-4), critic_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-3), gpu=0) - config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=64) + config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=16) config.discount = 0.99 config.state_normalizer = ImageNormalizer() config.max_steps = 1e7 From 6f029acf4783136a2dfc8c2ea6ea51d6a9b05144 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Fri, 18 May 2018 15:44:46 -0600 Subject: [PATCH 2/2] Tune params --- deep_rl/agent/DDPG_agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deep_rl/agent/DDPG_agent.py b/deep_rl/agent/DDPG_agent.py index bf8f143..ead45ac 100644 --- a/deep_rl/agent/DDPG_agent.py +++ b/deep_rl/agent/DDPG_agent.py @@ -7,6 +7,7 @@ from ..network import * from ..component import * from .BaseAgent import * +import torchvision class DDPGAgent(BaseAgent): def __init__(self, config): @@ -47,6 +48,7 @@ class DDPGAgent(BaseAgent): if not deterministic: action += self.random_process.sample() next_state, reward, done, info = self.task.step(action) + # torchvision.utils.save_image(torch.tensor(np.asarray(next_state)).unsqueeze(1), 'data/image/%s.png' % get_time_str()) next_state = self.config.state_normalizer(next_state) total_reward += reward reward = self.config.reward_normalizer(reward)