Merge branch 'ddpg-pixel'

deep_rl/agent/DDPG_agent.py

@@ -7,6 +7,7 @@
 from ..network import *
 from ..component import *
 from .BaseAgent import *
+import torchvision
 
 class DDPGAgent(BaseAgent):
     def __init__(self, config):
@@ -47,6 +48,7 @@ class DDPGAgent(BaseAgent):
             if not deterministic:
                 action += self.random_process.sample()
             next_state, reward, done, info = self.task.step(action)
+            # torchvision.utils.save_image(torch.tensor(np.asarray(next_state)).unsqueeze(1), 'data/image/%s.png' % get_time_str())
             next_state = self.config.state_normalizer(next_state)
             total_reward += reward
             reward = self.config.reward_normalizer(reward)

deep_rl/network/network_bodies.py

@@ -23,6 +23,19 @@ class NatureConvBody(nn.Module):
         y = F.relu(self.fc4(y))
         return y
 
+class DDPGConvBody(nn.Module):
+    def __init__(self, in_channels=4):
+        super(DDPGConvBody, self).__init__()
+        self.feature_dim = 39 * 39 * 32
+        self.conv1 = layer_init(nn.Conv2d(in_channels, 32, kernel_size=3, stride=2))
+        self.conv2 = layer_init(nn.Conv2d(32, 32, kernel_size=3))
+
+    def forward(self, x):
+        y = F.elu(self.conv1(x))
+        y = F.elu(self.conv2(y))
+        y = y.view(y.size(0), -1)
+        return y
+
 class FCBody(nn.Module):
     def __init__(self, state_dim, hidden_units=(64, 64), gate=F.relu):
         super(FCBody, self).__init__()
@@ -50,6 +63,18 @@ class TwoLayerFCBodyWithAction(nn.Module):
         phi = self.gate(self.fc2(torch.cat([x, action], dim=1)))
         return phi
 
+class OneLayerFCBodyWithAction(nn.Module):
+    def __init__(self, state_dim, action_dim, hidden_units, gate=F.relu):
+        super(OneLayerFCBodyWithAction, self).__init__()
+        self.fc_s = layer_init(nn.Linear(state_dim, hidden_units))
+        self.fc_a = layer_init(nn.Linear(action_dim, hidden_units))
+        self.gate = gate
+        self.feature_dim = hidden_units * 2
+
+    def forward(self, x, action):
+        phi = self.gate(torch.cat([self.fc_s(x), self.fc_a(action)], dim=1))
+        return phi
+
 class DummyBody(nn.Module):
     def __init__(self, state_dim):
         super(DummyBody, self).__init__()

examples.py

@@ -363,7 +363,8 @@ def ddpg_low_dim_state():
 def ddpg_pixel():
     config = Config()
     log_dir = get_default_log_dir(ddpg_pixel.__name__)
-    task_fn = lambda **kwargs: PixelBullet('AntBulletEnv-v0', frame_skip=4, **kwargs)
+    task_fn = lambda **kwargs: PixelBullet('AntBulletEnv-v0', frame_skip=1,
+                                           history_length=4, **kwargs)
 
     # each bullet environment should be started in a new process, it is a workaround
     # to the issue of self-collision
@@ -371,15 +372,15 @@ def ddpg_pixel():
     config.task_fn = lambda: ProcessTask(task_fn)
     config.evaluation_env = ProcessTask(task_fn, log_dir=log_dir)
 
-    phi_body=NatureConvBody()
+    phi_body=DDPGConvBody()
     config.network_fn = lambda state_dim, action_dim: DeterministicActorCriticNet(
         state_dim, action_dim, phi_body=phi_body,
-        actor_body=FCBody(phi_body.feature_dim, (200, 200), gate=F.relu),
-        critic_body=TwoLayerFCBodyWithAction(phi_body.feature_dim, action_dim, (200, 200), gate=F.relu),
+        actor_body=FCBody(phi_body.feature_dim, (50, ), gate=F.tanh),
+        critic_body=OneLayerFCBodyWithAction(phi_body.feature_dim, action_dim, 50, gate=F.tanh),
         actor_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-4),
         critic_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-3), gpu=0)
 
-    config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=64)
+    config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=16)
     config.discount = 0.99
     config.state_normalizer = ImageNormalizer()
     config.max_steps = 1e7