Update DDPG

agent/DDPG_agent.py

@@ -17,13 +17,13 @@ class DDPGAgent:
     def __init__(self, config):
         self.config = config
         self.task = config.task_fn()
-        self.worker_network = config.network_fn()
-        self.target_network = config.network_fn()
+        self.worker_network = config.network_fn(self.task.state_dim, self.task.action_dim)
+        self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim)
         self.target_network.load_state_dict(self.worker_network.state_dict())
         self.actor_opt = config.actor_optimizer_fn(self.worker_network.actor.parameters())
         self.critic_opt = config.critic_optimizer_fn(self.worker_network.critic.parameters())
         self.replay = config.replay_fn()
-        self.random_process = config.random_process_fn()
+        self.random_process = config.random_process_fn(self.task.action_dim)
         self.criterion = nn.MSELoss()
         self.total_steps = 0
 
@@ -57,8 +57,9 @@ class DDPGAgent:
         total_reward = 0.0
         while True:
             actor.eval()
-            action = actor.predict(np.stack([state])).flatten()
+            action = actor.predict(np.stack([state]), True).flatten()
             if not deterministic:
+                # action += config.gaussian_noise_scale * np.random.randn(*action.shape)
                 action += self.random_process.sample()
             next_state, reward, done, info = self.task.step(action)
             if video_recorder is not None:

component/task.py

@@ -33,9 +33,6 @@ class BasicTask:
         done = (done or self.steps >= self.max_steps)
         return next_state, reward, done, info
 
-    def random_action(self):
-        return self.env.action_space.sample()
-
 class ClassicalControl(BasicTask):
     def __init__(self, name='CartPole-v0', max_steps=200, log_dir=None):
         BasicTask.__init__(self, max_steps)
@@ -100,50 +97,47 @@ class RamAtari(BasicTask):
     def normalize_state(self, state):
         return np.asarray(state) / 255.0
 
-class ContinuousMountainCar(BasicTask):
-    name = 'MountainCarContinuous-v0'
-    success_threshold = 90
-
-    def __init__(self, max_steps=sys.maxsize):
-        BasicTask.__init__(self, max_steps)
-        self.env = gym.make(self.name)
-        self.max_episode_steps = self.env._max_episode_steps
-        self.env._max_episode_steps = sys.maxsize
-        self.action_dim = self.env.action_space.shape[0]
-        self.state_dim = self.env.observation_space.shape[0]
-
 class Pendulum(BasicTask):
     name = 'Pendulum-v0'
     success_threshold = -10
 
-    def __init__(self, max_steps=sys.maxsize):
+    def __init__(self, max_steps=sys.maxsize, log_dir=None):
         BasicTask.__init__(self, max_steps)
         self.env = gym.make(self.name)
         self.action_dim = self.env.action_space.shape[0]
         self.state_dim = self.env.observation_space.shape[0]
+        if log_dir is not None:
+            mkdir(log_dir)
+            self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1()))
 
     def step(self, action):
-        return BasicTask.step(self, np.clip(action, -2, 2))
+        return BasicTask.step(self, np.clip(2 * action, -2, 2))
 
 class Box2DContinuous(BasicTask):
-    def __init__(self, name, max_steps=sys.maxsize):
+    def __init__(self, name, max_steps=sys.maxsize, log_dir=None):
         BasicTask.__init__(self, max_steps)
         self.name = name
         self.env = gym.make(self.name)
         self.action_dim = self.env.action_space.shape[0]
         self.state_dim = self.env.observation_space.shape[0]
+        if log_dir is not None:
+            mkdir(log_dir)
+            self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1()))
 
     def step(self, action):
         return BasicTask.step(self, np.clip(action, -1, 1))
 
 class Roboschool(BasicTask):
-    def __init__(self, name, success_threshold=sys.maxsize, max_steps=sys.maxsize):
+    def __init__(self, name, max_steps=sys.maxsize, log_dir=None):
         import roboschool
         BasicTask.__init__(self, max_steps)
         self.name = name
         self.env = gym.make(self.name)
         self.action_dim = self.env.action_space.shape[0]
         self.state_dim = self.env.observation_space.shape[0]
+        if log_dir is not None:
+            mkdir(log_dir)
+            self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1()))
 
     def step(self, action):
         return BasicTask.step(self, np.clip(action, -1, 1))

main.py

@@ -307,26 +307,20 @@ def ddpg_continuous():
     # config.task_fn = lambda: Roboschool('RoboschoolHopper-v1')
     # config.task_fn = lambda: Roboschool('RoboschoolAnt-v1')
     # config.task_fn = lambda: Roboschool('RoboschoolWalker2d-v1')
-    task = config.task_fn()
-    config.actor_network_fn = lambda: DeterministicActorNet(
-        task.state_dim, task.action_dim, F.tanh, 1, non_linear=F.relu, batch_norm=False, gpu=-1)
-    config.critic_network_fn = lambda: DeterministicCriticNet(
-        task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False, gpu=-1)
-    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
+    actor_network_fn = lambda state_dim, action_dim: DeterministicActorNet(state_dim, action_dim)
+    critic_network_fn = lambda state_dim, action_dim: DeterministicCriticNet(state_dim, action_dim)
+    config.network_fn = lambda state_dim, action_dim: \
+        DisjointActorCriticNet(state_dim, action_dim, actor_network_fn, critic_network_fn)
     config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
     config.critic_optimizer_fn =\
         lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
     config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
     config.discount = 0.99
     config.random_process_fn = \
-        lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2,
+        lambda action_dim: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2,
                                          n_steps_annealing=100000)
-    config.worker = DeterministicPolicyGradient
     config.min_memory_size = 50
     config.target_network_mix = 0.001
-    config.test_interval = 0
-    config.test_repetitions = 1
-    config.gradient_clip = 40
     config.render_episode_freq = 0
     config.logger = Logger('./log', logger)
     run_episodes(DDPGAgent(config))
@@ -352,7 +346,8 @@ if __name__ == '__main__':
     # n_step_dqn_pixel_atari('BreakoutNoFrameskip-v4')
     # dqn_ram_atari('Breakout-ramNoFrameskip-v4')
 
-    # ddpg_continuous()
+    ddpg_continuous()
+
     # dqn_pixel_atari('BreakoutNoFrameskip-v4')
     # dqn_ram_atari('Pong-ramNoFrameskip-v4')
     # acvp.train('PongNoFrameskip-v4')

network/continuous_action_network.py

@@ -10,27 +10,20 @@ class DeterministicActorNet(nn.Module, BasicNet):
     def __init__(self,
                  state_dim,
                  action_dim,
-                 action_gate,
-                 action_scale,
+                 action_gate=F.tanh,
+                 action_scale=1,
                  gpu=-1,
-                 batch_norm=False,
-                 non_linear=F.relu,
+                 non_linear=F.tanh,
                  hidden_size=64):
         super(DeterministicActorNet, self).__init__()
         self.layer1 = nn.Linear(state_dim, hidden_size)
+        self.layer2 = nn.Linear(hidden_size, hidden_size)
         self.layer3 = nn.Linear(hidden_size, action_dim)
         self.action_gate = action_gate
         self.action_scale = action_scale
         self.non_linear = non_linear
-
-        if batch_norm:
-            self.bn1 = nn.BatchNorm1d(hidden_size)
-            self.bn2 = nn.BatchNorm1d(hidden_size)
-        self.layer2 = nn.Linear(hidden_size, hidden_size)
-
-        self.batch_norm = batch_norm
         self.init_weights()
-        BasicNet.__init__(self, gpu, False)
+        BasicNet.__init__(self, gpu)
 
     def init_weights(self):
         bound = 3e-3
@@ -45,16 +38,12 @@ class DeterministicActorNet(nn.Module, BasicNet):
     def forward(self, x):
         x = self.variable(x)
         x = self.non_linear(self.layer1(x))
-        if self.batch_norm:
-            x = self.bn1(x)
         x = self.non_linear(self.layer2(x))
-        if self.batch_norm:
-            x = self.bn2(x)
         x = self.layer3(x)
         x = self.action_scale * self.action_gate(x)
         return x
 
-    def predict(self, x, to_numpy=True):
+    def predict(self, x, to_numpy=False):
         y = self.forward(x)
         if to_numpy:
             y = y.cpu().data.numpy()
@@ -65,22 +54,15 @@ class DeterministicCriticNet(nn.Module, BasicNet):
                  state_dim,
                  action_dim,
                  gpu=-1,
-                 batch_norm=False,
-                 non_linear=F.relu,
+                 non_linear=F.tanh,
                  hidden_size=64):
         super(DeterministicCriticNet, self).__init__()
         self.layer1 = nn.Linear(state_dim, hidden_size)
         self.layer2 = nn.Linear(hidden_size + action_dim, hidden_size)
         self.layer3 = nn.Linear(hidden_size, 1)
         self.non_linear = non_linear
-
-        if batch_norm:
-            self.bn1 = nn.BatchNorm1d(hidden_size)
-            self.bn2 = nn.BatchNorm1d(hidden_size)
-        self.batch_norm = batch_norm
-
         self.init_weights()
-        BasicNet.__init__(self, gpu, False)
+        BasicNet.__init__(self, gpu)
 
     def init_weights(self):
         bound = 3e-3
@@ -96,11 +78,7 @@ class DeterministicCriticNet(nn.Module, BasicNet):
         x = self.variable(x)
         action = self.variable(action)
         x = self.non_linear(self.layer1(x))
-        if self.batch_norm:
-            x = self.bn1(x)
         x = self.non_linear(self.layer2(torch.cat([x, action], dim=1)))
-        if self.batch_norm:
-            x = self.bn2(x)
         x = self.layer3(x)
         return x
 
@@ -111,11 +89,12 @@ class GaussianActorNet(nn.Module, BasicNet):
     def __init__(self,
                  state_dim,
                  action_dim,
-                 action_scale=1.0,
-                 action_gate=None,
+                 action_scale=1,
+                 action_gate=F.tanh,
                  gpu=-1,
                  unit_std=True,
-                 hidden_size=64):
+                 hidden_size=64,
+                 non_linear=F.tanh):
         super(GaussianActorNet, self).__init__()
         self.fc1 = nn.Linear(state_dim, hidden_size)
         self.fc2 = nn.Linear(hidden_size, hidden_size)
@@ -129,13 +108,14 @@ class GaussianActorNet(nn.Module, BasicNet):
         self.unit_std = unit_std
         self.action_scale = action_scale
         self.action_gate = action_gate
+        self.non_linear = non_linear
 
-        BasicNet.__init__(self, gpu, False)
+        BasicNet.__init__(self, gpu)
 
     def forward(self, x):
         x = self.variable(x)
-        phi = F.tanh(self.fc1(x))
-        phi = F.tanh(self.fc2(phi))
+        phi = self.non_linear(self.fc1(x))
+        phi = self.non_linear(self.fc2(phi))
         mean = self.action_mean(phi)
         if self.action_gate is not None:
             mean = self.action_scale * self.action_gate(mean)
@@ -162,17 +142,19 @@ class GaussianCriticNet(nn.Module, BasicNet):
     def __init__(self,
                  state_dim,
                  gpu=-1,
-                 hidden_size=64):
+                 hidden_size=64,
+                 non_linear=F.tanh):
         super(GaussianCriticNet, self).__init__()
         self.fc1 = nn.Linear(state_dim, hidden_size)
         self.fc2 = nn.Linear(hidden_size, hidden_size)
         self.fc_value = nn.Linear(hidden_size, 1)
-        BasicNet.__init__(self, gpu, False)
+        self.non_linear = non_linear
+        BasicNet.__init__(self, gpu)
 
     def forward(self, x):
         x = self.variable(x)
-        phi = F.tanh(self.fc1(x))
-        phi = F.tanh(self.fc2(phi))
+        phi = self.non_linear(self.fc1(x))
+        phi = self.non_linear(self.fc2(phi))
         value = self.fc_value(phi)
         return value
 
@@ -180,9 +162,9 @@ class GaussianCriticNet(nn.Module, BasicNet):
         return self.forward(x)
 
 class DisjointActorCriticNet:
-    def __init__(self, actor_network_fn, critic_network_fn):
-        self.actor = actor_network_fn()
-        self.critic = critic_network_fn()
+    def __init__(self, state_dim, action_dim, actor_network_fn, critic_network_fn):
+        self.actor = actor_network_fn(state_dim, action_dim)
+        self.critic = critic_network_fn(state_dim, action_dim)
 
     def state_dict(self):
         return [self.actor.state_dict(), self.critic.state_dict()]
@@ -191,10 +173,6 @@ class DisjointActorCriticNet:
         self.actor.load_state_dict(state_dicts[0])
         self.critic.load_state_dict(state_dicts[1])
 
-    def share_memory(self):
-        self.actor.share_memory()
-        self.critic.share_memory()
-
     def parameters(self):
         return list(self.actor.parameters()) + list(self.critic.parameters())
 
@@ -205,7 +183,3 @@ class DisjointActorCriticNet:
     def train(self):
         self.actor.train()
         self.critic.train()
-
-    def eval(self):
-        self.actor.eval()
-        self.critic.eval()

utils/config.py

@@ -5,8 +5,6 @@
 #######################################################################
 
 class Config:
-    q_target = 0
-    expected_sarsa_target = 1
     def __init__(self):
         self.task_fn = None
         self.optimizer_fn = None
@@ -41,7 +39,6 @@ class Config:
         self.reward_shift_fn = lambda r: r
         self.reward_weight = 1
         self.hybrid_reward = False
-        self.target_type = self.q_target
         self.episode_limit = 0
         self.min_memory_size = 200
         self.master_fn = None
@@ -59,3 +56,4 @@ class Config:
         self.categorical_v_max = 10
         self.categorical_n_atoms = 51
         self.num_quantiles = 10
+        self.gaussian_noise_scale = 0.3

utils/normalizer.py

@@ -7,29 +7,25 @@ import torch
 import numpy as np
 
 class Normalizer:
-    def __init__(self, o_size):
-        self.stats = SharedStats(o_size)
+    def __init__(self, x_size):
+        self.m = np.zeros(x_size)
+        self.v = np.zeros(x_size)
+        self.n = 1.0
 
-    def __call__(self, o_):
-        if np.isscalar(o_):
-            o = torch.FloatTensor([o_])
-        else:
-            o = torch.FloatTensor(o_)
-        self.stats.feed(o)
-        std = (self.stats.v + 1e-6) ** .5
-        o = (o - self.stats.m) / std
-        o = o.numpy()
-        if np.isscalar(o_):
-            o = np.asscalar(o)
-        else:
-            o = o.reshape(o_.shape)
-        return o
-   
-    def state_dict(self):
-        return self.stats.state_dict()
+    def __call__(self, x):
+        is_scalar = np.isscalar(x)
+        if is_scalar:
+            x = np.asarray([x])
+        new_m = self.m * (self.n / (self.n + 1)) + x / (self.n + 1)
+        self.v = self.v * (self.n / (self.n + 1)) + (x - self.m) * (x - new_m) / (self.n + 1)
+        self.m = new_m
+        self.n += 1
 
-    def load_state_dict(self, saved):
-        self.stats.load_state_dict(saved)
+        std = (self.v + 1e-6) ** .5
+        x = (x - self.m) / std
+        if is_scalar:
+            x = np.asscalar(x)
+        return x
 
 class StaticNormalizer:
     def __init__(self, o_size):