DPPO

agent/PPO_agent.py

@@ -27,6 +27,9 @@ class PPOWorker:
         self.worker_network = config.network_fn()
         self.worker_network.load_state_dict(shared_network.state_dict())
 
+        # self.actor_opt = config.actor_optimizer_fn(self.worker_network.actor.parameters())
+        # self.critic_opt = config.critic_optimizer_fn(self.worker_network.critic.parameters())
+
         self.shared_state_normalizer = extra[0]
         self.state_normalizer = StaticNormalizer(self.task.state_dim)
         self.shared_reward_normalizer = extra[1]
@@ -78,8 +81,6 @@ class PPOWorker:
                 batched_steps += 1
                 episode_length += 1
 
-
-
                 reward = np.asscalar(self.reward_normalizer(np.array([reward])))
                 rewards.append(reward)
 
@@ -112,8 +113,8 @@ class PPOWorker:
         if deterministic:
             return batched_steps, batched_rewards
 
-        with config.steps_lock:
-            config.total_steps.value += replay.memory_size
+        # with config.steps_lock:
+        #     config.total_steps.value += replay.memory_size
 
         self.shared_state_normalizer.offline_stats.merge(self.state_normalizer.online_stats)
         self.state_normalizer.online_stats.zero()
@@ -144,7 +145,7 @@ class PPOWorker:
 
             v = critic_net.predict(states)
             value_loss = 0.5 * (returns - v).pow(2).mean()
-            actor_net_old.load_state_dict(self.actor_net.state_dict())
+            actor_net_old.load_state_dict(actor_net.state_dict())
 
             self.worker_network.zero_grad()
             self.actor_opt.zero_grad()
@@ -153,11 +154,10 @@ class PPOWorker:
             value_loss.backward()
             nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
 
+            self.shared_network.zero_grad()
             for param, worker_param in zip(
                     self.shared_network.parameters(), self.worker_network.parameters()):
-                if param.grad is not None:
-                    break
-                param._grad = worker_param.grad
+                param._grad = worker_param.grad.clone()
             self.actor_opt.step()
             self.critic_opt.step()
 
@@ -168,5 +168,13 @@ class PPOAgent:
         self.config = config
 
     def run(self):
-        worker = PPOWorker(self.config, None, None)
-        worker.rollout()
+        state_normalizer = StaticNormalizer(3)
+        reward_normalizer = StaticNormalizer(1)
+        extra = [state_normalizer, reward_normalizer]
+        shared_network = self.config.network_fn()
+        worker = PPOWorker(self.config, shared_network, extra)
+        i = 0
+        while True:
+            _, rewards = worker.episode()
+            print i, rewards
+            i += 1

agent/async_agent.py

@@ -24,6 +24,7 @@ def train(id, config, learning_network, extra):
         if len(rewards) > 100: rewards.pop(0)
         config.logger.debug('worker %d, episode %d, return %f, avg return %f, episode steps %d, total steps %d' % (
             id, episode, rewards[-1], np.mean(rewards[-100:]), steps, config.total_steps.value))
+        episode += 1
 
 def evaluate(config, task, learning_network, extra):
     test_rewards = []
@@ -32,7 +33,7 @@ def evaluate(config, task, learning_network, extra):
     # config.logger = Logger('./evaluation_log', gym.logger)
     while True:
         steps = config.total_steps.value
-        if steps % config.test_interval == 0:
+        if config.test_interval and steps % config.test_interval == 0:
             worker.worker_network.load_state_dict(learning_network.state_dict())
             with open('data/%s-%s-model-%s.bin' % (
                     config.tag, config.worker.__name__, task.name), 'wb') as f:

async_worker/ppo.py

@@ -72,14 +72,13 @@ class ProximalPolicyOptimization:
                 values.append(value)
                 state, reward, done, _ = self.task.step(action)
                 state = self.state_normalizer(state)
+                # print state
                 done = (done or (config.max_episode_length and episode_length > config.max_episode_length))
 
                 batched_rewards += reward
                 batched_steps += 1
                 episode_length += 1
 
-
-
                 reward = np.asscalar(self.reward_normalizer(np.array([reward])))
                 rewards.append(reward)
 
@@ -122,7 +121,7 @@ class ProximalPolicyOptimization:
         self.reward_normalizer.online_stats.zero()
 
         for _ in np.arange(self.config.optimize_epochs):
-            # self.worker_network.load_state_dict(self.shared_network.state_dict())
+            self.worker_network.load_state_dict(self.shared_network.state_dict())
 
             states, actions, returns, advantages = replay.sample()
             states = actor_net.to_torch_variable(np.stack(states))
@@ -147,17 +146,16 @@ class ProximalPolicyOptimization:
             actor_net_old.load_state_dict(actor_net.state_dict())
 
             self.worker_network.zero_grad()
-            self.actor_opt.zero_grad()
-            self.critic_opt.zero_grad()
             policy_loss.backward()
             value_loss.backward()
             nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
-            for param, worker_param in zip(
-                    self.shared_network.parameters(), self.worker_network.parameters()):
-                if param.grad is not None:
-                    break
-                param._grad = worker_param.grad
-            self.actor_opt.step()
-            self.critic_opt.step()
+            with config.network_lock:
+                self.shared_network.zero_grad()
+                self.actor_opt.zero_grad()
+                self.critic_opt.zero_grad()
+                for param, worker_param in zip(self.shared_network.parameters(), self.worker_network.parameters()):
+                    param._grad = worker_param.grad.clone()
+                self.actor_opt.step()
+                self.critic_opt.step()
 
         return batched_steps, batched_rewards

main.py

@@ -303,32 +303,33 @@ def hrmsdqn_fruit():
 
 def ppo_pendulum():
     config = Config()
-    # config.task_fn = lambda: Pendulum()
-    config.task_fn = lambda: BipedalWalker()
-    # config.reward_shift_fn = lambda reward: reward / 10
+    config.task_fn = lambda: Pendulum()
+    # config.task_fn = lambda: BipedalWalker()
     task = config.task_fn()
     config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim)
     config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim)
     config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001)
+    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
     config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
 
     config.policy_fn = lambda: GaussianPolicy()
-    config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=64)
+    config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=2048)
     config.worker = ProximalPolicyOptimization
     config.discount = 0.99
     config.gae_tau = 0.97
-    config.max_episode_length = 200
     config.num_workers = 8
     config.test_interval = 1
     config.test_repetitions = 1
+    config.max_episode_length = 200
+    # config.max_episode_length = 999
     config.entropy_weight = 0
     config.gradient_clip = 40
     config.rollout_length = 10000
-    config.optimize_epochs = 10
+    config.optimize_epochs = 1
     config.ppo_ratio_clip = 0.2
     config.logger = Logger('./log', gym.logger)
     agent = AsyncAgent(config)
+    # agent = PPOAgent(config)
     agent.run()
 
 if __name__ == '__main__':

utils/normalizer.py

@@ -13,6 +13,8 @@ class StaticNormalizer:
     def __call__(self, o_):
         o = torch.FloatTensor(o_)
         self.online_stats.feed(o)
+        if self.offline_stats.n[0] == 0:
+            return o_
         std = (self.offline_stats.v + 1e-6) ** .5
         o = (o - self.offline_stats.m) / std
         return o.numpy().reshape(o_.shape)