A single thread DDPG

agent/DDPG_agent.py

@@ -0,0 +1,109 @@
+#######################################################################
+# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
+# Permission given to modify the code as long as you keep this        #
+# declaration at the top                                              #
+#######################################################################
+
+import numpy as np
+import torch.multiprocessing as mp
+from network import *
+from utils import *
+from component import *
+import pickle
+import os
+import time
+import gym.monitoring
+
+class DDPGAgent:
+    def __init__(self, config):
+        self.config = config
+        self.task = config.task_fn()
+        self.worker_network = config.network_fn()
+        self.target_network = config.network_fn()
+        self.target_network.load_state_dict(self.worker_network.state_dict())
+        self.actor_opt = config.actor_optimizer_fn(self.worker_network.actor.parameters())
+        self.critic_opt = config.critic_optimizer_fn(self.worker_network.critic.parameters())
+        self.replay = config.replay_fn()
+        self.random_process = config.random_process_fn()
+        self.criterion = nn.MSELoss()
+        self.total_steps = 0
+
+        self.state_normalizer = Normalizer(self.task.state_dim)
+        self.reward_normalizer = Normalizer(1)
+
+    def soft_update(self, target, src):
+        for target_param, param in zip(target.parameters(), src.parameters()):
+            target_param.data.copy_(target_param.data * (1.0 - self.config.target_network_mix) +
+                                    param.data * self.config.target_network_mix)
+
+    def save(self, file_name):
+        with open(file_name, 'wb') as f:
+            torch.save(self.worker_network.state_dict(), f)
+
+    def episode(self, deterministic=False, video_recorder=None):
+        self.random_process.reset_states()
+        state = self.task.reset()
+        state = self.state_normalizer(state)
+
+        config = self.config
+        actor = self.worker_network.actor
+        critic = self.worker_network.critic
+        target_actor = self.target_network.actor
+        target_critic = self.target_network.critic
+
+        steps = 0
+        total_reward = 0.0
+        while True:
+            actor.eval()
+            action = actor.predict(np.stack([state])).flatten()
+            if not deterministic:
+                action += self.random_process.sample()
+            next_state, reward, done, info = self.task.step(action)
+            if video_recorder is not None:
+                video_recorder.capture_frame()
+            done = (done or (config.max_episode_length and steps >= config.max_episode_length))
+            next_state = self.state_normalizer(next_state)
+            total_reward += reward
+            reward = self.reward_normalizer(reward)
+
+            if not deterministic:
+                self.replay.feed([state, action, reward, next_state, int(done)])
+                self.total_steps += 1
+
+            steps += 1
+            state = next_state
+
+            if done:
+                break
+
+            if not deterministic and self.replay.size() >= config.min_memory_size:
+                self.worker_network.train()
+                experiences = self.replay.sample()
+                states, actions, rewards, next_states, terminals = experiences
+                q_next = target_critic.predict(next_states, target_actor.predict(next_states))
+                terminals = critic.to_torch_variable(terminals).unsqueeze(1)
+                rewards = critic.to_torch_variable(rewards).unsqueeze(1)
+                q_next = config.discount * q_next * (1 - terminals)
+                q_next.add_(rewards)
+                q_next = q_next.detach()
+                q = critic.predict(states, actions)
+                critic_loss = self.criterion(q, q_next)
+
+                critic.zero_grad()
+                self.critic_opt.zero_grad()
+                critic_loss.backward()
+                self.critic_opt.step()
+
+                actions = actor.predict(states, False)
+                var_actions = Variable(actions.data, requires_grad=True)
+                q = critic.predict(states, var_actions)
+                q.backward(torch.ones(q.size()))
+
+                actor.zero_grad()
+                self.actor_opt.zero_grad()
+                actions.backward(-var_actions.grad.data)
+                self.actor_opt.step()
+
+                self.soft_update(self.target_network, self.worker_network)
+
+        return total_reward, steps

agent/__init__.py

@@ -1,2 +1,3 @@
 from .async_agent import *
 from .DQN_agent import *
+from .DDPG_agent import *

main.py

@@ -274,8 +274,45 @@ def d3pg_continuous():
     agent = AsyncAgent(config)
     agent.run()
 
+def ddpg_continuous():
+    config = Config()
+    # config.task_fn = lambda: Pendulum()
+    # config.task_fn = lambda: ContinuousLunarLander()
+    # config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1')
+    config.task_fn = lambda: Roboschool('RoboschoolReacher-v1')
+    # config.task_fn = lambda: Roboschool('RoboschoolHopper-v1')
+    # config.task_fn = lambda: Roboschool('RoboschoolAnt-v1')
+    # config.task_fn = lambda: Roboschool('RoboschoolWalker2d-v1')
+    # config.task_fn = lambda: BipedalWalker()
+    task = config.task_fn()
+    config.actor_network_fn = lambda: DeterministicActorNet(
+        task.state_dim, task.action_dim, F.tanh, 1, non_linear=F.relu, batch_norm=False)
+    config.critic_network_fn = lambda: DeterministicCriticNet(
+        task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False)
+    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
+    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
+    config.critic_optimizer_fn =\
+        lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
+    config.replay_fn = lambda: SharedReplay(memory_size=1000000, batch_size=64,
+                                            state_shape=(task.state_dim, ), action_shape=(task.action_dim, ))
+    config.discount = 0.99
+    config.max_episode_length = task.max_episode_steps
+    config.random_process_fn = \
+        lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2,
+                                         n_steps_annealing=100000)
+    config.worker = DeterministicPolicyGradient
+    config.min_memory_size = 50
+    config.target_network_mix = 0.001
+    config.test_interval = 0
+    config.test_repetitions = 1
+    config.gradient_clip = 40
+    config.render_episode_freq = 0
+    config.logger = Logger('./log', logger)
+    run_episodes(DDPGAgent(config))
+
 if __name__ == '__main__':
     mkdir('data')
+    mkdir('data/video')
     mkdir('log')
     os.system('export OMP_NUM_THREADS=1')
     # logger.setLevel(logging.DEBUG)
@@ -283,10 +320,11 @@ if __name__ == '__main__':
 
     # dqn_cart_pole()
     # async_cart_pole()
-    a3c_cart_pole()
+    # a3c_cart_pole()
     # a3c_continuous()
     # p3o_continuous()
     # d3pg_continuous()
+    ddpg_continuous()
 
     # dqn_fruit()
     # hrdqn_fruit()

utils/config.py

@@ -11,7 +11,7 @@ class Config:
         self.task_fn = None
         self.optimizer_fn = None
         self.actor_optimizer_fn = None
-        self.critic_optimizer_fn = None
+        self.critic_optimizer_fn = Nonea
         self.network_fn = None
         self.actor_network_fn = None
         self.critic_network_fn = None
@@ -50,3 +50,4 @@ class Config:
         self.save_interval = 0
         self.max_steps = 0
         self.success_threshold = float('inf')
+        self.render_episode_freq = 0

utils/misc.py

@@ -7,6 +7,7 @@
 import numpy as np
 import pickle
 import os
+import gym.monitoring
 
 def run_episodes(agent):
     config = agent.config
@@ -30,6 +31,12 @@ def run_episodes(agent):
                     agent_type, config.tag, agent.task.name), 'wb') as f:
                 pickle.dump([steps, rewards], f)
 
+        if config.render_episode_freq and ep % config.render_episode_freq == 0:
+            video_recoder = gym.monitoring.VideoRecorder(
+                env=agent.task.env, base_path='./data/video/%s-%s-%s-%d' % (agent_type, config.tag, agent.task.name, ep))
+            agent.episode(True, video_recoder)
+            video_recoder.close()
+
         if config.episode_limit and ep > config.episode_limit:
             break