DDPG

DDPG_agent.py

@@ -0,0 +1,137 @@
+#######################################################################
+# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
+# Permission given to modify the code as long as you keep this        #
+# declaration at the top                                              #
+#######################################################################
+
+from network import *
+from replay import *
+import pickle
+
+class DDPGAgent:
+    def __init__(self,
+                 task_fn,
+                 actor_network_fn,
+                 critic_network_fn,
+                 actor_optimizer_fn,
+                 critic_optimizer_fn,
+                 replay_fn,
+                 discount,
+                 step_limit,
+                 tau,
+                 exploration_steps,
+                 random_process_fn,
+                 test_interval,
+                 test_repetitions,
+                 tag,
+                 logger):
+        self.task = task_fn()
+        self.actor = actor_network_fn()
+        self.critic = critic_network_fn()
+        self.target_actor = actor_network_fn()
+        self.target_critic = critic_network_fn()
+        self.target_actor.load_state_dict(self.actor.state_dict())
+        self.target_critic.load_state_dict(self.critic.state_dict())
+        self.actor_opt = actor_optimizer_fn(self.actor.parameters())
+        self.critic_opt = critic_optimizer_fn(self.critic.parameters())
+        self.replay = replay_fn()
+        self.step_limit = step_limit
+        self.tau = tau
+        self.logger = logger
+        self.discount = discount
+        self.exploration_steps = exploration_steps
+        self.random_process = random_process_fn()
+        self.criterion = nn.MSELoss()
+        self.test_interval = test_interval
+        self.test_repetitions = test_repetitions
+        self.total_steps = 0
+        self.tag = tag
+
+    def soft_update(self, target, src):
+        for target_param, param in zip(target.parameters(), src.parameters()):
+            target_param.data.copy_(
+                target_param.data * (1.0 - self.tau) + param.data * self.tau
+            )
+
+    def episode(self, deterministic=False):
+        self.random_process.reset_states()
+        state = self.task.reset()
+
+        steps = 0
+        total_reward = 0.0
+        while not self.step_limit or steps < self.step_limit:
+            action = self.actor.predict(np.stack([state])).flatten()
+            if not deterministic:
+                action += self.random_process.sample()
+            action = np.clip(action, -1, 1)
+            next_state, reward, done, info = self.task.step(action)
+            if not deterministic:
+                self.replay.feed([state, action, reward, next_state, int(done)])
+                self.total_steps += 1
+            steps += 1
+            total_reward += reward
+            state = next_state
+
+            if done:
+                break
+
+            if not deterministic and self.total_steps > self.exploration_steps:
+                experiences = self.replay.sample()
+                states, actions, rewards, next_states, terminals = experiences
+                q_next = self.target_critic.predict(next_states, self.target_actor.predict(next_states))
+                terminals = self.critic.to_torch_variable(terminals).unsqueeze(1)
+                rewards = self.critic.to_torch_variable(rewards).unsqueeze(1)
+                q_next = self.discount * q_next * (1 - terminals)
+                q_next.add_(rewards)
+                q_next = Variable(q_next.data)
+                q = self.critic.predict(states, actions)
+                critic_loss = self.criterion(q, q_next)
+
+                self.critic.zero_grad()
+                critic_loss.backward()
+                self.critic_opt.step()
+
+                actor_loss = -self.critic.predict(states, self.actor.predict(states, False))
+                actor_loss = actor_loss.mean()
+
+                self.actor.zero_grad()
+                actor_loss.backward()
+                self.actor_opt.step()
+
+                self.soft_update(self.target_actor, self.actor)
+                self.soft_update(self.target_critic, self.critic)
+
+        return total_reward
+
+    def save(self, file_name):
+        with open(file_name, 'wb') as f:
+            pickle.dump(self.actor.state_dict(), f)
+
+    def run(self):
+        window_size = 100
+        ep = 0
+        rewards = []
+        avg_test_rewards = []
+        while True:
+            ep += 1
+            reward = self.episode()
+            rewards.append(reward)
+            avg_reward = np.mean(rewards[-window_size:])
+            self.logger.info('episode %d, reward %f, avg reward %f, total steps %d' % (
+                ep, reward, avg_reward, self.total_steps))
+
+            if self.test_interval and ep % self.test_interval == 0:
+                self.logger.info('Testing...')
+                self.save('data/%sdqn-model-%s.bin' % (self.tag, self.task.name))
+                test_rewards = []
+                for _ in range(self.test_repetitions):
+                    test_rewards.append(self.episode(True))
+                avg_reward = np.mean(test_rewards)
+                avg_test_rewards.append(avg_reward)
+                self.logger.info('Avg reward %f(%f)' % (
+                    avg_reward, np.std(test_rewards) / np.sqrt(self.test_repetitions)))
+                with open('data/%sdqn-statistics-%s.bin' % (self.tag, self.task.name), 'wb') as f:
+                    pickle.dump({'rewards': rewards,
+                                 'test_rewards': avg_test_rewards}, f)
+                if avg_reward > self.task.success_threshold:
+                    break

dqn_agent.py

@@ -89,7 +89,7 @@ class DQNAgent:
                     q_next, _ = q_next.max(1)
                 terminals = self.learning_network.to_torch_variable(terminals).unsqueeze(1)
                 rewards = self.learning_network.to_torch_variable(rewards).unsqueeze(1)
-                q_next = q_next * (1 - terminals)
+                q_next = self.discount * q_next * (1 - terminals)
                 q_next.add_(rewards)
                 actions = self.learning_network.to_torch_variable(actions, 'int64').unsqueeze(1)
                 q = self.learning_network.predict(states)
@@ -124,7 +124,7 @@ class DQNAgent:
             self.logger.info('episode %d, epsilon %f, reward %f, avg reward %f, total steps %d' % (
                 ep, self.policy.epsilon, reward, avg_reward, self.total_steps))
 
-            if self.test_repetitions and ep % self.test_interval == 0:
+            if self.test_interval and ep % self.test_interval == 0:
                 self.logger.info('Testing...')
                 self.save('data/%sdqn-model-%s.bin' % (self.tag, self.task.name))
                 test_rewards = []

main.py

@@ -1,7 +1,9 @@
 from async_agent import *
 from dqn_agent import *
+from DDPG_agent import *
 import logging
 import traceback
+from random_process import *
 
 def dqn_cart_pole():
     config = dict()
@@ -141,6 +143,51 @@ def a3c_pixel_atari(name):
     agent.tag = ''
     agent.run()
 
+def ddpg_montain_car():
+    config = dict()
+    config['task_fn'] = lambda: ContinuousMountainCar()
+    config['actor_network_fn'] = lambda: DDPGActorNet(2, 1)
+    config['critic_network_fn'] = lambda: DDPGCriticNet(2, 1)
+    config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4)
+    config['critic_optimizer_fn'] =\
+        lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
+    config['replay_fn'] = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
+    config['discount'] = 0.99
+    config['step_limit'] = 2500
+    config['tau'] = 0.001
+    config['exploration_steps'] = 100
+    config['random_process_fn'] = lambda: OrnsteinUhlenbeckProcess(theta=0.15, sigma=0.2)
+    config['test_interval'] = 50
+    config['test_repetitions'] = 10
+    config['tag'] = ''
+    config['logger'] = gym.logger
+    agent = DDPGAgent(**config)
+    agent.run()
+
+def ddpg_pendulum():
+    action_dim = 1
+    state_dim = 3
+    config = dict()
+    config['task_fn'] = lambda: Pendulum()
+    config['actor_network_fn'] = lambda: DDPGActorNet(state_dim, action_dim)
+    config['critic_network_fn'] = lambda: DDPGCriticNet(state_dim, action_dim)
+    config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4)
+    config['critic_optimizer_fn'] =\
+        lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
+    config['replay_fn'] = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
+    config['discount'] = 0.99
+    config['step_limit'] = 200
+    config['tau'] = 0.001
+    config['exploration_steps'] = 100
+    config['random_process_fn'] = \
+        lambda: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2)
+    config['test_interval'] = 10
+    config['test_repetitions'] = 10
+    config['tag'] = ''
+    config['logger'] = gym.logger
+    agent = DDPGAgent(**config)
+    agent.run()
+
 if __name__ == '__main__':
     gym.logger.setLevel(logging.DEBUG)
     # gym.logger.setLevel(logging.INFO)
@@ -150,9 +197,12 @@ if __name__ == '__main__':
     # a3c_cart_pole()
 
     # dqn_pixel_atari('PongNoFrameskip-v3')
-    async_pixel_atari('PongNoFrameskip-v3')
+    # async_pixel_atari('PongNoFrameskip-v3')
     # a3c_pixel_atari('PongNoFrameskip-v3')
 
     # dqn_pixel_atari('BreakoutNoFrameskip-v3')
     # async_pixel_atari('BreakoutNoFrameskip-v3')
     # a3c_pixel_atari('BreakoutNoFrameskip-v3')
+
+    # ddpg_montain_car()
+    ddpg_pendulum()

network.py

@@ -21,6 +21,8 @@ class BasicNet:
             self.cuda()
 
     def to_torch_variable(self, x, dtype='float32'):
+        if isinstance(x, Variable):
+            return x
         if not isinstance(x, torch.FloatTensor):
             x = torch.from_numpy(np.asarray(x, dtype=dtype))
         if self.gpu:
@@ -263,3 +265,79 @@ class OpenAIConvNet(nn.Module, VanillaNet):
         y = y.view(y.size(0), -1)
         phi = F.elu(self.layer5(y))
         return self.fc6(phi)
+
+class DDPGActorNet(nn.Module, BasicNet):
+    def __init__(self,
+                 state_dim,
+                 action_dim,
+                 gpu=False):
+        super(DDPGActorNet, self).__init__()
+        self.layer1 = nn.Linear(state_dim, 400)
+        self.layer2 = nn.Linear(400, 300)
+        self.layer3 = nn.Linear(300, action_dim)
+        BasicNet.__init__(self, None, False, False)
+        self.init_weights()
+
+    def init_weights(self):
+        bound = 3e-3
+        self.layer3.weight.data.uniform_(-bound, bound)
+        # self.layer3.bias.data.uniform_(-bound, bound)
+
+        def fanin(size):
+            v = 1.0 / np.sqrt(size[1])
+            return torch.FloatTensor(size).uniform_(-v, v)
+
+        self.layer1.weight.data = fanin(self.layer1.weight.data.size())
+        # self.layer1.bias.data = fanin(self.layer1.bias.data.size())
+        self.layer2.weight.data = fanin(self.layer2.weight.data.size())
+        # self.layer2.bias.data = fanin(self.layer2.bias.data.size())
+
+    def forward(self, x):
+        x = self.to_torch_variable(x)
+        x = F.relu(self.layer1(x))
+        x = F.relu(self.layer2(x))
+        x = F.tanh(self.layer3(x))
+        return x
+
+    def predict(self, x, to_numpy=True):
+        y = self.forward(x)
+        if to_numpy:
+            y = y.cpu().data.numpy()
+        return y
+
+class DDPGCriticNet(nn.Module, BasicNet):
+    def __init__(self,
+                 state_dim,
+                 action_dim,
+                 gpu=False):
+        super(DDPGCriticNet, self).__init__()
+        self.layer1 = nn.Linear(state_dim, 400)
+        self.layer2 = nn.Linear(400 + action_dim, 300)
+        self.layer3 = nn.Linear(300, 1)
+        BasicNet.__init__(self, None, False, False)
+        self.init_weights()
+
+    def init_weights(self):
+        bound = 3e-3
+        self.layer3.weight.data.uniform_(-bound, bound)
+        # self.layer3.bias.data.uniform_(-bound, bound)
+
+        def fanin(size):
+            v = 1.0 / np.sqrt(size[1])
+            return torch.FloatTensor(size).uniform_(-v, v)
+
+        self.layer1.weight.data = fanin(self.layer1.weight.data.size())
+        # self.layer1.bias.data = fanin(self.layer1.bias.data.size())
+        self.layer2.weight.data = fanin(self.layer2.weight.data.size())
+        # self.layer2.bias.data = fanin(self.layer2.bias.data.size())
+
+    def forward(self, x, action):
+        x = self.to_torch_variable(x)
+        action = self.to_torch_variable(action)
+        x = F.relu(self.layer1(x))
+        x = F.relu(self.layer2(torch.cat([x, action], dim=1)))
+        x = self.layer3(x)
+        return x
+
+    def predict(self, x, action):
+        return self.forward(x, action)

random_process.py

@@ -0,0 +1,49 @@
+# copy from https://github.com/ghliu/pytorch-ddpg/blob/master/random_process.py
+import numpy as np
+
+# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/random.py
+
+class RandomProcess(object):
+    def reset_states(self):
+        pass
+
+class AnnealedGaussianProcess(RandomProcess):
+    def __init__(self, mu, sigma, sigma_min, n_steps_annealing):
+        self.mu = mu
+        self.sigma = sigma
+        self.n_steps = 0
+
+        if sigma_min is not None:
+            self.m = -float(sigma - sigma_min) / float(n_steps_annealing)
+            self.c = sigma
+            self.sigma_min = sigma_min
+        else:
+            self.m = 0.
+            self.c = sigma
+            self.sigma_min = sigma
+
+    @property
+    def current_sigma(self):
+        sigma = max(self.sigma_min, self.m * float(self.n_steps) + self.c)
+        return sigma
+
+
+# Based on http://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab
+class OrnsteinUhlenbeckProcess(AnnealedGaussianProcess):
+    def __init__(self, theta, mu=0., sigma=1., dt=1e-2, x0=None, size=1, sigma_min=None, n_steps_annealing=1000):
+        super(OrnsteinUhlenbeckProcess, self).__init__(mu=mu, sigma=sigma, sigma_min=sigma_min, n_steps_annealing=n_steps_annealing)
+        self.theta = theta
+        self.mu = mu
+        self.dt = dt
+        self.x0 = x0
+        self.size = size
+        self.reset_states()
+
+    def sample(self):
+        x = self.x_prev + self.theta * (self.mu - self.x_prev) * self.dt + self.current_sigma * np.sqrt(self.dt) * np.random.normal(size=self.size)
+        self.x_prev = x
+        self.n_steps += 1
+        return x
+
+    def reset_states(self):
+        self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size)

replay.py

@@ -48,3 +48,47 @@ class Replay:
                 self.rewards[sampled_indices],
                 self.next_states[sampled_indices],
                 self.terminals[sampled_indices]]
+
+class HighDimActionReplay:
+    def __init__(self, memory_size, batch_size, dtype=np.float32):
+        self.memory_size = memory_size
+        self.batch_size = batch_size
+        self.dtype = dtype
+
+        self.states = None
+        self.actions = None
+        self.rewards = np.empty(self.memory_size)
+        self.next_states = None
+        self.terminals = np.empty(self.memory_size, dtype=np.int8)
+
+        self.pos = 0
+        self.full = False
+
+
+    def feed(self, experience):
+        state, action, reward, next_state, done = experience
+
+        if self.states is None:
+            self.states = np.empty((self.memory_size, ) + state.shape, dtype=self.dtype)
+            self.actions = np.empty((self.memory_size, ) + action.shape)
+            self.next_states = np.empty((self.memory_size, ) + state.shape, dtype=self.dtype)
+
+        self.states[self.pos][:] = state
+        self.actions[self.pos][:] = action
+        self.rewards[self.pos] = reward
+        self.next_states[self.pos][:] = next_state
+        self.terminals[self.pos] = done
+
+        self.pos += 1
+        if self.pos == self.memory_size:
+            self.full = True
+            self.pos = 0
+
+    def sample(self):
+        upper_bound = self.memory_size if self.full else self.pos
+        sampled_indices = np.random.randint(0, upper_bound, size=self.batch_size)
+        return [self.states[sampled_indices],
+                self.actions[sampled_indices],
+                self.rewards[sampled_indices],
+                self.next_states[sampled_indices],
+                self.terminals[sampled_indices]]

task.py

@@ -27,6 +27,8 @@ class BasicTask:
             next_state = self.normalize_state(next_state)
         return next_state, np.sign(reward), done, info
 
+
+
 class MountainCar(BasicTask):
     name = 'MountainCar-v0'
     success_threshold = -110
@@ -71,3 +73,29 @@ class PixelAtari(BasicTask):
 
     def normalize_state(self, state):
         return np.asarray(state, dtype=np.float32) / 255.0
+
+
+class ContinuousMountainCar(BasicTask):
+    name = 'MountainCarContinuous-v0'
+    success_threshold = 1000
+
+    def __init__(self):
+        BasicTask.__init__(self)
+        self.env = gym.make(self.name)
+        self.env._max_episode_steps = sys.maxsize
+
+
+class Pendulum(BasicTask):
+    name = 'Pendulum-v0'
+    success_threshold = 200
+
+    def __init__(self):
+        BasicTask.__init__(self)
+        self.env = gym.make(self.name)
+        self.env._max_episode_steps = sys.maxsize
+
+    def step(self, action):
+        next_state, reward, done, info = self.env.step(action)
+        if self.normalized_state:
+            next_state = self.normalize_state(next_state)
+        return next_state, reward, done, info