Add random initial action in ddpg (#13)

* Add random initial actions in ddpg

* Add reacher-v2 example of ddpg

scripts/algorithms/common/buffer/priortized_replay_buffer.py

@@ -36,18 +36,17 @@ class PrioritizedReplayBuffer(ReplayBuffer):
         """
 
     def __init__(
-        self, buffer_size: int, batch_size: int, demo: list = None, alpha: float = 0.6
+        self, buffer_size: int, batch_size: int, alpha: float = 0.6
     ):
         """Initialization.
 
         Args:
             buffer_size (int): size of replay buffer for experience
             batch_size (int): size of a batched sampled from replay buffer for training
-            demo (list): demonstration
             alpha (float): alpha parameter for prioritized replay buffer
 
         """
-        super(PrioritizedReplayBuffer, self).__init__(buffer_size, batch_size, demo)
+        super(PrioritizedReplayBuffer, self).__init__(buffer_size, batch_size)
         assert alpha >= 0
         self.buffer_size = buffer_size
         self.alpha = alpha
@@ -62,13 +61,6 @@ class PrioritizedReplayBuffer(ReplayBuffer):
         self.min_tree = MinSegmentTree(tree_capacity)
         self.init_priority = 1.0
 
-        # for init priority of demo
-        if demo:
-            for _ in range(len(demo)):
-                self.sum_tree[self.tree_idx] = self.init_priority ** self.alpha
-                self.min_tree[self.tree_idx] = self.init_priority ** self.alpha
-                self.tree_idx += 1
-
     def add(
         self,
         state: np.ndarray,

scripts/algorithms/common/buffer/replay_buffer.py

@@ -22,7 +22,7 @@ class ReplayBuffer:
 
     """
 
-    def __init__(self, buffer_size: int, batch_size: int, demo: list = None):
+    def __init__(self, buffer_size: int, batch_size: int):
         """Initialize a ReplayBuffer object.
 
         Args:
@@ -31,7 +31,7 @@ class ReplayBuffer:
             demo (list) : demonstration list
 
         """
-        self.buffer = list() if not demo else demo
+        self.buffer: list = list()
         self.buffer_size = buffer_size
         self.batch_size = batch_size
         self.idx = 0
@@ -55,13 +55,14 @@ class ReplayBuffer:
 
     def extend(self, transitions: list):
         """Add experiences to memory."""
-        self.buffer.extend(transitions)
+        for transition in transitions:
+            self.add(*transition)
 
     def sample(
         self
     ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
         """Randomly sample a batch of experiences from memory."""
-        idxs = np.random.randint(0, len(self.buffer), size=self.batch_size)
+        idxs = np.random.choice(len(self.buffer), size=self.batch_size, replace=False)
 
         states, actions, rewards, next_states, dones = [], [], [], [], []
 

scripts/algorithms/common/noise.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""Noise classes for baselines."""
+"""Noise classes for algorithms."""
 
 import copy
 import random
@@ -7,6 +7,31 @@ import random
 import numpy as np
 
 
+class GaussianNoise:
+    """Gaussian Noise.
+
+    Taken from https://github.com/vitchyr/rlkit
+    """
+
+    def __init__(
+        self,
+        min_sigma: float = 1.0,
+        max_sigma: float = 1.0,
+        decay_period: int = 1000000,
+    ):
+        """Initialization."""
+        self.max_sigma = max_sigma
+        self.min_sigma = min_sigma
+        self.decay_period = decay_period
+
+    def sample(self, action_size: int, t: int = 0) -> float:
+        """Get an action with gaussian noise."""
+        sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(
+            1.0, t / self.decay_period
+        )
+        return np.random.normal(0, sigma, size=action_size)
+
+
 class OUNoise:
     """Ornstein-Uhlenbeck process.
 
@@ -15,7 +40,9 @@ class OUNoise:
     ddpg-pendulum/ddpg_agent.py
     """
 
-    def __init__(self, size, mu=0.0, theta=0.15, sigma=0.2):
+    def __init__(
+        self, size: int, mu: float = 0.0, theta: float = 0.15, sigma: float = 0.2
+    ):
         """Initialize parameters and noise process."""
         self.state = np.float64(0.0)
         self.mu = mu * np.ones(size)
@@ -27,7 +54,7 @@ class OUNoise:
         """Reset the internal state (= noise) to mean (mu)."""
         self.state = copy.copy(self.mu)
 
-    def sample(self):
+    def sample(self) -> float:
         """Update internal state and return it as a noise sample."""
         x = self.state
         dx = self.theta * (self.mu - x) + self.sigma * np.array(

scripts/algorithms/ddpg/agent.py

@@ -38,6 +38,8 @@ class Agent(AbstractAgent):
         actor_optimizer (Optimizer): optimizer for training actor
         critic_optimizer (Optimizer): optimizer for training critic
         curr_state (np.ndarray): temporary storage of the current state
+        total_step (int): total step numbers
+        episode_step (int): step number of the current episode
 
     """
 
@@ -68,20 +70,30 @@ class Agent(AbstractAgent):
         self.hyper_params = hyper_params
         self.curr_state = np.zeros((1,))
         self.noise = noise
+        self.total_step = 0
+        self.episode_step = 0
 
         # load the optimizer and model parameters
         if args.load_from is not None and os.path.exists(args.load_from):
             self.load_params(args.load_from)
 
-        # replay memory
-        self.memory = ReplayBuffer(
-            hyper_params["BUFFER_SIZE"], hyper_params["BATCH_SIZE"]
-        )
+        if not self.args.test:
+            # replay memory
+            self.memory = ReplayBuffer(
+                hyper_params["BUFFER_SIZE"], hyper_params["BATCH_SIZE"]
+            )
 
-    def select_action(self, state: np.ndarray) -> torch.Tensor:
+    def select_action(self, state: np.ndarray) -> np.ndarray:
         """Select an action from the input space."""
         self.curr_state = state
 
+        # if initial random action should be conducted
+        if (
+            self.total_step < self.hyper_params["INITIAL_RANDOM_ACTION"]
+            and not self.args.test
+        ):
+            return self.env.action_space.sample()
+
         state = torch.FloatTensor(state).to(device)
         selected_action = self.actor(state)
 
@@ -89,14 +101,21 @@ class Agent(AbstractAgent):
             selected_action += torch.FloatTensor(self.noise.sample()).to(device)
             selected_action = torch.clamp(selected_action, -1.0, 1.0)
 
-        return selected_action
+        return selected_action.detach().cpu().numpy()
 
-    def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]:
+    def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:
         """Take an action and return the response of the env."""
-        action = action.detach().cpu().numpy()
+        self.total_step += 1
+        self.episode_step += 1
+
         next_state, reward, done, _ = self.env.step(action)
 
-        self.memory.add(self.curr_state, action, reward, next_state, done)
+        if not self.args.test:
+            # if the last state is not a terminal state, store done as false
+            done_bool = (
+                False if self.episode_step == self.args.max_episode_steps else done
+            )
+            self.memory.add(self.curr_state, action, reward, next_state, done_bool)
 
         return next_state, reward, done
 
@@ -171,9 +190,17 @@ class Agent(AbstractAgent):
         total_loss = loss.sum()
 
         print(
-            "[INFO] episode %d total score: %d, total loss: %f\n"
-            "actor_loss: %.3f critic_loss: %.3f\n"
-            % (i, score, total_loss, loss[0], loss[1])  # actor loss  # critic loss
+            "[INFO] episode %d, episode step: %d, total step: %d, total score: %d\n"
+            "total loss: %f actor_loss: %.3f critic_loss: %.3f\n"
+            % (
+                i,
+                self.episode_step,
+                self.total_step,
+                score,
+                total_loss,
+                loss[0],
+                loss[1],
+            )  # actor loss  # critic loss
         )
 
         if self.args.log:
@@ -198,6 +225,7 @@ class Agent(AbstractAgent):
             state = self.env.reset()
             done = False
             score = 0
+            self.episode_step = 0
             loss_episode = list()
 
             while not done:

scripts/algorithms/per/ddpg_agent.py

@@ -75,24 +75,24 @@ class Agent(AbstractAgent):
             self.load_params(args.load_from)
 
         # replay memory
-        self.beta = self.hyper_params["PER_BETA"]
-        self.memory = PrioritizedReplayBuffer(
-            self.hyper_params["BUFFER_SIZE"],
-            self.hyper_params["BATCH_SIZE"],
-            alpha=self.hyper_params["PER_ALPHA"],
-        )
+        if not self.args.test:
+            self.beta = self.hyper_params["PER_BETA"]
+            self.memory = PrioritizedReplayBuffer(
+                self.hyper_params["BUFFER_SIZE"],
+                self.hyper_params["BATCH_SIZE"],
+                alpha=self.hyper_params["PER_ALPHA"],
+            )
 
     def select_action(self, state: np.ndarray) -> torch.Tensor:
         """Select an action from the input space."""
         self.curr_state = state
 
         state = torch.FloatTensor(state).to(device)
+        selected_action = self.actor(state)
 
         if not self.args.test:
-            selected_action = self.actor(state)
             selected_action += torch.FloatTensor(self.noise.sample()).to(device)
-
-        selected_action = torch.clamp(selected_action, -1.0, 1.0)
+            selected_action = torch.clamp(selected_action, -1.0, 1.0)
 
         return selected_action
 
@@ -101,7 +101,8 @@ class Agent(AbstractAgent):
         action = action.detach().cpu().numpy()
         next_state, reward, done, _ = self.env.step(action)
 
-        self.memory.add(self.curr_state, action, reward, next_state, done)
+        if not self.args.test:
+            self.memory.add(self.curr_state, action, reward, next_state, done)
 
         return next_state, reward, done
 

scripts/examples/lunarlander_continuous_v2/ddpg.py

@@ -28,6 +28,7 @@ hyper_params = {
     "OU_NOISE_THETA": 0.0,
     "OU_NOISE_SIGMA": 0.0,
     "WEIGHT_DECAY": 1e-6,
+    "INITIAL_RANDOM_ACTION": 10000,
 }
 
 

scripts/examples/reacher-v2/ddpg.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""Run module for DDPG on Reacher-v2.
+
+- Author: Curt Park
+- Contact: curt.park@medipixel.io
+"""
+
+import argparse
+
+import gym
+import torch
+import torch.optim as optim
+
+from algorithms.common.networks.mlp import MLP
+from algorithms.common.noise import OUNoise
+from algorithms.ddpg.agent import Agent
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# hyper parameters
+hyper_params = {
+    "GAMMA": 0.99,
+    "TAU": 1e-3,
+    "BUFFER_SIZE": int(1e5),
+    "BATCH_SIZE": 128,
+    "LR_ACTOR": 1e-3,
+    "LR_CRITIC": 1e-3,
+    "OU_NOISE_THETA": 0.0,
+    "OU_NOISE_SIGMA": 0.0,
+    "WEIGHT_DECAY": 1e-6,
+    "INITIAL_RANDOM_ACTION": 10000,
+}
+
+
+def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int):
+    """Run training or test.
+
+    Args:
+        env (gym.Env): openAI Gym environment with continuous action space
+        args (argparse.Namespace): arguments including training settings
+        state_dim (int): dimension of states
+        action_dim (int): dimension of actions
+
+    """
+    hidden_sizes_actor = [256, 256]
+    hidden_sizes_critic = [256, 256]
+
+    # create actor
+    actor = MLP(
+        input_size=state_dim,
+        output_size=action_dim,
+        hidden_sizes=hidden_sizes_actor,
+        output_activation=torch.tanh,
+    ).to(device)
+
+    actor_target = MLP(
+        input_size=state_dim,
+        output_size=action_dim,
+        hidden_sizes=hidden_sizes_actor,
+        output_activation=torch.tanh,
+    ).to(device)
+    actor_target.load_state_dict(actor.state_dict())
+
+    # create critic
+    critic = MLP(
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
+    ).to(device)
+
+    critic_target = MLP(
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
+    ).to(device)
+    critic_target.load_state_dict(critic.state_dict())
+
+    # create optimizer
+    actor_optim = optim.Adam(
+        actor.parameters(),
+        lr=hyper_params["LR_ACTOR"],
+        weight_decay=hyper_params["WEIGHT_DECAY"],
+    )
+
+    critic_optim = optim.Adam(
+        critic.parameters(),
+        lr=hyper_params["LR_CRITIC"],
+        weight_decay=hyper_params["WEIGHT_DECAY"],
+    )
+
+    # noise
+    noise = OUNoise(
+        action_dim,
+        theta=hyper_params["OU_NOISE_THETA"],
+        sigma=hyper_params["OU_NOISE_SIGMA"],
+    )
+
+    # make tuples to create an agent
+    models = (actor, actor_target, critic, critic_target)
+    optims = (actor_optim, critic_optim)
+
+    # create an agent
+    agent = Agent(env, args, hyper_params, models, optims, noise)
+
+    # run
+    if args.test:
+        agent.test()
+    else:
+        agent.train()

scripts/requirements.txt

@@ -1,5 +1,5 @@
 gym
 numpy
-torch==1.0.0
+torch==0.4.1
 typing
 wandb

scripts/run_reacher_v2.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""Train or test algorithms on Reacher-v2 of Mujoco.
+
+- Author: Kh Kim
+- Contact: kh.kim@medipixel.io
+"""
+
+import argparse
+import importlib
+
+import gym
+
+import algorithms.common.helper_functions as common_utils
+
+# configurations
+parser = argparse.ArgumentParser(description="Pytorch RL algorithms")
+parser.add_argument(
+    "--seed", type=int, default=777, help="random seed for reproducibility"
+)
+parser.add_argument("--algo", type=str, default="ddpg", help="choose an algorithm")
+parser.add_argument(
+    "--test", dest="test", action="store_true", help="test mode (no training)"
+)
+parser.add_argument(
+    "--load-from", type=str, help="load the saved model and optimizer at the beginning"
+)
+parser.add_argument(
+    "--off-render", dest="render", action="store_false", help="turn off rendering"
+)
+parser.add_argument(
+    "--render-after",
+    type=int,
+    default=0,
+    help="start rendering after the input number of episode",
+)
+parser.add_argument("--log", dest="log", action="store_true", help="turn on logging")
+parser.add_argument("--save-period", type=int, default=200, help="save model period")
+parser.add_argument("--episode-num", type=int, default=20000, help="total episode num")
+parser.add_argument(
+    "--max-episode-steps", type=int, default=-1, help="max episode step"
+)
+parser.add_argument(
+    "--demo-path",
+    type=str,
+    default="data/lunarlander_continuous_demo.pkl",
+    help="demonstration path",
+)
+
+parser.set_defaults(test=False)
+parser.set_defaults(load_from=None)
+parser.set_defaults(render=True)
+parser.set_defaults(log=False)
+args = parser.parse_args()
+
+
+def main():
+    """Main."""
+    # env initialization
+    env = gym.make("Reacher-v2")
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.shape[0]
+
+    # set a random seed
+    common_utils.set_random_seed(args.seed, env)
+
+    # run
+    module_path = "examples.reacher-v2." + args.algo
+    example = importlib.import_module(module_path)
+    example.run(env, args, state_dim, action_dim)
+
+
+if __name__ == "__main__":
+    main()