Add random initial action in ddpg (#13)

* Add random initial actions in ddpg

* Add reacher-v2 example of ddpg

scripts/algorithms/common/buffer/priortized_replay_buffer.py

@@ -36,18 +36,17 @@ class PrioritizedReplayBuffer(ReplayBuffer):
         """
 
     def __init__(
-        self, buffer_size: int, batch_size: int, demo: list = None, alpha: float = 0.6
+        self, buffer_size: int, batch_size: int, alpha: float = 0.6
     ):
         """Initialization.
 
         Args:
             buffer_size (int): size of replay buffer for experience
             batch_size (int): size of a batched sampled from replay buffer for training
-            demo (list): demonstration
             alpha (float): alpha parameter for prioritized replay buffer
 
         """
-        super(PrioritizedReplayBuffer, self).__init__(buffer_size, batch_size, demo)
+        super(PrioritizedReplayBuffer, self).__init__(buffer_size, batch_size)
         assert alpha >= 0
         self.buffer_size = buffer_size
         self.alpha = alpha
@@ -62,13 +61,6 @@ class PrioritizedReplayBuffer(ReplayBuffer):
         self.min_tree = MinSegmentTree(tree_capacity)
         self.init_priority = 1.0
 
-        # for init priority of demo
-        if demo:
-            for _ in range(len(demo)):
-                self.sum_tree[self.tree_idx] = self.init_priority ** self.alpha
-                self.min_tree[self.tree_idx] = self.init_priority ** self.alpha
-                self.tree_idx += 1
-
     def add(
         self,
         state: np.ndarray,

scripts/algorithms/common/buffer/replay_buffer.py

@@ -22,7 +22,7 @@ class ReplayBuffer:
 
     """
 
-    def __init__(self, buffer_size: int, batch_size: int, demo: list = None):
+    def __init__(self, buffer_size: int, batch_size: int):
         """Initialize a ReplayBuffer object.
 
         Args:
@@ -31,7 +31,7 @@ class ReplayBuffer:
             demo (list) : demonstration list
 
         """
-        self.buffer = list() if not demo else demo
+        self.buffer: list = list()
         self.buffer_size = buffer_size
         self.batch_size = batch_size
         self.idx = 0
@@ -55,13 +55,14 @@ class ReplayBuffer:
 
     def extend(self, transitions: list):
         """Add experiences to memory."""
-        self.buffer.extend(transitions)
+        for transition in transitions:
+            self.add(*transition)
 
     def sample(
         self
     ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
         """Randomly sample a batch of experiences from memory."""
-        idxs = np.random.randint(0, len(self.buffer), size=self.batch_size)
+        idxs = np.random.choice(len(self.buffer), size=self.batch_size, replace=False)
 
         states, actions, rewards, next_states, dones = [], [], [], [], []