Add random initial action in ddpg (#13)

* Add random initial actions in ddpg

* Add reacher-v2 example of ddpg

scripts/algorithms/common/buffer/priortized_replay_buffer.py

@@ -36,18 +36,17 @@ class PrioritizedReplayBuffer(ReplayBuffer):
         """
 
     def __init__(
-        self, buffer_size: int, batch_size: int, demo: list = None, alpha: float = 0.6
+        self, buffer_size: int, batch_size: int, alpha: float = 0.6
     ):
         """Initialization.
 
         Args:
             buffer_size (int): size of replay buffer for experience
             batch_size (int): size of a batched sampled from replay buffer for training
-            demo (list): demonstration
             alpha (float): alpha parameter for prioritized replay buffer
 
         """
-        super(PrioritizedReplayBuffer, self).__init__(buffer_size, batch_size, demo)
+        super(PrioritizedReplayBuffer, self).__init__(buffer_size, batch_size)
         assert alpha >= 0
         self.buffer_size = buffer_size
         self.alpha = alpha
@@ -62,13 +61,6 @@ class PrioritizedReplayBuffer(ReplayBuffer):
         self.min_tree = MinSegmentTree(tree_capacity)
         self.init_priority = 1.0
 
-        # for init priority of demo
-        if demo:
-            for _ in range(len(demo)):
-                self.sum_tree[self.tree_idx] = self.init_priority ** self.alpha
-                self.min_tree[self.tree_idx] = self.init_priority ** self.alpha
-                self.tree_idx += 1
-
     def add(
         self,
         state: np.ndarray,

scripts/algorithms/common/buffer/replay_buffer.py

@@ -22,7 +22,7 @@ class ReplayBuffer:
 
     """
 
-    def __init__(self, buffer_size: int, batch_size: int, demo: list = None):
+    def __init__(self, buffer_size: int, batch_size: int):
         """Initialize a ReplayBuffer object.
 
         Args:
@@ -31,7 +31,7 @@ class ReplayBuffer:
             demo (list) : demonstration list
 
         """
-        self.buffer = list() if not demo else demo
+        self.buffer: list = list()
         self.buffer_size = buffer_size
         self.batch_size = batch_size
         self.idx = 0
@@ -55,13 +55,14 @@ class ReplayBuffer:
 
     def extend(self, transitions: list):
         """Add experiences to memory."""
-        self.buffer.extend(transitions)
+        for transition in transitions:
+            self.add(*transition)
 
     def sample(
         self
     ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
         """Randomly sample a batch of experiences from memory."""
-        idxs = np.random.randint(0, len(self.buffer), size=self.batch_size)
+        idxs = np.random.choice(len(self.buffer), size=self.batch_size, replace=False)
 
         states, actions, rewards, next_states, dones = [], [], [], [], []
 

scripts/algorithms/common/noise.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""Noise classes for baselines."""
+"""Noise classes for algorithms."""
 
 import copy
 import random
@@ -7,6 +7,31 @@ import random
 import numpy as np
 
 
+class GaussianNoise:
+    """Gaussian Noise.
+
+    Taken from https://github.com/vitchyr/rlkit
+    """
+
+    def __init__(
+        self,
+        min_sigma: float = 1.0,
+        max_sigma: float = 1.0,
+        decay_period: int = 1000000,
+    ):
+        """Initialization."""
+        self.max_sigma = max_sigma
+        self.min_sigma = min_sigma
+        self.decay_period = decay_period
+
+    def sample(self, action_size: int, t: int = 0) -> float:
+        """Get an action with gaussian noise."""
+        sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(
+            1.0, t / self.decay_period
+        )
+        return np.random.normal(0, sigma, size=action_size)
+
+
 class OUNoise:
     """Ornstein-Uhlenbeck process.
 
@@ -15,7 +40,9 @@ class OUNoise:
     ddpg-pendulum/ddpg_agent.py
     """
 
-    def __init__(self, size, mu=0.0, theta=0.15, sigma=0.2):
+    def __init__(
+        self, size: int, mu: float = 0.0, theta: float = 0.15, sigma: float = 0.2
+    ):
         """Initialize parameters and noise process."""
         self.state = np.float64(0.0)
         self.mu = mu * np.ones(size)
@@ -27,7 +54,7 @@ class OUNoise:
         """Reset the internal state (= noise) to mean (mu)."""
         self.state = copy.copy(self.mu)
 
-    def sample(self):
+    def sample(self) -> float:
         """Update internal state and return it as a noise sample."""
         x = self.state
         dx = self.theta * (self.mu - x) + self.sigma * np.array(