Remove OU process

component/random_process.py

@@ -1,49 +1,13 @@
-# copy from https://github.com/ghliu/pytorch-ddpg/blob/master/random_process.py
 import numpy as np
 
-# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/random.py
-
 class RandomProcess(object):
     def reset_states(self):
         pass
 
-class AnnealedGaussianProcess(RandomProcess):
-    def __init__(self, mu, sigma, sigma_min, n_steps_annealing):
-        self.mu = mu
-        self.sigma = sigma
-        self.n_steps = 0
-
-        if sigma_min is not None:
-            self.m = -float(sigma - sigma_min) / float(n_steps_annealing)
-            self.c = sigma
-            self.sigma_min = sigma_min
-        else:
-            self.m = 0.
-            self.c = sigma
-            self.sigma_min = sigma
-
-    @property
-    def current_sigma(self):
-        sigma = max(self.sigma_min, self.m * float(self.n_steps) + self.c)
-        return sigma
-
-
-# Based on http://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab
-class OrnsteinUhlenbeckProcess(AnnealedGaussianProcess):
-    def __init__(self, theta, mu=0., sigma=1., dt=1e-2, x0=None, size=1, sigma_min=None, n_steps_annealing=1000):
-        super(OrnsteinUhlenbeckProcess, self).__init__(mu=mu, sigma=sigma, sigma_min=sigma_min, n_steps_annealing=n_steps_annealing)
-        self.theta = theta
-        self.mu = mu
-        self.dt = dt
-        self.x0 = x0
+class GaussianProcess(RandomProcess):
+    def __init__(self, size, std_schedule):
         self.size = size
-        self.reset_states()
+        self.std_schedule = std_schedule
 
     def sample(self):
-        x = self.x_prev + self.theta * (self.mu - self.x_prev) * self.dt + self.current_sigma * np.sqrt(self.dt) * np.random.normal(size=self.size)
-        self.x_prev = x
-        self.n_steps += 1
-        return x
-
-    def reset_states(self):
-        self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size)
+        return np.random.randn(self.size) * self.std_schedule()

main.py

@@ -326,9 +326,7 @@ def ddpg_continuous():
     config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=64)
     config.discount = 0.99
     config.state_normalizer = RunningStatsNormalizer()
-    config.random_process_fn = \
-        lambda action_dim: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.3,
-                                         n_steps_annealing=1000000)
+    config.random_process_fn = lambda action_dim: GaussianProcess(action_dim, LinearSchedule(0.3, 0, 1e6))
     config.min_memory_size = 64
     config.target_network_mix = 1e-3
     config.logger = Logger('./log', logger)