diff --git a/component/random_process.py b/component/random_process.py index 6e8d252..5853e69 100644 --- a/component/random_process.py +++ b/component/random_process.py @@ -1,49 +1,13 @@ -# copy from https://github.com/ghliu/pytorch-ddpg/blob/master/random_process.py import numpy as np -# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/random.py - class RandomProcess(object): def reset_states(self): pass -class AnnealedGaussianProcess(RandomProcess): - def __init__(self, mu, sigma, sigma_min, n_steps_annealing): - self.mu = mu - self.sigma = sigma - self.n_steps = 0 - - if sigma_min is not None: - self.m = -float(sigma - sigma_min) / float(n_steps_annealing) - self.c = sigma - self.sigma_min = sigma_min - else: - self.m = 0. - self.c = sigma - self.sigma_min = sigma - - @property - def current_sigma(self): - sigma = max(self.sigma_min, self.m * float(self.n_steps) + self.c) - return sigma - - -# Based on http://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab -class OrnsteinUhlenbeckProcess(AnnealedGaussianProcess): - def __init__(self, theta, mu=0., sigma=1., dt=1e-2, x0=None, size=1, sigma_min=None, n_steps_annealing=1000): - super(OrnsteinUhlenbeckProcess, self).__init__(mu=mu, sigma=sigma, sigma_min=sigma_min, n_steps_annealing=n_steps_annealing) - self.theta = theta - self.mu = mu - self.dt = dt - self.x0 = x0 +class GaussianProcess(RandomProcess): + def __init__(self, size, std_schedule): self.size = size - self.reset_states() + self.std_schedule = std_schedule def sample(self): - x = self.x_prev + self.theta * (self.mu - self.x_prev) * self.dt + self.current_sigma * np.sqrt(self.dt) * np.random.normal(size=self.size) - self.x_prev = x - self.n_steps += 1 - return x - - def reset_states(self): - self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size) + return np.random.randn(self.size) * self.std_schedule() diff --git a/main.py b/main.py index 1732751..e0d3cf3 100644 --- a/main.py +++ b/main.py @@ -326,9 +326,7 @@ def ddpg_continuous(): config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=64) config.discount = 0.99 config.state_normalizer = RunningStatsNormalizer() - config.random_process_fn = \ - lambda action_dim: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.3, - n_steps_annealing=1000000) + config.random_process_fn = lambda action_dim: GaussianProcess(action_dim, LinearSchedule(0.3, 0, 1e6)) config.min_memory_size = 64 config.target_network_mix = 1e-3 config.logger = Logger('./log', logger)