From 849dcfa5b80b09e7739bbf738aa355036cc0fb55 Mon Sep 17 00:00:00 2001 From: Brian Delhaisse Date: Tue, 7 May 2019 03:43:28 +0200 Subject: [PATCH] update reward processors: account for reward.range --- .../rewards/processors/reward_processor.py | 126 +++++++++++++----- 1 file changed, 92 insertions(+), 34 deletions(-) diff --git a/pyrobolearn/rewards/processors/reward_processor.py b/pyrobolearn/rewards/processors/reward_processor.py index f1daae4..a7ca4a0 100644 --- a/pyrobolearn/rewards/processors/reward_processor.py +++ b/pyrobolearn/rewards/processors/reward_processor.py @@ -21,20 +21,39 @@ __status__ = "Development" class RewardProcessor(Reward): r"""Reward Processor - Wraps the reward and process it. It also acts as a memory of the last received reward signal. + Wraps the reward and process it. It also acts as a memory of the last received reward signal, which can be + accessed via the `value` attribute. Examples: reward = Reward1() + Reward2() reward = RewardProcessor(reward, ) """ - def __init__(self, reward): - super(RewardProcessor, self).__init__() - self.reward = reward - self.value = 0 + def __init__(self, reward, range=None): + """ + Initialize the reward processor. - def compute(self): - self.value = self.reward() + Args: + reward (Reward): reward to process. + range (tuple of float/int, None): range of the reward processor. + """ + super(RewardProcessor, self).__init__() + + # set the reward to process + if not isinstance(reward, Reward): + raise TypeError("Expecting the given 'reward' to be an instance of `Reward`, instead got: " + "{}".format(type(reward))) + self.reward = reward + + # set the range + self.range = self.reward.range if range is None else range + + # set the initial value (randomly) + self.value = np.random.uniform(low=self.range[0], high=self.range[1]) + + def _compute(self): + """Compute the reward and cache its value.""" + self.value = self.reward._compute() return self.value @@ -47,17 +66,20 @@ class ShiftRewardProcessor(RewardProcessor): def __init__(self, reward, x): """ - Initialize the Center Reward Processor. + Initialize the shift reward processor. Args: - reward (Reward): Reward instance to process. + reward (Reward): Reward instance to shift. x (int, float): amount to be shifted. """ - super(ShiftRewardProcessor, self).__init__(reward) + if not isinstance(x, (int, float)): + raise TypeError("Expecting the given 'x' (=the amount to be shifted) to be an int or float, instead got: " + "{}".format(type(x))) self.x = x + super(ShiftRewardProcessor, self).__init__(reward, range=self.reward.range + x) - def compute(self): - reward = self.reward() + def _compute(self): + reward = self.reward._compute() self.value = reward + self.x return self.value @@ -74,7 +96,7 @@ class ClipRewardProcessor(RewardProcessor): Initialize the Clip processor. Args: - reward (Reward): Reward instance to process. + reward (Reward): Reward instance to clip. low (int, float): lower bound high (int, float): higher bound """ @@ -82,8 +104,8 @@ class ClipRewardProcessor(RewardProcessor): self.low = low self.high = high - def compute(self): - reward = self.reward() + def _compute(self): + reward = self.reward._compute() self.value = np.clip(reward, self.low, self.high) return self.value @@ -95,6 +117,12 @@ class CenterRewardProcessor(RewardProcessor): """ def __init__(self, reward): + """ + Initialize the center reward processor. + + Args: + reward (Reward): Reward instance to center. + """ super(CenterRewardProcessor, self).__init__(reward) self.mean = 0 self.N = 0 @@ -104,8 +132,8 @@ class CenterRewardProcessor(RewardProcessor): self.N = 0 self.reward.reset() - def compute(self): - reward = self.reward() + def _compute(self): + reward = self.reward._compute() # update the mean self.mean = self.N / (self.N + 1.) * self.mean + 1. / (self.N + 1) * reward self.N += 1 @@ -124,6 +152,12 @@ class NormalizeRewardProcessor(RewardProcessor): """ def __init__(self, reward): + """ + Initialize the normalizer reward processor. + + Args: + reward (Reward): Reward instance to normalize. + """ super(NormalizeRewardProcessor, self).__init__(reward) self.min = np.infty self.max = -np.infty @@ -133,8 +167,8 @@ class NormalizeRewardProcessor(RewardProcessor): self.max = -np.infty self.reward.reset() - def compute(self): - reward = self.reward() + def _compute(self): + reward = self.reward._compute() self.min = np.minimum(reward, self.min) self.max = np.maximum(reward, self.max) den = self.max - self.min @@ -153,6 +187,14 @@ class StandardizeRewardProcessor(RewardProcessor): """ def __init__(self, reward, epsilon=1.e-4, center=True): + """ + Initialize the standardizer reward processor. + + Args: + reward (Reward): Reward instance to standardize. + epsilon (float): threshold to be added to the standard deviation in order to avoid a division by 0. + center (bool): if we should center the data. + """ super(StandardizeRewardProcessor, self).__init__(reward) self.eps = epsilon self.mean = 0 @@ -166,16 +208,16 @@ class StandardizeRewardProcessor(RewardProcessor): self.N = 0 self.reward.reset() - def compute(self): - reward = self.reward() + def _compute(self): + reward = self.reward._compute() # update the mean old_mean = self.mean self.mean = self.N / (self.N + 1.) * self.mean + 1. / (self.N + 1) * reward # update the var / stddev - self.var = self.N / (self.N + 1) * self.var + \ - 1. / (self.N + 1) * (self.value - old_mean) * (self.value - self.mean) + frac = 1. / (self.N + 1) + self.var = self.N * frac * self.var + frac * (self.value - old_mean) * (self.value - self.mean) std = np.sqrt(self.var) # update total number of data points @@ -196,18 +238,25 @@ class GammaAccumulatedRewardProcessor(RewardProcessor): """ def __init__(self, reward, gamma=0.99): + """ + Initialize the gamma accumulator reward processor. + + Args: + reward (Reward): Reward instance to process. + gamma (float): discount factor. + """ super(GammaAccumulatedRewardProcessor, self).__init__(reward) self.gamma = gamma - self.ret = 0. + self.value = 0. # return value def reset(self): - self.ret = 0. + self.value = 0. self.reward.reset() - def compute(self): - reward = self.reward() - self.ret = reward + self.gamma * self.ret - return self.ret + def _compute(self): + reward = self.reward._compute() + self.value = reward + self.gamma * self.value + return self.value class GammaStandardizeRewardProcessor(RewardProcessor): @@ -218,6 +267,14 @@ class GammaStandardizeRewardProcessor(RewardProcessor): """ def __init__(self, reward, gamma=0.99, epsilon=1.e-4): + """ + Initialize the gamma standardizer reward processor. + + Args: + reward (Reward): Reward instance to process. + gamma (float): discount factor. + epsilon (float): threshold to be added to the standard deviation in order to avoid a division by 0. + """ super(GammaStandardizeRewardProcessor, self).__init__(reward) self.gamma = gamma self.eps = epsilon @@ -233,8 +290,8 @@ class GammaStandardizeRewardProcessor(RewardProcessor): self.N = 0 self.reward.reset() - def compute(self): - reward = self.reward() + def _compute(self): + reward = self.reward._compute() # update return self.ret = reward + self.gamma * self.ret @@ -265,7 +322,7 @@ class ScaleRewardProcessor(RewardProcessor): Initialize the scale reward processor Args: - reward (Reward): reward function to process. + reward (Reward): reward function to scale. x1 (int, float): lower bound of the original reward x2 (int, float): upper bound of the original reward y1 (int, float): lower bound of the final reward @@ -277,8 +334,9 @@ class ScaleRewardProcessor(RewardProcessor): self.y1 = y1 self.y2 = y2 self.ratio = (self.y2 - self.y1) / (self.x2 - self.x1) + self.range = (self.y1, self.y2) - def compute(self): - reward = self.reward() + def _compute(self): + reward = self.reward._compute() self.value = self.y1 + (reward - self.x1) * self.ratio return self.value