diff --git a/pyrobolearn/algos/ddpg.py b/pyrobolearn/algos/ddpg.py index c3c2aa9..c34d1d5 100755 --- a/pyrobolearn/algos/ddpg.py +++ b/pyrobolearn/algos/ddpg.py @@ -1,5 +1,7 @@ #!/usr/bin/env python -"""Provide the Deep Deterministic Policy Gradient (DDPG) and the Twin Delayed DDPG algorithm. +"""Provide the Deep Deterministic Policy Gradient (DDPG). + +For the Twin Delayed DDPG algorithm, see `pyrobolearn/algos/td3.py` """ import copy @@ -11,6 +13,7 @@ from pyrobolearn.values import QValue from pyrobolearn.exploration import ActionExploration, GaussianActionExploration from pyrobolearn.storages import ExperienceReplay +from pyrobolearn.samplers import BatchRandomSampler from pyrobolearn.estimators import TDQValueReturn from pyrobolearn.losses import MSBELoss, QLoss from pyrobolearn.optimizers import Adam @@ -261,7 +264,7 @@ class DDPG(GradientRLAlgo): if not actions.is_continuous(): raise ValueError("The DDPG assumes that the actions are continuous, however got an action which is not.") - # evaluate target Q-value fct by copying Q-value function approximator + # Set target parameters equal to main parameters q_target = copy.deepcopy(q_value) policy_target = copy.deepcopy(policy) @@ -270,6 +273,7 @@ class DDPG(GradientRLAlgo): # create experience replay storage = ExperienceReplay(observation_shapes=policy.states, action_shapes=policy.actions, capacity=capacity) + sampler = BatchRandomSampler(storage) # create target return estimator estimator = TDQValueReturn(q_value=q_value, policy=policy_target, target_qvalue=q_target, gamma=gamma) diff --git a/pyrobolearn/algos/evaluator.py b/pyrobolearn/algos/evaluator.py index c10eee9..1aa43e7 100644 --- a/pyrobolearn/algos/evaluator.py +++ b/pyrobolearn/algos/evaluator.py @@ -34,7 +34,7 @@ class Evaluator(object): Initialize the Evaluation phase. Args: - estimator (Estimator): estimator used to evaluate the actions performed by the policy. + estimator (Estimator, None): estimator used to evaluate the actions performed by the policy. """ self.estimator = estimator @@ -50,8 +50,8 @@ class Evaluator(object): @estimator.setter def estimator(self, estimator): """Set the estimator.""" - if not isinstance(estimator, Estimator): - raise TypeError("Expecting estimator to be an instance of `Estimator`, instead got: " + if not None and not isinstance(estimator, Estimator): + raise TypeError("Expecting estimator to be an instance of `Estimator` or None, instead got: " "{}".format(type(estimator))) self._estimator = estimator @@ -64,11 +64,12 @@ class Evaluator(object): # Methods # ########### - def evaluate(self): # , storage): + def evaluate(self): """ Evaluate the actions. """ - self.estimator.evaluate(self.storage) + if self.estimator is not None: + self.estimator.evaluate(self.storage) ############# # Operators # diff --git a/pyrobolearn/algos/explorer.py b/pyrobolearn/algos/explorer.py index 0f88d6a..6c5db1d 100644 --- a/pyrobolearn/algos/explorer.py +++ b/pyrobolearn/algos/explorer.py @@ -11,7 +11,7 @@ from pyrobolearn.tasks import RLTask from pyrobolearn.envs import Env from pyrobolearn.policies import Policy from pyrobolearn.exploration import Exploration -from pyrobolearn.storages import RolloutStorage +from pyrobolearn.storages import DictStorage # RolloutStorage from pyrobolearn import logger @@ -113,8 +113,8 @@ class Explorer(object): @storage.setter def storage(self, storage): """Set the storage unit.""" - if not isinstance(storage, RolloutStorage): - raise TypeError("Expecting the storage to be an instance of `RolloutStorage`, instead got: " + if not isinstance(storage, DictStorage): + raise TypeError("Expecting the storage to be an instance of `DictStorage`, instead got: " "{}".format(type(storage))) self._storage = storage diff --git a/pyrobolearn/algos/ppo.py b/pyrobolearn/algos/ppo.py index 8cd9577..77a08a0 100755 --- a/pyrobolearn/algos/ppo.py +++ b/pyrobolearn/algos/ppo.py @@ -15,7 +15,7 @@ from pyrobolearn.actorcritics import ActorCritic from pyrobolearn.exploration import ActionExploration from pyrobolearn.storages import RolloutStorage -from pyrobolearn.samplers import StorageSampler +from pyrobolearn.samplers import BatchRandomSampler from pyrobolearn.estimators import GAE from pyrobolearn.losses import CLIPLoss, ValueLoss, EntropyLoss from pyrobolearn.optimizers import Adam @@ -44,6 +44,28 @@ class PPO(GradientRLAlgo): outside the PPO class, and providing them as input to the constructor and thus privileging composition over inheritance. + Most of the rest of the documentation has been copied-pasted from [3], and is reproduced here for completeness. + If you use this algorithm, please acknowledge / cite [1, 2, 3]. + + Background + ---------- + + "PPO is motivated by the same question as TRPO: how can we take the biggest possible improvement step on a policy + using the data we currently have, without stepping so far that we accidentally cause performance collapse? Where + TRPO tries to solve this problem with a complex second-order method, PPO is a family of first-order methods that + use a few other tricks to keep new policies close to old. PPO methods are significantly simpler to implement, and + empirically seem to perform at least as well as TRPO. + + There are two primary variants of PPO: PPO-Penalty and PPO-Clip. + + * PPO-Penalty approximately solves a KL-constrained update like TRPO, but penalizes the KL-divergence in the + objective function instead of making it a hard constraint, and automatically adjusts the penalty coefficient over + the course of training so that it's scaled appropriately. + + * PPO-Clip doesn't have a KL-divergence term in the objective and doesn't have a constraint at all. Instead + relies on specialized clipping in the objective function to remove incentives for the new policy to get far + from the old policy." [3] + Mathematics ----------- @@ -62,6 +84,16 @@ class PPO(GradientRLAlgo): This algorithm uses an actor-critic model, and a GAE estimator. + Exploration vs Exploitation + --------------------------- + + "PPO trains a stochastic policy in an on-policy way. This means that it explores by sampling actions according to + the latest version of its stochastic policy. The amount of randomness in action selection depends on both initial + conditions and the training procedure. Over the course of training, the policy typically becomes progressively + less random, as the update rule encourages it to exploit rewards that it has already found. This may cause the + policy to get trapped in local optima." [3] + + Properties ---------- @@ -156,7 +188,7 @@ class PPO(GradientRLAlgo): logger.debug('create return estimator (GAE)') estimator = GAE(storage, gamma=gamma, tau=tau) logger.debug('create storage sampler') - sampler = StorageSampler(storage) + sampler = BatchRandomSampler(storage) # create loss logger.debug('create loss') diff --git a/pyrobolearn/algos/sac.py b/pyrobolearn/algos/sac.py new file mode 100755 index 0000000..022d99d --- /dev/null +++ b/pyrobolearn/algos/sac.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python +"""Provide the Soft-Actor Critic algorithm. + +Define the SAC reinforcement learning algorithm. This is a model-free, off-policy, actor-critic method. +""" + +import copy +import collections + +from pyrobolearn.algos.rl_algo import GradientRLAlgo, Explorer, Evaluator, Updater + +from pyrobolearn.policies import Policy +from pyrobolearn.values import Value, QValue +from pyrobolearn.actorcritics import ActorCritic +from pyrobolearn.exploration import ActionExploration, GaussianActionExploration + +from pyrobolearn.storages import ExperienceReplay +from pyrobolearn.samplers import BatchRandomSampler +from pyrobolearn.estimators import TDQValueReturn +from pyrobolearn.losses import MSBELoss, QLoss +from pyrobolearn.optimizers import Adam + +from pyrobolearn.parameters.updater import PolyakAveraging + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse", "OpenAI Spinning Up (Josh Achiam)"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class SAC(GradientRLAlgo): + r"""Soft Actor-Critic + + Type: policy-gradient, off-policy, continuous + + The following documentation is copied-pasted from [2], and is reproduced here for completeness. If you use + this algorithm, please acknowledge / cite [1, 2]. + + + Background + ---------- + + "Soft Actor Critic (SAC) is an algorithm which optimizes a stochastic policy in an off-policy way, forming a + bridge between stochastic policy optimization and DDPG-style approaches. It isn't a direct successor to TD3 + (having been published roughly concurrently), but it incorporates the clipped double-Q trick, and due to the + inherent stochasticity of the policy in SAC, it also winds up benefiting from something like target policy + smoothing. + + A central feature of SAC is entropy regularization. The policy is trained to maximize a trade-off between + expected return and entropy, a measure of randomness in the policy. This has a close connection to the + exploration-exploitation trade-off: increasing entropy results in more exploration, which can accelerate + learning later on. It can also prevent the policy from prematurely converging to a bad local optimum" [2] + + + Key Equations + ------------- + + "To explain Soft Actor Critic, we first have to introduce the entropy-regularized reinforcement learning setting. + In entropy-regularized RL, there are slightly-different equations for value functions. + + * Entropy-Regularized Reinforcement Learning + + Entropy is a quantity which, roughly speaking, says how random a random variable is. If a coin is weighted so that + it almost always comes up heads, it has low entropy; if it's evenly weighted and has a half chance of either + outcome, it has high entropy. + + Let :math:`x` be a random variable with probability mass or density function :math:`P`. The entropy :math:`H` of + :math:`x` is computed from its distribution :math:`P` according to + + .. math:: H(P) = \underE{x \sim P}{-\log P(x)}. + + In entropy-regularized reinforcement learning, the agent gets a bonus reward at each time step proportional to + the entropy of the policy at that timestep. This changes the RL problem to: + + .. math:: + + \pi^* = \arg \max_{\pi} \underE{\tau \sim \pi}{ \sum_{t=0}^{\infty} \gamma^t \bigg( R(s_t, a_t, s_{t+1}) + + \alpha H\left(\pi(\cdot|s_t)\right) \bigg)}, + + where :math:`\alpha > 0` is the trade-off coefficient. (Note: we're assuming an infinite-horizon discounted + setting here, and we'll do the same for the rest of this page.) We can now define the slightly-different value + functions in this setting. :math:`V^{\pi}` is changed to include the entropy bonuses from every timestep: + + .. math:: + + V^{\pi}(s) = \underE{\tau \sim \pi}{ \left. \sum_{t=0}^{\infty} \gamma^t \bigg( R(s_t, a_t, s_{t+1}) + + \alpha H\left(\pi(\cdot|s_t)\right) \bigg) \right| s_0 = s} + + :math:`Q^{\pi}` is changed to include the entropy bonuses from every timestep except the first: + + .. math:: + + Q^{\pi}(s,a) = \underE{\tau \sim \pi}{ \left. \sum_{t=0}^{\infty} \gamma^t R(s_t, a_t, s_{t+1}) + \alpha + \sum_{t=1}^{\infty} \gamma^t H\left(\pi(\cdot|s_t)\right)\right| s_0 = s, a_0 = a} + + With these definitions, :math:`V^{\pi}` and :math:`Q^{\pi}` are connected by: + + .. math:: V^{\pi}(s) = \underE{a \sim \pi}{Q^{\pi}(s,a)} + \alpha H\left(\pi(\cdot|s)\right) + + and the Bellman equation for :math:`Q^{\pi}` is + + .. math:: + + Q^{\pi}(s,a) &= \underE{s' \sim P \\ a' \sim \pi}{R(s,a,s') + \gamma\left(Q^{\pi}(s',a') + \alpha + H\left(\pi(\cdot|s')\right) \right)} \\ &= \underE{s' \sim P}{R(s,a,s') + \gamma V^{\pi}(s')}. + + * Soft Actor-Critic + + SAC concurrently learns a policy :math:`\pi_{\theta}`, two Q-functions :math:`Q_{\phi_1}`, :math:`Q_{\phi_2}`, and + a value function :math:`V_{\psi}`. + + **Learning Q**: the Q-functions are learned by MSBE minimization, using a target value network to form the Bellman + backups. They both use the same target, like in TD3, and have loss functions: + + .. math:: + + L(\phi_i, {\mathcal D}) = \underset{(s,a,r,s',d) \sim {\mathcal D}}{{\mathrm E}}\left[ \Bigg( Q_{\phi_i}(s,a) + - \left(r + \gamma (1 - d) V_{\psi_{\text{targ}}}(s') \right) \Bigg)^2 \right]. + + The target value network, like the target networks in DDPG and TD3, is obtained by polyak averaging the value + network parameters over the course of training. + + **Learning V**: the value function is learned by exploiting (a sample-based approximation of) the connection + between :math:`Q^{\pi}` and :math:`V^{\pi}`. Before we go into the learning rule, let's first rewrite the + connection equation by using the definition of entropy to obtain: + + .. math:: + + V^{\pi}(s) &= \underE{a \sim \pi}{Q^{\pi}(s,a)} + \alpha H\left(\pi(\cdot|s)\right) \\ + &= \underE{a \sim \pi}{Q^{\pi}(s,a) - \alpha \log \pi(a|s)}. + + The RHS is an expectation over actions, so we can approximate it by sampling from the policy: + + .. math:: V^{\pi}(s) \approx Q^{\pi}(s,\tilde{a}) - \alpha \log \pi(\tilde{a}|s), \quad \tilde{a} \sim \pi(\cdot|s). + + SAC sets up a mean-squared-error loss for :math:`V_{\psi}` based on this approximation. But what Q-value do we use? + SAC uses clipped double-Q like TD3 for learning the value function, and takes the minimum Q-value between the two + approximators. So the SAC loss for value function parameters is: + + .. math:: L(\psi, {\mathcal D}) = \underE{s \sim \mathcal{D} \\ \tilde{a} \sim \pi_{\theta}}{\Bigg(V_{\psi}(s) - + \left(\min_{i=1,2} Q_{\phi_i}(s,\tilde{a}) - \alpha \log \pi_{\theta}(\tilde{a}|s) \right)\Bigg)^2}. + + Importantly, we do not use actions from the replay buffer here: these actions are sampled fresh from the current + version of the policy. + + **Learning the Policy**: the policy should, in each state, act to maximize the expected future return plus + expected future entropy. That is, it should maximize :math:`V^{\pi}(s)`, which we expand out (as before) into + + .. math:: \underE{a \sim \pi}{Q^{\pi}(s,a) - \alpha \log \pi(a|s)}. + + The way we optimize the policy makes use of the reparameterization trick, in which a sample from + :math:`\pi_{\theta}(\cdot|s)` is drawn by computing a deterministic function of state, policy parameters, and + independent noise. To illustrate: following the authors of the SAC paper, we use a squashed Gaussian policy, + which means that samples are obtained according to + + .. math:: + + \tilde{a}_{\theta}(s, \xi) = \tanh\left( \mu_{\theta}(s) + \sigma_{\theta}(s) \odot \xi \right), \quad + \xi \sim \mathcal{N}(0, I). + + **Notice**: + + This policy has two key differences from the policies we use in the other policy optimization algorithms: + + 1. The squashing function. The :math:`\tanh` in the SAC policy ensures that actions are bounded to a finite range. + This is absent in the VPG, TRPO, and PPO policies. It also changes the distribution: before the :math:`\tanh` the + SAC policy is a factored Gaussian like the other algorithms' policies, but after the :math:`\tanh` it is not. (You + can still evaluate the log-probabilities of actions in closed form, though: see the paper appendix for details.) + + 2. The way standard deviations are parameterized. In VPG, TRPO, and PPO, we represent the log std devs with + state-independent parameter vectors. In SAC, we represent the log std devs as outputs from the neural network, + meaning that they depend on state in a complex way. SAC with state-independent log std devs, in our experience, + did not work. (Can you think of why? Or better yet: run an experiment to verify?) + + **End of notice** + + The reparameterization trick allows us to rewrite the expectation over actions (which contains a pain point: + the distribution depends on the policy parameters) into an expectation over noise (which removes the pain point: + the distribution now has no dependence on parameters): + + .. math:: + + \underE{a \sim \pi_{\theta}}{Q^{\pi_{\theta}}(s,a) - \alpha \log \pi_{\theta}(a|s)} + = \underE{\xi \sim \mathcal{N}}{Q^{\pi_{\theta}}(s,\tilde{a}_{\theta}(s,\xi)) + - \alpha \log \pi_{\theta}(\tilde{a}_{\theta}(s,\xi)|s)} + + To get the policy loss, the final step is that we need to substitute :math:`Q^{\pi_{\theta}}` with one of our + function approximators. The same as in TD3, we use Q_{\phi_1}. The policy is thus optimized according to + + .. math:: + + \max_{\theta} \underE{s \sim \mathcal{D} \\ \xi \sim \mathcal{N}}{Q_{\phi_1}(s,\tilde{a}_{\theta}(s,\xi)) + - \alpha \log \pi_{\theta}(\tilde{a}_{\theta}(s,\xi)|s)}, + + which is almost the same as the DDPG and TD3 policy optimization, except for the stochasticity and entropy term." + [2] + + + Exploration vs. Exploitation + ---------------------------- + + "SAC trains a stochastic policy with entropy regularization, and explores in an on-policy way. The entropy + regularization coefficient \alpha explicitly controls the explore-exploit tradeoff, with higher :math:`\alpha` + corresponding to more exploration, and lower \alpha corresponding to more exploitation. The right coefficient + (the one which leads to the stablest / highest-reward learning) may vary from environment to environment, and + could require careful tuning. + + At test time, to see how well the policy exploits what it has learned, we remove stochasticity and use the mean + action instead of a sample from the distribution. This tends to improve performance over the original stochastic + policy." [2] + + + Pseudo-algo + ----------- + + Pseudo-algorithm (taken from [2] and reproduce here for completeness):: + 1. Input: initial policy parameters :math:`\theta_0`, Q-function parameters :math:`\phi_1, \phi_2`, + V-function parameters :math:`\psi`, empty replay buffer :math:`D` + 2. Set target parameters equal to main parameters :math:`\psi_{\text{target}} \leftarrow \psi` + 3. repeat: + 4. Observe state :math:`s` and select action :math:`a \sim \pi_{\theta}(\cdot|s)` + 5. Execute :math:`a` in the environment + 6. Observe next state :math:`s'`, reward :math:`r`, and done signal :math:`d` to indicate whether :math:`s'` + is terminal + 7. Store :math:`(s, a, r, s', d)` in replay buffer :math:`D` + 8. If :math:`s'` is terminal, reset environment state. + 9. if it's time to update then: + 10. for j in range(num_updates) do: + 11. Randomly sample a batch of transitions, :math:`B = {(s, a, r, s', d)}` from :math:`D` + 12. Compute targets for Q and V functions: + :math:`y_q(r,s',d) = r + \gamma (1-d) V_{\psi_{\text{targ}}}(s')` + :math:`y_v(s) = \min_{i=1,2} Q_{\phi_i} (s, \tilde{a}) - \alpha \log \pi_{\theta}(\tilde{a}|s)`, + with :math:`\tilde{a} \sim \pi_{\theta}(\cdot|s)` + 13. Update Q-functions by one step of gradient descent using: + :math:`\nabla_{\phi_i} \frac{1}{|B|}\sum_{(s,a,r,s',d) \in B} \left( Q_{\phi,i}(s,a) - + y_q(r,s',d) \right)^2`, for :math:`i=1,2` + 14. Update V-function by one step of gradient descent using: + :math:`\nabla_{\psi} \frac{1}{|B|}\sum_{s \in B} \left( V_{\psi}(s) - y_v(s) \right)^2` + 15. Update policy by one step of gradient ascent using: + :math:`\nabla_{\theta} \frac{1}{|B|}\sum_{s \in B} \Big( Q_{\phi,1}(s, \tilde{a}_{\theta}(s)) + - \alpha \log \pi_{\theta} \left(\left. \tilde{a}_{\theta}(s) | s\right) \Big)`, + where :math:`\tilde{a}_{\theta}(s)` is a sample from :math:`\pi_{\theta}(\cdot|s)` which is + differentiable wrt :math:`\theta` via the reparametrization trick. + 16. Update target value network with: + :math:`\psi_{\text{targ}} &\leftarrow \rho \psi_{\text{targ}} + (1-\rho) \psi` + 17. end for + 18. end if + 19. until convergence + + + References: + [1] "Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor", + Haarnoja et al, 2018 + [2] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/sac.html + [3] RLKit: https://github.com/vitchyr/rlkit/blob/master/rlkit/torch/sac/sac.py + """ + + def __init__(self, task, approximators, gamma=0.99, lr=5e-4, polyak=0.995, capacity=10000, num_workers=1): + """ + Initialize the SAC off-policy RL algorithm. + + Args: + task (RLTask, Env): RL task/env to run + approximators ([Policy, Value, QValue]): approximators to optimize. + gamma (float): discount factor (which is a bias-variance tradeoff). This parameter describes how much + importance has the future rewards we get. + lr (float): learning rate + polyak (float): coefficient in the polyak averaging when updating the target approximators. + capacity (int): capacity of the experience replay storage. + num_workers (int): number of processes / workers to run in parallel + """ + + # check approximators + if not isinstance(approximators, collections.Iterable): + raise TypeError("Expecting the approximators to be a list containing a Policy, a Value, and at least 2 " + "QValues") + policy, value, q_values = None, None, [] + for approximator in approximators: + if isinstance(approximator, Policy): + policy = approximator + elif isinstance(approximator, Value): + value = approximator + elif isinstance(approximator, ActorCritic): + policy = approximator.actor + value = approximator.critic + elif isinstance(approximator, QValue): + q_values.append(approximator) + + if policy is None: + raise TypeError("No policy was given to the algorithm.") + if value is None: + raise TypeError("No value function approximator was given to the algorithm.") + if len(q_values) == 0: + raise TypeError("No Q-value function approximators were given to the algorithm.") + + # set target parameters equal to main parameters for the value function + value_target = copy.deepcopy(value) + + # create experience replay + storage = ExperienceReplay(observation_shapes=policy.states, action_shapes=policy.actions, capacity=capacity) + sampler = BatchRandomSampler(storage) + + # create action exploration + exploration = ActionExploration(policy) + + # create targets + # TODO + + + # create losses + q_loss = MSBELoss(td_return=estimator) + policy_loss = QLoss(q_value=q_values[0], policy=policy) # only the first q-value is used to train the policy + losses = [q_loss, policy_loss] + + # create optimizer + optimizer = Adam(learning_rate=lr) + + # create parameter updater for target value function + params_updater = PolyakAveraging(rho=polyak) + params_updater = (params_updater, value_target) + + # define the 3 main steps in RL: explore, evaluate, and update + explorer = Explorer(task, exploration, storage, num_workers=num_workers) + evaluator = Evaluator(None) + updater = Updater(approximators, sampler, losses, optimizer, updaters=params_updater) + + # initialize RL algorithm + super(SAC, self).__init__(explorer, evaluator, updater) diff --git a/pyrobolearn/algos/td3.py b/pyrobolearn/algos/td3.py new file mode 100755 index 0000000..899ad31 --- /dev/null +++ b/pyrobolearn/algos/td3.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +"""Provide the Twin Delayed Deep Deterministic Policy Gradient (TD3) algorithm. + +The REINFORCE/VPG is a model-free, off-policy, gradient policy-based algorithm that works only for continuous actions. +""" + +import copy + +from pyrobolearn.algos.rl_algo import GradientRLAlgo, Explorer, Evaluator, Updater + +from pyrobolearn.policies import Policy +from pyrobolearn.values import QValue +from pyrobolearn.exploration import ActionExploration, GaussianActionExploration + +from pyrobolearn.storages import ExperienceReplay +from pyrobolearn.samplers import BatchRandomSampler +from pyrobolearn.estimators import TDQValueReturn +from pyrobolearn.losses import MSBELoss, QLoss +from pyrobolearn.optimizers import Adam + +from pyrobolearn.parameters.updater import PolyakAveraging + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse", "OpenAI Spinning Up (Josh Achiam)"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class TD3(GradientRLAlgo): + r"""Twin Delayed Deep Deterministic Policy Gradient + + Type:: model-free, off-policy, gradient-based actor-critic algorithm for continuous only action spaces. + + The documentation has been copied-pasted from [2], and is reproduced here for completeness. If you use this + algorithm, please acknowledge / cite [1, 2]. + + + Background + ---------- + + "While DDPG can achieve great performance sometimes, it is frequently brittle with respect to hyperparameters and + other kinds of tuning. A common failure mode for DDPG is that the learned Q-function begins to dramatically + overestimate Q-values, which then leads to the policy breaking, because it exploits the errors in the Q-function. + Twin Delayed DDPG (TD3) is an algorithm which addresses this issue by introducing three critical tricks: + + 1. Trick One: Clipped Double-Q Learning. TD3 learns two Q-functions instead of one (hence 'twin'), and uses the + smaller of the two Q-values to form the targets in the Bellman error loss functions. + 2. Trick Two: 'Delayed' Policy Updates. TD3 updates the policy (and target networks) less frequently than the + Q-function. The paper recommends one policy update for every two Q-function updates. + 3. Trick Three: Target Policy Smoothing. TD3 adds noise to the target action, to make it harder for the policy + to exploit Q-function errors by smoothing out Q along changes in action. + + Together, these three tricks result in substantially improved performance over baseline DDPG." [2] + + + Key Equations + ------------- + + "TD3 concurrently learns two Q-functions, :math:`Q_{\phi_1}` and :math:`Q_{\phi_2}`, by mean square Bellman error + minimization, in almost the same way that DDPG learns its single Q-function. To show exactly how TD3 does this + and how it differs from normal DDPG, we'll work from the innermost part of the loss function outwards. + + First: target policy smoothing. Actions used to form the Q-learning target are based on the target policy, + :math:`\mu_{\theta_{\text{targ}}}`, but with clipped noise added on each dimension of the action. + After adding the clipped noise, the target action is then clipped to lie in the valid action range (all valid + actions, :math:`a`, satisfy :math:`a_{Low} \leq a \leq a_{High}`). The target actions are thus: + + .. math:: a'(s') = \text{clip}\left(\mu_{\theta_{\text{targ}}}(s') + \text{clip}(\epsilon,-c,c), a_{Low}, + a_{High}\right), \qquad \epsilon \sim \mathcal{N}(0, \sigma) + + Target policy smoothing essentially serves as a regularizer for the algorithm. It addresses a particular failure + mode that can happen in DDPG: if the Q-function approximator develops an incorrect sharp peak for some actions, + the policy will quickly exploit that peak and then have brittle or incorrect behavior. This can be averted by + smoothing out the Q-function over similar actions, which target policy smoothing is designed to do. + + Next: clipped double-Q learning. Both Q-functions use a single target, calculated using whichever of the two + Q-functions gives a smaller target value: + + .. math:: y(r,s',d) = r + \gamma (1 - d) \min_{i=1,2} Q_{\phi_{i, \text{targ}}}(s', a'(s')), + + and then both are learned by regressing to this target: + + .. math:: + + L(\phi_1,{\mathcal D}) = \underE{(s,a,r,s',d) \sim {\mathcal D}}{\left(Q_{\phi_1}(s,a) - y(r,s',d) \right)^2}, + + L(\phi_2,{\mathcal D}) = \underE{(s,a,r,s',d) \sim {\mathcal D}}{\left(Q_{\phi_2}(s,a) - y(r,s',d) \right)^2 }. + + Using the smaller Q-value for the target, and regressing towards that, helps fend off overestimation in the + Q-function. + + Lastly: the policy is learned just by maximizing Q_{\phi_1}: + + .. math:: \max_{\theta} \underset{s \sim {\mathcal D}}{{\mathrm E}}\left[ Q_{\phi_1}(s, \mu_{\theta}(s)) \right], + + which is pretty much unchanged from DDPG. However, in TD3, the policy is updated less frequently than the + Q-functions are. This helps damp the volatility that normally arises in DDPG because of how a policy update changes + the target." [2] + + + Exploration vs. Exploitation + ---------------------------- + + "TD3 trains a deterministic policy in an off-policy way. Because the policy is deterministic, if the agent were to + explore on-policy, in the beginning it would probably not try a wide enough variety of actions to find useful + learning signals. To make TD3 policies explore better, we add noise to their actions at training time, typically + uncorrelated mean-zero Gaussian noise. To facilitate getting higher-quality training data, you may reduce the + scale of the noise over the course of training. (We do not do this in our implementation, and keep noise scale + fixed throughout.) + + At test time, to see how well the policy exploits what it has learned, we do not add noise to the actions." [2] + + + Pseudo-algo: + ----------- + + Pseudo-algorithm (taken from [2] and reproduce here for completeness):: + 1. Input: initial policy parameters :math:`\theta`, Q-function parameters :math:`\phi_1, \phi_2`, empty replay + buffer :math:`D` + 2. Set target parameters equal to main parameters :math:`\theta_{target} \leftarrow \theta`, + :math:`\phi_{target, i} \leftarrow \phi_i` for i=1,2 + 3. repeat: + 4. Observe state :math:`s` and select action :math:`a = clip(\mu_{\theta}(s) + \epsilon, a_{Low}, a_{High})`, + where :math:`\epsilon \sim \mathcal{N}` + 5. Execute :math:`a` in the environment + 6. Observe next state :math:`s'`, reward :math:`r`, and done signal :math:`d` to indicate whether :math:`s'` + is terminal. + 7. Store :math:`(s, a, r, s', d)` in replay buffer :math:`D` + 8. If :math:`s'` is terminal, reset environment state. + 9. if it's time to update then: + 10. for j in range(num_updates) do: + 11. Randomly sample a batch of transitions, :math:`B = {(s, a, r, s', d)}` from :math:`D` + 12. Compute target actions + :math:`a'(s') = clip(\mu_{\theta_target}(s') + clip(\epsilon, -c, c), a_{Low}, a_{High})`, + where :math:`\epsilon \sim \mathcal{N}` + 13. Compute targets + :math:`y(r,s',d) = r + \gamma (1 - d) \min_{i=1,2} Q_{\phi_{target, i}}(s', a'(s))` + 14. Update Q-functions by one step of gradient descent using + :math:`\grad_{\phi_i} \frac{1}{|B|} \sum_{(s,a,r,s',d) \in B} (Q_{\phi_i}(s,a) - y(r,s',d))^2`, + for i=1,2 + 15. if (j % policy_delay) == 0 then: + 16. Update policy by one step of gradient ascent using + :math:`\grad_{\theta} \frac{1}{|B|} \sum_{s \in B} Q_{\phi_1}(s, \mu_{\theta}(s))` + 17. Update target networks with + :math:`\phi_{target,i} \leftarrow \rho \phi_{target,i} + (1 - \rho)\phi_i` for i=1,2 + :math:`\theta_{target} \leftarrow \rho \theta_{target} + (1 - \rho) \theta` + 18. end if + 19. end for + 20. end if + 21. until convergence + + + References:: + [1] "Addressing Function Approximation Error in Actor-Critic Methods", Fujimoto et al., 2018 + [2] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/td3.html + """ + + def __init__(self, task, approximators, gamma=0.99, lr=0.001, polyak=0.995, capacity=10000, num_workers=1): + """ + Initialize the TD3 off-policy RL algorithm. + + Args: + task (RLTask, Env): RL task/env to run + approximators (Policy, [Policy, Value], ActorCritic): approximators to optimize + gamma (float): discount factor (which is a bias-variance tradeoff). This parameter describes how much + importance has the future rewards we get. + lr (float): learning rate. + polyak (float): coefficient in the polyak averaging when updating the target approximators. + capacity (int): capacity of the experience replay storage. + num_workers (int): number of processes / workers to run in parallel + """ + + # check given approximators + if isinstance(approximators, (tuple, list)): + + # get the policy and Q-value approximator + policy, q_values = None, [] + for approximator in approximators: + if isinstance(approximator, (Policy, QValue)): + policy = approximator + elif isinstance(approximator, QValue): + q_values.append(approximator) + + # check that the policy and Q-value approximator are different than None + if policy is None: + raise ValueError("No policy approximator was given to the algorithm.") + if not q_values: + raise ValueError("No Q-value approximator was given to the algorithm.") + + else: + raise TypeError("Expecting a list/tuple of a policy and a Q-value functions.") + + # check that there is at least 2 Q-value function approximators (the user can have more) + if len(q_values) < 2: + raise ValueError("Expecting at least 2 Q-value function approximators for the TD3 algorithm.") + + # check that the actions are continuous + actions = policy.actions + if not actions.is_continuous(): + raise ValueError("The TD3 assumes that the actions are continuous, however got an action which is not.") + + # evaluate target Q-value fct by copying Q-value function approximator + q_targets = [copy.deepcopy(q_value) for q_value in q_values] + policy_target = copy.deepcopy(policy) + + # create action exploration strategy + exploration = ActionExploration(policy=policy, action=policy.actions) + + # create experience replay + storage = ExperienceReplay(observation_shapes=policy.states, action_shapes=policy.actions, capacity=capacity) + sampler = BatchRandomSampler(storage) + + # create target return estimator + estimator = TDQValueReturn(q_value=q_values, policy=policy_target, target_qvalue=q_targets, gamma=gamma) + + # create Q-value loss and policy loss + q_loss = MSBELoss(td_return=estimator) + policy_loss = QLoss(q_value=q_values[0], policy=policy) # only the first q-value is used to train the policy + losses = [q_loss, policy_loss] + + # create optimizer + optimizer = Adam(learning_rate=lr) + + # create q value and policy updaters + params_updater = PolyakAveraging(rho=polyak) + params_updaters = [(params_updater, q_target) for q_target in q_targets] + [(params_updater, policy_target)] + + # define the 3 main steps in RL: explore, evaluate, and update + explorer = Explorer(task, exploration, storage, num_workers=num_workers) + evaluator = Evaluator(estimator) + updater = Updater(approximators, sampler, losses, optimizer, params_updaters) + + # initialize RL algorithm + super(TD3, self).__init__(explorer, evaluator, updater) + + +# alias +TDDDPG = TD3 diff --git a/pyrobolearn/algos/updater.py b/pyrobolearn/algos/updater.py index d8abbf7..1e3d651 100644 --- a/pyrobolearn/algos/updater.py +++ b/pyrobolearn/algos/updater.py @@ -41,7 +41,7 @@ class Updater(object): This class focuses on the third step of RL algorithms. """ - def __init__(self, approximators, sampler, losses, optimizers, updaters=None): + def __init__(self, approximators, sampler, losses, optimizers, updaters=None, subevaluators=None, delays=None): """ Initialize the update phase. @@ -52,6 +52,9 @@ class Updater(object): optimizers (Optimizer, or list/dict of optimizers): optimizer to use. If dict: key=approximator, value=optimizer. updaters (None, dictionary, list of tuple): list of parameter updaters to run at the end. + subevaluators (list of Estimator/Return): list of sub-evaluators that are evaluated on batches. + delays (None, dictionary): dictionary containing as the key the number of time steps to wait before + updating the specified values (can be the updaters or losses). """ self.approximators = approximators self.sampler = sampler