wassname/pyrobolearn
1
0
Fork 0
mirror of https://github.com/wassname/pyrobolearn.git synced 2026-09-10 12:21:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

update algos: phases, ddpg, td3, sac (need still to be cleaned/tested)

Browse Source
This commit is contained in:
Brian Delhaisse
2019-04-15 01:56:49 +02:00
parent e8b9c23ebf
commit 7b7a151c99
7 changed files with 629 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyrobolearn/algos/ddpg.py
+6 -2
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env python
"""Provide the Deep Deterministic Policy Gradient (DDPG) and the Twin Delayed DDPG algorithm.
"""Provide the Deep Deterministic Policy Gradient (DDPG).
For the Twin Delayed DDPG algorithm, see `pyrobolearn/algos/td3.py`
"""
import copy
@@ -11,6 +13,7 @@ from pyrobolearn.values import QValue
from pyrobolearn.exploration import ActionExploration, GaussianActionExploration
from pyrobolearn.storages import ExperienceReplay
from pyrobolearn.samplers import BatchRandomSampler
from pyrobolearn.estimators import TDQValueReturn
from pyrobolearn.losses import MSBELoss, QLoss
from pyrobolearn.optimizers import Adam
@@ -261,7 +264,7 @@ class DDPG(GradientRLAlgo):
if not actions.is_continuous():
raise ValueError("The DDPG assumes that the actions are continuous, however got an action which is not.")
# evaluate target Q-value fct by copying Q-value function approximator
# Set target parameters equal to main parameters
q_target = copy.deepcopy(q_value)
policy_target = copy.deepcopy(policy)
@@ -270,6 +273,7 @@ class DDPG(GradientRLAlgo):
# create experience replay
storage = ExperienceReplay(observation_shapes=policy.states, action_shapes=policy.actions, capacity=capacity)
sampler = BatchRandomSampler(storage)
# create target return estimator
estimator = TDQValueReturn(q_value=q_value, policy=policy_target, target_qvalue=q_target, gamma=gamma)
pyrobolearn/algos/evaluator.py
+6 -5
View File
@@ -34,7 +34,7 @@ class Evaluator(object):
Initialize the Evaluation phase.
Args:
estimator (Estimator): estimator used to evaluate the actions performed by the policy.
estimator (Estimator, None): estimator used to evaluate the actions performed by the policy.
"""
self.estimator = estimator
@@ -50,8 +50,8 @@ class Evaluator(object):
@estimator.setter
def estimator(self, estimator):
"""Set the estimator."""
if not isinstance(estimator, Estimator):
raise TypeError("Expecting estimator to be an instance of `Estimator`, instead got: "
if not None and not isinstance(estimator, Estimator):
raise TypeError("Expecting estimator to be an instance of `Estimator` or None, instead got: "
"{}".format(type(estimator)))
self._estimator = estimator
@@ -64,11 +64,12 @@ class Evaluator(object):
# Methods #
###########
def evaluate(self): # , storage):
def evaluate(self):
"""
Evaluate the actions.
"""
self.estimator.evaluate(self.storage)
if self.estimator is not None:
self.estimator.evaluate(self.storage)
#############
# Operators #
pyrobolearn/algos/explorer.py
+3 -3
View File
@@ -11,7 +11,7 @@ from pyrobolearn.tasks import RLTask
from pyrobolearn.envs import Env
from pyrobolearn.policies import Policy
from pyrobolearn.exploration import Exploration
from pyrobolearn.storages import RolloutStorage
from pyrobolearn.storages import DictStorage # RolloutStorage
from pyrobolearn import logger
@@ -113,8 +113,8 @@ class Explorer(object):
@storage.setter
def storage(self, storage):
"""Set the storage unit."""
if not isinstance(storage, RolloutStorage):
raise TypeError("Expecting the storage to be an instance of `RolloutStorage`, instead got: "
if not isinstance(storage, DictStorage):
raise TypeError("Expecting the storage to be an instance of `DictStorage`, instead got: "
"{}".format(type(storage)))
self._storage = storage
pyrobolearn/algos/ppo.py
+34 -2
View File
@@ -15,7 +15,7 @@ from pyrobolearn.actorcritics import ActorCritic
from pyrobolearn.exploration import ActionExploration
from pyrobolearn.storages import RolloutStorage
from pyrobolearn.samplers import StorageSampler
from pyrobolearn.samplers import BatchRandomSampler
from pyrobolearn.estimators import GAE
from pyrobolearn.losses import CLIPLoss, ValueLoss, EntropyLoss
from pyrobolearn.optimizers import Adam
@@ -44,6 +44,28 @@ class PPO(GradientRLAlgo):
outside the PPO class, and providing them as input to the constructor and thus privileging composition over
inheritance.
Most of the rest of the documentation has been copied-pasted from [3], and is reproduced here for completeness.
If you use this algorithm, please acknowledge / cite [1, 2, 3].
Background
----------
"PPO is motivated by the same question as TRPO: how can we take the biggest possible improvement step on a policy
using the data we currently have, without stepping so far that we accidentally cause performance collapse? Where
TRPO tries to solve this problem with a complex second-order method, PPO is a family of first-order methods that
use a few other tricks to keep new policies close to old. PPO methods are significantly simpler to implement, and
empirically seem to perform at least as well as TRPO.
There are two primary variants of PPO: PPO-Penalty and PPO-Clip.
* PPO-Penalty approximately solves a KL-constrained update like TRPO, but penalizes the KL-divergence in the
objective function instead of making it a hard constraint, and automatically adjusts the penalty coefficient over
the course of training so that it's scaled appropriately.
* PPO-Clip doesn't have a KL-divergence term in the objective and doesn't have a constraint at all. Instead
relies on specialized clipping in the objective function to remove incentives for the new policy to get far
from the old policy." [3]
Mathematics
-----------
@@ -62,6 +84,16 @@ class PPO(GradientRLAlgo):
This algorithm uses an actor-critic model, and a GAE estimator.
Exploration vs Exploitation
---------------------------
"PPO trains a stochastic policy in an on-policy way. This means that it explores by sampling actions according to
the latest version of its stochastic policy. The amount of randomness in action selection depends on both initial
conditions and the training procedure. Over the course of training, the policy typically becomes progressively
less random, as the update rule encourages it to exploit rewards that it has already found. This may cause the
policy to get trapped in local optima." [3]
Properties
----------
@@ -156,7 +188,7 @@ class PPO(GradientRLAlgo):
logger.debug('create return estimator (GAE)')
estimator = GAE(storage, gamma=gamma, tau=tau)
logger.debug('create storage sampler')
sampler = StorageSampler(storage)
sampler = BatchRandomSampler(storage)
# create loss
logger.debug('create loss')
pyrobolearn/algos/sac.py
Executable
+333
View File
@@ -0,0 +1,333 @@
#!/usr/bin/env python
"""Provide the Soft-Actor Critic algorithm.
Define the SAC reinforcement learning algorithm. This is a model-free, off-policy, actor-critic method.
"""
import copy
import collections
from pyrobolearn.algos.rl_algo import GradientRLAlgo, Explorer, Evaluator, Updater
from pyrobolearn.policies import Policy
from pyrobolearn.values import Value, QValue
from pyrobolearn.actorcritics import ActorCritic
from pyrobolearn.exploration import ActionExploration, GaussianActionExploration
from pyrobolearn.storages import ExperienceReplay
from pyrobolearn.samplers import BatchRandomSampler
from pyrobolearn.estimators import TDQValueReturn
from pyrobolearn.losses import MSBELoss, QLoss
from pyrobolearn.optimizers import Adam
from pyrobolearn.parameters.updater import PolyakAveraging
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse", "OpenAI Spinning Up (Josh Achiam)"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class SAC(GradientRLAlgo):
r"""Soft Actor-Critic
Type: policy-gradient, off-policy, continuous
The following documentation is copied-pasted from [2], and is reproduced here for completeness. If you use
this algorithm, please acknowledge / cite [1, 2].
Background
----------
"Soft Actor Critic (SAC) is an algorithm which optimizes a stochastic policy in an off-policy way, forming a
bridge between stochastic policy optimization and DDPG-style approaches. It isn't a direct successor to TD3
(having been published roughly concurrently), but it incorporates the clipped double-Q trick, and due to the
inherent stochasticity of the policy in SAC, it also winds up benefiting from something like target policy
smoothing.
A central feature of SAC is entropy regularization. The policy is trained to maximize a trade-off between
expected return and entropy, a measure of randomness in the policy. This has a close connection to the
exploration-exploitation trade-off: increasing entropy results in more exploration, which can accelerate
learning later on. It can also prevent the policy from prematurely converging to a bad local optimum" [2]
Key Equations
-------------
"To explain Soft Actor Critic, we first have to introduce the entropy-regularized reinforcement learning setting.
In entropy-regularized RL, there are slightly-different equations for value functions.
* Entropy-Regularized Reinforcement Learning
Entropy is a quantity which, roughly speaking, says how random a random variable is. If a coin is weighted so that
it almost always comes up heads, it has low entropy; if it's evenly weighted and has a half chance of either
outcome, it has high entropy.
Let :math:`x` be a random variable with probability mass or density function :math:`P`. The entropy :math:`H` of
:math:`x` is computed from its distribution :math:`P` according to
.. math:: H(P) = \underE{x \sim P}{-\log P(x)}.
In entropy-regularized reinforcement learning, the agent gets a bonus reward at each time step proportional to
the entropy of the policy at that timestep. This changes the RL problem to:
.. math::
\pi^* = \arg \max_{\pi} \underE{\tau \sim \pi}{ \sum_{t=0}^{\infty} \gamma^t \bigg( R(s_t, a_t, s_{t+1}) +
\alpha H\left(\pi(\cdot|s_t)\right) \bigg)},
where :math:`\alpha > 0` is the trade-off coefficient. (Note: we're assuming an infinite-horizon discounted
setting here, and we'll do the same for the rest of this page.) We can now define the slightly-different value
functions in this setting. :math:`V^{\pi}` is changed to include the entropy bonuses from every timestep:
.. math::
V^{\pi}(s) = \underE{\tau \sim \pi}{ \left. \sum_{t=0}^{\infty} \gamma^t \bigg( R(s_t, a_t, s_{t+1}) +
\alpha H\left(\pi(\cdot|s_t)\right) \bigg) \right| s_0 = s}
:math:`Q^{\pi}` is changed to include the entropy bonuses from every timestep except the first:
.. math::
Q^{\pi}(s,a) = \underE{\tau \sim \pi}{ \left. \sum_{t=0}^{\infty} \gamma^t R(s_t, a_t, s_{t+1}) + \alpha
\sum_{t=1}^{\infty} \gamma^t H\left(\pi(\cdot|s_t)\right)\right| s_0 = s, a_0 = a}
With these definitions, :math:`V^{\pi}` and :math:`Q^{\pi}` are connected by:
.. math:: V^{\pi}(s) = \underE{a \sim \pi}{Q^{\pi}(s,a)} + \alpha H\left(\pi(\cdot|s)\right)
and the Bellman equation for :math:`Q^{\pi}` is
.. math::
Q^{\pi}(s,a) &= \underE{s' \sim P \\ a' \sim \pi}{R(s,a,s') + \gamma\left(Q^{\pi}(s',a') + \alpha
H\left(\pi(\cdot|s')\right) \right)} \\ &= \underE{s' \sim P}{R(s,a,s') + \gamma V^{\pi}(s')}.
* Soft Actor-Critic
SAC concurrently learns a policy :math:`\pi_{\theta}`, two Q-functions :math:`Q_{\phi_1}`, :math:`Q_{\phi_2}`, and
a value function :math:`V_{\psi}`.
**Learning Q**: the Q-functions are learned by MSBE minimization, using a target value network to form the Bellman
backups. They both use the same target, like in TD3, and have loss functions:
.. math::
L(\phi_i, {\mathcal D}) = \underset{(s,a,r,s',d) \sim {\mathcal D}}{{\mathrm E}}\left[ \Bigg( Q_{\phi_i}(s,a)
- \left(r + \gamma (1 - d) V_{\psi_{\text{targ}}}(s') \right) \Bigg)^2 \right].
The target value network, like the target networks in DDPG and TD3, is obtained by polyak averaging the value
network parameters over the course of training.
**Learning V**: the value function is learned by exploiting (a sample-based approximation of) the connection
between :math:`Q^{\pi}` and :math:`V^{\pi}`. Before we go into the learning rule, let's first rewrite the
connection equation by using the definition of entropy to obtain:
.. math::
V^{\pi}(s) &= \underE{a \sim \pi}{Q^{\pi}(s,a)} + \alpha H\left(\pi(\cdot|s)\right) \\
&= \underE{a \sim \pi}{Q^{\pi}(s,a) - \alpha \log \pi(a|s)}.
The RHS is an expectation over actions, so we can approximate it by sampling from the policy:
.. math:: V^{\pi}(s) \approx Q^{\pi}(s,\tilde{a}) - \alpha \log \pi(\tilde{a}|s), \quad \tilde{a} \sim \pi(\cdot|s).
SAC sets up a mean-squared-error loss for :math:`V_{\psi}` based on this approximation. But what Q-value do we use?
SAC uses clipped double-Q like TD3 for learning the value function, and takes the minimum Q-value between the two
approximators. So the SAC loss for value function parameters is:
.. math:: L(\psi, {\mathcal D}) = \underE{s \sim \mathcal{D} \\ \tilde{a} \sim \pi_{\theta}}{\Bigg(V_{\psi}(s) -
\left(\min_{i=1,2} Q_{\phi_i}(s,\tilde{a}) - \alpha \log \pi_{\theta}(\tilde{a}|s) \right)\Bigg)^2}.
Importantly, we do not use actions from the replay buffer here: these actions are sampled fresh from the current
version of the policy.
**Learning the Policy**: the policy should, in each state, act to maximize the expected future return plus
expected future entropy. That is, it should maximize :math:`V^{\pi}(s)`, which we expand out (as before) into
.. math:: \underE{a \sim \pi}{Q^{\pi}(s,a) - \alpha \log \pi(a|s)}.
The way we optimize the policy makes use of the reparameterization trick, in which a sample from
:math:`\pi_{\theta}(\cdot|s)` is drawn by computing a deterministic function of state, policy parameters, and
independent noise. To illustrate: following the authors of the SAC paper, we use a squashed Gaussian policy,
which means that samples are obtained according to
.. math::
\tilde{a}_{\theta}(s, \xi) = \tanh\left( \mu_{\theta}(s) + \sigma_{\theta}(s) \odot \xi \right), \quad
\xi \sim \mathcal{N}(0, I).
**Notice**:
This policy has two key differences from the policies we use in the other policy optimization algorithms:
1. The squashing function. The :math:`\tanh` in the SAC policy ensures that actions are bounded to a finite range.
This is absent in the VPG, TRPO, and PPO policies. It also changes the distribution: before the :math:`\tanh` the
SAC policy is a factored Gaussian like the other algorithms' policies, but after the :math:`\tanh` it is not. (You
can still evaluate the log-probabilities of actions in closed form, though: see the paper appendix for details.)
2. The way standard deviations are parameterized. In VPG, TRPO, and PPO, we represent the log std devs with
state-independent parameter vectors. In SAC, we represent the log std devs as outputs from the neural network,
meaning that they depend on state in a complex way. SAC with state-independent log std devs, in our experience,
did not work. (Can you think of why? Or better yet: run an experiment to verify?)
**End of notice**
The reparameterization trick allows us to rewrite the expectation over actions (which contains a pain point:
the distribution depends on the policy parameters) into an expectation over noise (which removes the pain point:
the distribution now has no dependence on parameters):
.. math::
\underE{a \sim \pi_{\theta}}{Q^{\pi_{\theta}}(s,a) - \alpha \log \pi_{\theta}(a|s)}
= \underE{\xi \sim \mathcal{N}}{Q^{\pi_{\theta}}(s,\tilde{a}_{\theta}(s,\xi))
- \alpha \log \pi_{\theta}(\tilde{a}_{\theta}(s,\xi)|s)}
To get the policy loss, the final step is that we need to substitute :math:`Q^{\pi_{\theta}}` with one of our
function approximators. The same as in TD3, we use Q_{\phi_1}. The policy is thus optimized according to
.. math::
\max_{\theta} \underE{s \sim \mathcal{D} \\ \xi \sim \mathcal{N}}{Q_{\phi_1}(s,\tilde{a}_{\theta}(s,\xi))
- \alpha \log \pi_{\theta}(\tilde{a}_{\theta}(s,\xi)|s)},
which is almost the same as the DDPG and TD3 policy optimization, except for the stochasticity and entropy term."
[2]
Exploration vs. Exploitation
----------------------------
"SAC trains a stochastic policy with entropy regularization, and explores in an on-policy way. The entropy
regularization coefficient \alpha explicitly controls the explore-exploit tradeoff, with higher :math:`\alpha`
corresponding to more exploration, and lower \alpha corresponding to more exploitation. The right coefficient
(the one which leads to the stablest / highest-reward learning) may vary from environment to environment, and
could require careful tuning.
At test time, to see how well the policy exploits what it has learned, we remove stochasticity and use the mean
action instead of a sample from the distribution. This tends to improve performance over the original stochastic
policy." [2]
Pseudo-algo
-----------
Pseudo-algorithm (taken from [2] and reproduce here for completeness)::
1. Input: initial policy parameters :math:`\theta_0`, Q-function parameters :math:`\phi_1, \phi_2`,
V-function parameters :math:`\psi`, empty replay buffer :math:`D`
2. Set target parameters equal to main parameters :math:`\psi_{\text{target}} \leftarrow \psi`
3. repeat:
4. Observe state :math:`s` and select action :math:`a \sim \pi_{\theta}(\cdot|s)`
5. Execute :math:`a` in the environment
6. Observe next state :math:`s'`, reward :math:`r`, and done signal :math:`d` to indicate whether :math:`s'`
is terminal
7. Store :math:`(s, a, r, s', d)` in replay buffer :math:`D`
8. If :math:`s'` is terminal, reset environment state.
9. if it's time to update then:
10. for j in range(num_updates) do:
11. Randomly sample a batch of transitions, :math:`B = {(s, a, r, s', d)}` from :math:`D`
12. Compute targets for Q and V functions:
:math:`y_q(r,s',d) = r + \gamma (1-d) V_{\psi_{\text{targ}}}(s')`
:math:`y_v(s) = \min_{i=1,2} Q_{\phi_i} (s, \tilde{a}) - \alpha \log \pi_{\theta}(\tilde{a}|s)`,
with :math:`\tilde{a} \sim \pi_{\theta}(\cdot|s)`
13. Update Q-functions by one step of gradient descent using:
:math:`\nabla_{\phi_i} \frac{1}{|B|}\sum_{(s,a,r,s',d) \in B} \left( Q_{\phi,i}(s,a) -
y_q(r,s',d) \right)^2`, for :math:`i=1,2`
14. Update V-function by one step of gradient descent using:
:math:`\nabla_{\psi} \frac{1}{|B|}\sum_{s \in B} \left( V_{\psi}(s) - y_v(s) \right)^2`
15. Update policy by one step of gradient ascent using:
:math:`\nabla_{\theta} \frac{1}{|B|}\sum_{s \in B} \Big( Q_{\phi,1}(s, \tilde{a}_{\theta}(s))
- \alpha \log \pi_{\theta} \left(\left. \tilde{a}_{\theta}(s) | s\right) \Big)`,
where :math:`\tilde{a}_{\theta}(s)` is a sample from :math:`\pi_{\theta}(\cdot|s)` which is
differentiable wrt :math:`\theta` via the reparametrization trick.
16. Update target value network with:
:math:`\psi_{\text{targ}} &\leftarrow \rho \psi_{\text{targ}} + (1-\rho) \psi`
17. end for
18. end if
19. until convergence
References:
[1] "Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor",
Haarnoja et al, 2018
[2] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/sac.html
[3] RLKit: https://github.com/vitchyr/rlkit/blob/master/rlkit/torch/sac/sac.py
"""
def __init__(self, task, approximators, gamma=0.99, lr=5e-4, polyak=0.995, capacity=10000, num_workers=1):
"""
Initialize the SAC off-policy RL algorithm.
Args:
task (RLTask, Env): RL task/env to run
approximators ([Policy, Value, QValue]): approximators to optimize.
gamma (float): discount factor (which is a bias-variance tradeoff). This parameter describes how much
importance has the future rewards we get.
lr (float): learning rate
polyak (float): coefficient in the polyak averaging when updating the target approximators.
capacity (int): capacity of the experience replay storage.
num_workers (int): number of processes / workers to run in parallel
"""
# check approximators
if not isinstance(approximators, collections.Iterable):
raise TypeError("Expecting the approximators to be a list containing a Policy, a Value, and at least 2 "
"QValues")
policy, value, q_values = None, None, []
for approximator in approximators:
if isinstance(approximator, Policy):
policy = approximator
elif isinstance(approximator, Value):
value = approximator
elif isinstance(approximator, ActorCritic):
policy = approximator.actor
value = approximator.critic
elif isinstance(approximator, QValue):
q_values.append(approximator)
if policy is None:
raise TypeError("No policy was given to the algorithm.")
if value is None:
raise TypeError("No value function approximator was given to the algorithm.")
if len(q_values) == 0:
raise TypeError("No Q-value function approximators were given to the algorithm.")
# set target parameters equal to main parameters for the value function
value_target = copy.deepcopy(value)
# create experience replay
storage = ExperienceReplay(observation_shapes=policy.states, action_shapes=policy.actions, capacity=capacity)
sampler = BatchRandomSampler(storage)
# create action exploration
exploration = ActionExploration(policy)
# create targets
# TODO
# create losses
q_loss = MSBELoss(td_return=estimator)
policy_loss = QLoss(q_value=q_values[0], policy=policy) # only the first q-value is used to train the policy
losses = [q_loss, policy_loss]
# create optimizer
optimizer = Adam(learning_rate=lr)
# create parameter updater for target value function
params_updater = PolyakAveraging(rho=polyak)
params_updater = (params_updater, value_target)
# define the 3 main steps in RL: explore, evaluate, and update
explorer = Explorer(task, exploration, storage, num_workers=num_workers)
evaluator = Evaluator(None)
updater = Updater(approximators, sampler, losses, optimizer, updaters=params_updater)
# initialize RL algorithm
super(SAC, self).__init__(explorer, evaluator, updater)
pyrobolearn/algos/td3.py
Executable
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python
"""Provide the Twin Delayed Deep Deterministic Policy Gradient (TD3) algorithm.
The REINFORCE/VPG is a model-free, off-policy, gradient policy-based algorithm that works only for continuous actions.
"""
import copy
from pyrobolearn.algos.rl_algo import GradientRLAlgo, Explorer, Evaluator, Updater
from pyrobolearn.policies import Policy
from pyrobolearn.values import QValue
from pyrobolearn.exploration import ActionExploration, GaussianActionExploration
from pyrobolearn.storages import ExperienceReplay
from pyrobolearn.samplers import BatchRandomSampler
from pyrobolearn.estimators import TDQValueReturn
from pyrobolearn.losses import MSBELoss, QLoss
from pyrobolearn.optimizers import Adam
from pyrobolearn.parameters.updater import PolyakAveraging
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse", "OpenAI Spinning Up (Josh Achiam)"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class TD3(GradientRLAlgo):
r"""Twin Delayed Deep Deterministic Policy Gradient
Type:: model-free, off-policy, gradient-based actor-critic algorithm for continuous only action spaces.
The documentation has been copied-pasted from [2], and is reproduced here for completeness. If you use this
algorithm, please acknowledge / cite [1, 2].
Background
----------
"While DDPG can achieve great performance sometimes, it is frequently brittle with respect to hyperparameters and
other kinds of tuning. A common failure mode for DDPG is that the learned Q-function begins to dramatically
overestimate Q-values, which then leads to the policy breaking, because it exploits the errors in the Q-function.
Twin Delayed DDPG (TD3) is an algorithm which addresses this issue by introducing three critical tricks:
1. Trick One: Clipped Double-Q Learning. TD3 learns two Q-functions instead of one (hence 'twin'), and uses the
smaller of the two Q-values to form the targets in the Bellman error loss functions.
2. Trick Two: 'Delayed' Policy Updates. TD3 updates the policy (and target networks) less frequently than the
Q-function. The paper recommends one policy update for every two Q-function updates.
3. Trick Three: Target Policy Smoothing. TD3 adds noise to the target action, to make it harder for the policy
to exploit Q-function errors by smoothing out Q along changes in action.
Together, these three tricks result in substantially improved performance over baseline DDPG." [2]
Key Equations
-------------
"TD3 concurrently learns two Q-functions, :math:`Q_{\phi_1}` and :math:`Q_{\phi_2}`, by mean square Bellman error
minimization, in almost the same way that DDPG learns its single Q-function. To show exactly how TD3 does this
and how it differs from normal DDPG, we'll work from the innermost part of the loss function outwards.
First: target policy smoothing. Actions used to form the Q-learning target are based on the target policy,
:math:`\mu_{\theta_{\text{targ}}}`, but with clipped noise added on each dimension of the action.
After adding the clipped noise, the target action is then clipped to lie in the valid action range (all valid
actions, :math:`a`, satisfy :math:`a_{Low} \leq a \leq a_{High}`). The target actions are thus:
.. math:: a'(s') = \text{clip}\left(\mu_{\theta_{\text{targ}}}(s') + \text{clip}(\epsilon,-c,c), a_{Low},
a_{High}\right), \qquad \epsilon \sim \mathcal{N}(0, \sigma)
Target policy smoothing essentially serves as a regularizer for the algorithm. It addresses a particular failure
mode that can happen in DDPG: if the Q-function approximator develops an incorrect sharp peak for some actions,
the policy will quickly exploit that peak and then have brittle or incorrect behavior. This can be averted by
smoothing out the Q-function over similar actions, which target policy smoothing is designed to do.
Next: clipped double-Q learning. Both Q-functions use a single target, calculated using whichever of the two
Q-functions gives a smaller target value:
.. math:: y(r,s',d) = r + \gamma (1 - d) \min_{i=1,2} Q_{\phi_{i, \text{targ}}}(s', a'(s')),
and then both are learned by regressing to this target:
.. math::
L(\phi_1,{\mathcal D}) = \underE{(s,a,r,s',d) \sim {\mathcal D}}{\left(Q_{\phi_1}(s,a) - y(r,s',d) \right)^2},
L(\phi_2,{\mathcal D}) = \underE{(s,a,r,s',d) \sim {\mathcal D}}{\left(Q_{\phi_2}(s,a) - y(r,s',d) \right)^2 }.
Using the smaller Q-value for the target, and regressing towards that, helps fend off overestimation in the
Q-function.
Lastly: the policy is learned just by maximizing Q_{\phi_1}:
.. math:: \max_{\theta} \underset{s \sim {\mathcal D}}{{\mathrm E}}\left[ Q_{\phi_1}(s, \mu_{\theta}(s)) \right],
which is pretty much unchanged from DDPG. However, in TD3, the policy is updated less frequently than the
Q-functions are. This helps damp the volatility that normally arises in DDPG because of how a policy update changes
the target." [2]
Exploration vs. Exploitation
----------------------------
"TD3 trains a deterministic policy in an off-policy way. Because the policy is deterministic, if the agent were to
explore on-policy, in the beginning it would probably not try a wide enough variety of actions to find useful
learning signals. To make TD3 policies explore better, we add noise to their actions at training time, typically
uncorrelated mean-zero Gaussian noise. To facilitate getting higher-quality training data, you may reduce the
scale of the noise over the course of training. (We do not do this in our implementation, and keep noise scale
fixed throughout.)
At test time, to see how well the policy exploits what it has learned, we do not add noise to the actions." [2]
Pseudo-algo:
-----------
Pseudo-algorithm (taken from [2] and reproduce here for completeness)::
1. Input: initial policy parameters :math:`\theta`, Q-function parameters :math:`\phi_1, \phi_2`, empty replay
buffer :math:`D`
2. Set target parameters equal to main parameters :math:`\theta_{target} \leftarrow \theta`,
:math:`\phi_{target, i} \leftarrow \phi_i` for i=1,2
3. repeat:
4. Observe state :math:`s` and select action :math:`a = clip(\mu_{\theta}(s) + \epsilon, a_{Low}, a_{High})`,
where :math:`\epsilon \sim \mathcal{N}`
5. Execute :math:`a` in the environment
6. Observe next state :math:`s'`, reward :math:`r`, and done signal :math:`d` to indicate whether :math:`s'`
is terminal.
7. Store :math:`(s, a, r, s', d)` in replay buffer :math:`D`
8. If :math:`s'` is terminal, reset environment state.
9. if it's time to update then:
10. for j in range(num_updates) do:
11. Randomly sample a batch of transitions, :math:`B = {(s, a, r, s', d)}` from :math:`D`
12. Compute target actions
:math:`a'(s') = clip(\mu_{\theta_target}(s') + clip(\epsilon, -c, c), a_{Low}, a_{High})`,
where :math:`\epsilon \sim \mathcal{N}`
13. Compute targets
:math:`y(r,s',d) = r + \gamma (1 - d) \min_{i=1,2} Q_{\phi_{target, i}}(s', a'(s))`
14. Update Q-functions by one step of gradient descent using
:math:`\grad_{\phi_i} \frac{1}{|B|} \sum_{(s,a,r,s',d) \in B} (Q_{\phi_i}(s,a) - y(r,s',d))^2`,
for i=1,2
15. if (j % policy_delay) == 0 then:
16. Update policy by one step of gradient ascent using
:math:`\grad_{\theta} \frac{1}{|B|} \sum_{s \in B} Q_{\phi_1}(s, \mu_{\theta}(s))`
17. Update target networks with
:math:`\phi_{target,i} \leftarrow \rho \phi_{target,i} + (1 - \rho)\phi_i` for i=1,2
:math:`\theta_{target} \leftarrow \rho \theta_{target} + (1 - \rho) \theta`
18. end if
19. end for
20. end if
21. until convergence
References::
[1] "Addressing Function Approximation Error in Actor-Critic Methods", Fujimoto et al., 2018
[2] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/td3.html
"""
def __init__(self, task, approximators, gamma=0.99, lr=0.001, polyak=0.995, capacity=10000, num_workers=1):
"""
Initialize the TD3 off-policy RL algorithm.
Args:
task (RLTask, Env): RL task/env to run
approximators (Policy, [Policy, Value], ActorCritic): approximators to optimize
gamma (float): discount factor (which is a bias-variance tradeoff). This parameter describes how much
importance has the future rewards we get.
lr (float): learning rate.
polyak (float): coefficient in the polyak averaging when updating the target approximators.
capacity (int): capacity of the experience replay storage.
num_workers (int): number of processes / workers to run in parallel
"""
# check given approximators
if isinstance(approximators, (tuple, list)):
# get the policy and Q-value approximator
policy, q_values = None, []
for approximator in approximators:
if isinstance(approximator, (Policy, QValue)):
policy = approximator
elif isinstance(approximator, QValue):
q_values.append(approximator)
# check that the policy and Q-value approximator are different than None
if policy is None:
raise ValueError("No policy approximator was given to the algorithm.")
if not q_values:
raise ValueError("No Q-value approximator was given to the algorithm.")
else:
raise TypeError("Expecting a list/tuple of a policy and a Q-value functions.")
# check that there is at least 2 Q-value function approximators (the user can have more)
if len(q_values) < 2:
raise ValueError("Expecting at least 2 Q-value function approximators for the TD3 algorithm.")
# check that the actions are continuous
actions = policy.actions
if not actions.is_continuous():
raise ValueError("The TD3 assumes that the actions are continuous, however got an action which is not.")
# evaluate target Q-value fct by copying Q-value function approximator
q_targets = [copy.deepcopy(q_value) for q_value in q_values]
policy_target = copy.deepcopy(policy)
# create action exploration strategy
exploration = ActionExploration(policy=policy, action=policy.actions)
# create experience replay
storage = ExperienceReplay(observation_shapes=policy.states, action_shapes=policy.actions, capacity=capacity)
sampler = BatchRandomSampler(storage)
# create target return estimator
estimator = TDQValueReturn(q_value=q_values, policy=policy_target, target_qvalue=q_targets, gamma=gamma)
# create Q-value loss and policy loss
q_loss = MSBELoss(td_return=estimator)
policy_loss = QLoss(q_value=q_values[0], policy=policy) # only the first q-value is used to train the policy
losses = [q_loss, policy_loss]
# create optimizer
optimizer = Adam(learning_rate=lr)
# create q value and policy updaters
params_updater = PolyakAveraging(rho=polyak)
params_updaters = [(params_updater, q_target) for q_target in q_targets] + [(params_updater, policy_target)]
# define the 3 main steps in RL: explore, evaluate, and update
explorer = Explorer(task, exploration, storage, num_workers=num_workers)
evaluator = Evaluator(estimator)
updater = Updater(approximators, sampler, losses, optimizer, params_updaters)
# initialize RL algorithm
super(TD3, self).__init__(explorer, evaluator, updater)
# alias
TDDDPG = TD3
pyrobolearn/algos/updater.py
+4 -1
View File
@@ -41,7 +41,7 @@ class Updater(object):
This class focuses on the third step of RL algorithms.
"""
def __init__(self, approximators, sampler, losses, optimizers, updaters=None):
def __init__(self, approximators, sampler, losses, optimizers, updaters=None, subevaluators=None, delays=None):
"""
Initialize the update phase.
@@ -52,6 +52,9 @@ class Updater(object):
optimizers (Optimizer, or list/dict of optimizers): optimizer to use. If dict: key=approximator,
value=optimizer.
updaters (None, dictionary, list of tuple): list of parameter updaters to run at the end.
subevaluators (list of Estimator/Return): list of sub-evaluators that are evaluated on batches.
delays (None, dictionary): dictionary containing as the key the number of time steps to wait before
updating the specified values (can be the updaters or losses).
"""
self.approximators = approximators
self.sampler = sampler