diff --git a/pyrobolearn/algos/ddpg.py b/pyrobolearn/algos/ddpg.py new file mode 100755 index 0000000..c3c2aa9 --- /dev/null +++ b/pyrobolearn/algos/ddpg.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python +"""Provide the Deep Deterministic Policy Gradient (DDPG) and the Twin Delayed DDPG algorithm. +""" + +import copy + +from pyrobolearn.algos.rl_algo import GradientRLAlgo, Explorer, Evaluator, Updater + +from pyrobolearn.policies import Policy +from pyrobolearn.values import QValue +from pyrobolearn.exploration import ActionExploration, GaussianActionExploration + +from pyrobolearn.storages import ExperienceReplay +from pyrobolearn.estimators import TDQValueReturn +from pyrobolearn.losses import MSBELoss, QLoss +from pyrobolearn.optimizers import Adam + +from pyrobolearn.parameters.updater import PolyakAveraging + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse", "OpenAI Spinning Up (Josh Achiam)"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class DDPG(GradientRLAlgo): + r"""Deep Deterministic Policy Gradients (DDPG) + + Type:: actor-critic method, off-policy, continuous action space, exploration in action-space (thus step-based). + + The documentation has been copied-pasted from [3], and is reproduced here for completeness. If you use this + algorithm, please acknowledge / cite [1, 3]. + + + Background + ---------- + + "Deep Deterministic Policy Gradient (DDPG) is an algorithm which concurrently learns a Q-function and a policy. + It uses off-policy data and the Bellman equation to learn the Q-function, and uses the Q-function to learn the + policy. + + This approach is closely connected to Q-learning, and is motivated the same way: if you know the optimal + action-value function :math:`Q^*(s,a)`, then in any given state, the optimal action :math:`a^*(s)` can be found by + solving: + + .. math:: a^*(s) = \arg \max_a Q^*(s,a). + + DDPG interleaves learning an approximator to :math:`Q^*(s,a)` with learning an approximator to :math:`a^*(s)`, and + it does so in a way which is specifically adapted for environments with continuous action spaces. But what does it + mean that DDPG is adapted specifically for environments with continuous action spaces? It relates to how we + compute the max over actions in :math:`\max_a Q^*(s,a)`. + + When there are a finite number of discrete actions, the max poses no problem, because we can just compute the + Q-values for each action separately and directly compare them. (This also immediately gives us the action which + maximizes the Q-value.) But when the action space is continuous, we can't exhaustively evaluate the space, and + solving the optimization problem is highly non-trivial. Using a normal optimization algorithm would make + calculating :math:`\max_a Q^*(s,a)` a painfully expensive subroutine. And since it would need to be run every + time the agent wants to take an action in the environment, this is unacceptable. + + Because the action space is continuous, the function :math:`Q^*(s,a)` is presumed to be differentiable with + respect to the action argument. This allows us to set up an efficient, gradient-based learning rule for a policy + :math:`\mu(s)` which exploits that fact. Then, instead of running an expensive optimization subroutine each time + we wish to compute :math:`\max_a Q(s,a)`, we can approximate it with :math:`\max_a Q(s,a) \approx Q(s,\mu(s))`. + See the Key Equations section details." [3] + + + Key Equations + ------------- + + "Here are the math behind the two parts of DDPG: learning a Q function, and learning a policy. + + * The Q-Learning Side of DDPG + + First, let's recap the Bellman equation describing the optimal action-value function, Q^*(s,a). It's given by + + .. math:: Q^*(s,a) = \underset{s' \sim P}{{\mathrm E}}\left[r(s,a) + \gamma \max_{a'} Q^*(s', a')\right] + + where :math:`s' \sim P` is shorthand for saying that the next state, :math:`s'`, is sampled by the environment + from a distribution :math:`P(\cdot| s, a)`. + + This Bellman equation is the starting point for learning an approximator to :math:`Q^*(s,a)`. Suppose the + approximator is a neural network :math:`Q_{\phi}(s,a)`, with parameters :math:`\phi`, and that we have collected + a set :math:`D` of transitions :math:`(s, a, r, s', d)` (where :math:`d` indicates whether state :math:`s'` is + terminal). We can set up a mean-squared Bellman error (MSBE) function, which tells us roughly how closely + :math:`Q_{\phi}` comes to satisfying the Bellman equation: + + .. math:: L(\phi, D) = \underset{(s,a,r,s',d) \sim D}{{\mathrm E}}\left[ \Bigg( Q_{\phi}(s,a) - \left(r + \gamma + (1 - d) \max_{a'} Q_{\phi}(s',a') \right) \Bigg)^2 \right] + + Here, in evaluating :math:`(1-d)`, we've used a Python convention of evaluating True to 1 and False to zero. Thus, + when d==True - which is to say, when :math:`s'` is a terminal state - the Q-function should show that the agent + gets no additional rewards after the current state. + + Q-learning algorithms for function approximators, such as DQN (and all its variants) and DDPG, are largely based + on minimizing this MSBE loss function. There are two main tricks employed by all of them which are worth + describing, and then a specific detail for DDPG. + + 1. Trick One: Replay Buffers. All standard algorithms for training a deep neural network to approximate + :math:`Q^*(s,a)` make use of an experience replay buffer. This is the set :math:`D` of previous experiences. + In order for the algorithm to have stable behavior, the replay buffer should be large enough to contain a wide + range of experiences, but it may not always be good to keep everything. If you only use the very-most recent data, + you will overfit to that and things will break; if you use too much experience, you may slow down your learning. + This may take some tuning to get right. + + **Notice**: + + We've mentioned that DDPG is an off-policy algorithm: this is as good a point as any to highlight why and how. + Observe that the replay buffer should contain old experiences, even though they might have been obtained using an + outdated policy. Why are we able to use these at all? The reason is that the Bellman equation doesn't care which + transition tuples are used, or how the actions were selected, or what happens after a given transition, because + the optimal Q-function should satisfy the Bellman equation for all possible transitions. So any transitions that + we've ever experienced are fair game when trying to fit a Q-function approximator via MSBE minimization. + + **End of notice** + + 2. Trick Two: Target Networks. Q-learning algorithms make use of target networks. The term + + .. math:: r + \gamma (1 - d) \max_{a'} Q_{\phi}(s',a') + + is called the target, because when we minimize the MSBE loss, we are trying to make the Q-function be more like + this target. Problematically, the target depends on the same parameters we are trying to train: :math:`\phi`. + This makes MSBE minimization unstable. The solution is to use a set of parameters which comes close to + :math:`\phi`, but with a time delay - that is to say, a second network, called the target network, which lags the + first. The parameters of the target network are denoted :math:`\phi_{\text{targ}}`. + + In DQN-based algorithms, the target network is just copied over from the main network every some-fixed-number of + steps. In DDPG-style algorithms, the target network is updated once per main network update by polyak averaging: + + .. math:: \phi_{\text{targ}} \leftarrow \rho \phi_{\text{targ}} + (1 - \rho) \phi, + + where :math:`\rho` is a hyperparameter between 0 and 1 (usually close to 1). + + **DDPG Detail: Calculating the Max Over Actions in the Target**. As mentioned earlier: computing the maximum over + actions in the target is a challenge in continuous action spaces. DDPG deals with this by using a target policy + network to compute an action which approximately maximizes :math:`Q_{\phi_{\text{targ}}}`. The target policy + network is found the same way as the target Q-function: by polyak averaging the policy parameters over the course + of training. + + Putting it all together, Q-learning in DDPG is performed by minimizing the following MSBE loss with stochastic + gradient descent: + + .. math:: L(\phi, D) = \underset{(s,a,r,s',d) \sim D}{{\mathrm E}}\left[ \Bigg( Q_{\phi}(s,a) - \left(r + \gamma + (1 - d) Q_{\phi_{\text{targ}}}(s', \mu_{\theta_{\text{targ}}}(s')) \right) \Bigg)^2 \right], + + where :math:`\mu_{\theta_{\text{targ}}}` is the target policy. + + + * The Policy Learning Side of DDPG + + Policy learning in DDPG is fairly simple. We want to learn a deterministic policy :math:`\mu_{\theta}(s)` which + gives the action that maximizes :math:`Q_{\phi}(s,a)`. Because the action space is continuous, and we assume the + Q-function is differentiable with respect to action, we can just perform gradient ascent (with respect to policy + parameters only) to solve + + .. math:: \max_{\theta} \underset{s \sim D}{{\mathrm E}}\left[ Q_{\phi}(s, \mu_{\theta}(s)) \right]. + + Note that the Q-function parameters are treated as constants here." [3] + + + Exploration vs. Exploitation + ---------------------------- + + "DDPG trains a deterministic policy in an off-policy way. Because the policy is deterministic, if the agent were + to explore on-policy, in the beginning it would probably not try a wide enough variety of actions to find useful + learning signals. To make DDPG policies explore better, we add noise to their actions at training time. + The authors of the original DDPG paper recommended time-correlated OU noise, but more recent results suggest that + uncorrelated, mean-zero Gaussian noise works perfectly well. Since the latter is simpler, it is preferred. + To facilitate getting higher-quality training data, you may reduce the scale of the noise over the course of + training. + + At test time, to see how well the policy exploits what it has learned, we do not add noise to the actions." [3] + + + Pseudo-algo + ----------- + + Pseudo-algorithm (taken from [3] and reproduce here for completeness):: + 1. Input: initial policy parameters :math:`\theta_0`, initial Q-value function parameters :math:`\phi_0`, + empty replay buffer :math:`D`. + 2. Set target parameters equal to main parameters :math:`\theta_{\text{targ}} \leftarrow \theta`, + :math:`\phi_{\text{targ}} \leftarrow \phi` + 3. repeat: + 4. Observe state :math:`s` and select action :math:`a = \text{clip}(\mu_{\theta}(s) + \epsilon, a_{Low}, + a_{High})`, where :math:`\epsilon \sim \mathcal{N}` + 5. Execute :math:`a` in the environment + 6. Observe next state :math:`s'`, reward :math:`r`, and done signal :math:`d` to indicate whether + :math:`s'` is terminal. + 7. Store :math:`(s, a, r, s', d)` in replay buffer :math:`D` + 8. If :math:`s'` is terminal, reset environment state. + 9. if it's time to update then: + 10. for j in range(num_updates) do: + 11. Randomly sample a batch of transitions, :math:`B = {(s, a, r, s', d)}` from :math:`D` + 12. Compute targets + :math:`y(r,s',d) = r + \gamma (1-d) Q_{\phi_{\text{targ}}}(s', \mu_{\theta_{\text{targ}}}(s'))` + 13. Update Q-function by one step of gradient descent using + :math:`\nabla_{\phi} \frac{1}{|B|}\sum_{(s,a,r,s',d) \in B} ( Q_{\phi}(s,a) - y(r,s',d) )^2` + 14. Update policy by one step of gradient ascent using + :math:`\nabla_{\theta} \frac{1}{|B|}\sum_{s \in B} Q_{\phi}(s, \mu_{\theta}(s))` + 15. Update target networks with + :math:`\phi_{\text{targ}} \leftarrow \rho \phi_{\text{targ}} + (1-\rho) \phi` + :math:`\theta_{\text{targ}} \leftarrow \rho \theta_{\text{targ}} + (1-\rho) \theta` + 16. end for + 17. end if + 18. until convergence + + + .. seealso:: For the "Twin Delayed Deep Deterministic Policy Gradients", see `pyrobolearn/algos/td3.py`. + + + References: + [1] "Deterministic Policy Gradient Algorithm", Silver et al., 2014 + [2] "Continuous Control with Deep Reinforcement Learning", Lillicrap et al., 2015 + [3] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/ddpg.html + [4] PyTorch implementation by Kostrikov: https://github.com/ikostrikov/pytorch-ddpg-naf + [5] "Policy Gradient Algorithms": + https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html + """ + + def __init__(self, task, approximators, gamma=0.99, lr=0.001, polyak=0.995, capacity=10000, num_workers=1): + """ + Initialize the DDPG off-policy RL algorithm. + + Args: + task (RLTask, Env): RL task/env to run + approximators ([Policy, QValue]): policy and Q-value function approximator to optimize. + gamma (float): discount factor (which is a bias-variance tradeoff). This parameter describes how much + importance has the future rewards we get. + lr (float): learning rate + polyak (float): coefficient in the polyak averaging when updating the target approximators. + capacity (int): capacity of the experience replay storage. + num_workers (int): number of processes / workers to run in parallel + """ + + # check given approximators + if isinstance(approximators, (tuple, list)) and len(approximators) != 2: + + # get the policy and Q-value approximator + policy, q_value = None, None + for approximator in approximators: + if isinstance(approximator, (Policy, QValue)): + policy = approximator + elif isinstance(approximator, QValue): + q_value = approximator + + # check that the policy and Q-value approximator are different than None + if policy is None: + raise ValueError("No policy approximator was given to the algorithm.") + if q_value is None: + raise ValueError("No Q-value approximator was given to the algorithm.") + + else: + raise TypeError("Expecting a list/tuple of a policy and a Q-value function.") + + # check that the actions are continuous + actions = policy.actions + if not actions.is_continuous(): + raise ValueError("The DDPG assumes that the actions are continuous, however got an action which is not.") + + # evaluate target Q-value fct by copying Q-value function approximator + q_target = copy.deepcopy(q_value) + policy_target = copy.deepcopy(policy) + + # create action exploration strategy + exploration = ActionExploration(policy=policy, action=policy.actions) + + # create experience replay + storage = ExperienceReplay(observation_shapes=policy.states, action_shapes=policy.actions, capacity=capacity) + + # create target return estimator + estimator = TDQValueReturn(q_value=q_value, policy=policy_target, target_qvalue=q_target, gamma=gamma) + + # create Q-value loss and policy loss + q_loss = MSBELoss(td_return=estimator) + policy_loss = QLoss(q_value=q_value, policy=policy) + losses = [q_loss, policy_loss] + + # create optimizer + optimizer = Adam(learning_rate=lr) + + # create q value and policy updaters + q_value_updater = PolyakAveraging(rho=polyak) + policy_updater = PolyakAveraging(rho=polyak) + approximator_updaters = {q_value_updater: q_target, policy_updater: policy_target} + + # define the 3 main steps in RL: explore, evaluate, and update + explorer = Explorer(task, exploration, storage, num_workers=num_workers) + evaluator = Evaluator(estimator) + updater = Updater(approximators, sampler, losses, optimizer, approximator_updaters) + + # initialize RL algorithm + super(DDPG, self).__init__(explorer, evaluator, updater) diff --git a/pyrobolearn/algos/dqn.py b/pyrobolearn/algos/dqn.py new file mode 100755 index 0000000..42f5ebd --- /dev/null +++ b/pyrobolearn/algos/dqn.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +"""Provide the DQN algorithm. + +The DQN is a value-based, off-policy, model-free RL algorithm with exploration in the action space. +""" + +import copy +import torch + +# from pyrobolearn.envs import Env +from pyrobolearn.policies import PolicyFromQValue +# from pyrobolearn.tasks import RLTask +from pyrobolearn.algos.rl_algo import GradientRLAlgo, Explorer, Evaluator, Updater + +from pyrobolearn.values import ParametrizedQValueOutput +from pyrobolearn.exploration import EpsilonGreedyActionExploration + +from pyrobolearn.storages import ExperienceReplay +from pyrobolearn.estimators import TDQLearningReturn +from pyrobolearn.losses import MSBELoss, HuberLoss +from pyrobolearn.optimizers import Adam + +from pyrobolearn.parameters.updater import CopyParameter + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse", "PyTorch (Adam Paszke)"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class DQN(GradientRLAlgo): + r"""Deep Q-Network + + Type: model-free, value-based, off-policy RL algorithm for discrete action spaces with exploration in the + action space. + + + Background + ---------- + + Deep Q-Networks (DQNs) work by approximating the Q-value function using a deep neural network [1,2]. The optimal + policy is then inferred from it by returning the discrete action that maximizes the Q-value function, i.e. + :math:`\pi^*(s_t) = argmax_{a_t} Q(s_t,a_t)`.The DQN is trained by minimizing the mean-squared Bellman error loss, + which minimizes the TD-error, and thus enforce the validity of Bellman's equations. + + The loss is given by: + + .. math:: \mathcal{L}(\phi) = \mathbb{E}_{s,a}[(y - Q_{\phi}(s,a))^2] + + where the targets are given by :math:`y = r(s_t, a_t) + \gamma \max_{a' \in \mathcal{A}} Q(s_{t+1}, a')`. The + Q-learning TD(0) error is given by :math:`\delta = y - Q_{\phi}(s,a)` and thus, + :math:`\delta = (r(s_t, a_t) + \gamma \max_{a' \in \mathcal{A}} Q(s_{t+1}, a') - Q_{\phi}(s,a))` + + There are three important notes: + 1. In order to select the action that maximizes the Q-values, they have to be discrete. The Q-value function + approximator thus accepts as input the state and outputs for each action its corresponding Q-value. + 2. The data distribution that we have is dependent of the policy that we are optimizing. Additionally, the + generated samples are highly correlated in time. In order to address these issues and have i.i.d data samples, an + experience replay (ER) memory is used from which transition tuples :math:`(s_t, a_t, s_{t+1}, r_t, d, \gamma)` are + sampled (uniformly in general). Its necessity and utility has been shown in [1,2,3]. + 3. Using the same Q-value function approximator to evaluate the targets lead to instable behaviors [1,2]. In order + to improve stability during the training, an old Q-value function is thus used when computing the targets and + updated every once in a while (at a lesser frequency than the current Q-value function approximator). + + Exploration is usually performed in the action space by using :math:`\epsilon`-greedy exploration or a Boltzmann + policy. + + + Pseudo-algo + ----------- + + Pseudo-algorithm using DQN, an experience replay and :math:`\epsilon`-greedy exploration (mainly taken from [1], + modified a bit, and reproduced here for completeness):: + + 1. Initialize replay memory :math:`D` to capacity N + 2. Initialize action-value function Q with random weights :math:`\phi_0` + 3. for episode = 1 to M do + 4. for t = 1 to T do + 5. With probability :math:`\epsilon` select a random action :math:`a_t` otherwise select + :math:`a_t = argmax_a Q_{\phi_{target}}(s_t, a)` + 6. Execute action :math:`a_t`in environment and observe reward :math:`r_t`, next state :math:`s_{t+1}`, and + the binary signal :math:`d` if the task is done (d=1) or not (d=0). + 7. Store transition :math:`(s_t, a_t, r_t, s_{t+1}, d)` in D + 8. for batch = 1 to K do + 9. Sample random minibatch of transitions :math:`(s_t, a_t, r_t, s_{t+1}, d)` from D + 10. Compute target: y= r_t + \gamma (1 - d) \max_{a'} Q_{\phi_{target}}(s_{t+1}, a') + 11. Minimize MSBE loss :math:`(y - Q_{\phi}(s_t,a_t))^2` with respect to the parameters :math:`\phi`, by + performing a descent gradient step (for instance) + 12. Update :math:`\phi_{target}` every C steps (:math:`\phi_{target} = \phi`) or using polyak averaging + (:math:`\phi_{target} = \rho \phi_{target} + (1 - \rho) \phi`) + 13. end for + 14. end for + + + References: + [1] "Playing Atari with Deep Reinforcement Learning", Mnih et al., 2013 + [2] "Human-level Control through Deep Reinforcement Learning", Mnih et al., 2015 + [3] "Reinforcement Learning for robots using neural networks", Lin, 1993 + [4] "Reinforcement Learning (DQN) Tutorial" (in PyTorch): + https://pytorch.org/tutorials/intermediate/reinforcement_q_learning.html + [5] "A (Long) Peek into Reinforcement Learning": + https://lilianweng.github.io/lil-log/2018/02/19/a-long-peek-into-reinforcement-learning.html + [6] "Implementing Deep Reinforcement Learning Models with Tensorflow + OpenAI Gym": + https://lilianweng.github.io/lil-log/2018/05/05/implementing-deep-reinforcement-learning-models.html + """ + + def __init__(self, task, approximator, gamma=0.99, lr=5e-4, capacity=10000, num_workers=1): + """ + Initialize the DQN reinforcement learning algorithm. + + Args: + task (RLTask, Env): RL task/env to run. + approximator (ParametrizedQValueOutput, PolicyFromQValue): approximator to use and update. + gamma (float): discount factor (which is a bias-variance tradeoff). This parameter describes how much + importance has the future rewards we get. + lr (float): learning rate. + capacity (int): capacity of the experience replay storage. + num_workers (int): number of processes / workers to run in parallel. + """ + # check given approximator + if isinstance(approximator, ParametrizedQValueOutput): + policy = PolicyFromQValue(approximator) + q_value = approximator + elif isinstance(approximator, PolicyFromQValue): + policy = approximator + q_value = approximator.value + else: + raise TypeError("Expecting the given approximator to be an instance of `PolicyFromQValue`, or " + "`ParametrizedQValueOutput`, instead got: {}".format(type(approximator))) + + # evaluate target Q-value fct by copying Q-value function approximator + q_target = copy.deepcopy(q_value) + + # create action exploration strategy + exploration = EpsilonGreedyActionExploration(policy=policy, action=policy.actions) + + # create experience replay + storage = ExperienceReplay(capacity=capacity) + + # create target return estimator + estimator = TDQLearningReturn(q_value=q_value, target_qvalue=q_target, gamma=gamma) + + # create loss + loss = HuberLoss(MSBELoss(td_return=estimator), delta=1.) + + # create optimizer + optimizer = Adam(learning_rate=lr) + + # create target updater + target_updater = CopyParameter(sleep_count=100) # PolyakAveraging(rho=0.5) + + # define the 3 main steps in RL: explore, evaluate, and update + explorer = Explorer(task, exploration, storage, num_workers=num_workers) + evaluator = Evaluator(estimator) + updater = Updater(policy, storage, loss, optimizer, updaters={target_updater: q_target}) + + # initialize RL algorithm + super(DQN, self).__init__(explorer, evaluator, updater) diff --git a/pyrobolearn/algos/explorer.py b/pyrobolearn/algos/explorer.py index f65ce54..0f88d6a 100644 --- a/pyrobolearn/algos/explorer.py +++ b/pyrobolearn/algos/explorer.py @@ -160,7 +160,7 @@ class Explorer(object): print("5. \\pi(.|s): {}".format(dist)) print("6. log \\pi(a|s): {}".format([d.log_prob(act) for d in dist])) - self.storage.insert(next_obs, act, reward, masks=done, distributions=dist) + self.storage.insert(next_obs, act, reward, mask=done, distributions=dist) raw_input('enter') diff --git a/pyrobolearn/algos/ppo.py b/pyrobolearn/algos/ppo.py new file mode 100755 index 0000000..8cd9577 --- /dev/null +++ b/pyrobolearn/algos/ppo.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +"""Provide the Proximal Policy Optimization algorithm. + +The PPO is a model-free, on-policy, gradient-based on-policy algorithm that works with discrete and continuous action +spaces. +""" + +from pyrobolearn.algos.rl_algo import GradientRLAlgo, Explorer, Evaluator, Updater + +from pyrobolearn.policies import Policy +from pyrobolearn.values import ValueApproximator +from pyrobolearn.actorcritics import ActorCritic + +# from pyrobolearn.distributions import GaussianModule, DiagonalCovarianceModule, IdentityModule +from pyrobolearn.exploration import ActionExploration + +from pyrobolearn.storages import RolloutStorage +from pyrobolearn.samplers import StorageSampler +from pyrobolearn.estimators import GAE +from pyrobolearn.losses import CLIPLoss, ValueLoss, EntropyLoss +from pyrobolearn.optimizers import Adam + +from pyrobolearn import logger + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse", "Ilya Kostrikov", "OpenAI Spinning Up (Josh Achiam)"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class PPO(GradientRLAlgo): + r"""Proximal Policy Optimization + + Type:: model-free, on-policy, gradient-based policy search algorithm for discrete and continuous action spaces. + + This class implements the PPO algorithm which was presented in [1], and is inspired on the implementation of [2, 3]. + Compared to [2, 3], the algorithm is made such that it is more modular and flexible by decoupling and defining the + various concepts (storages, losses, estimators / returns, policies, value function approximators, and others) + outside the PPO class, and providing them as input to the constructor and thus privileging composition over + inheritance. + + + Mathematics + ----------- + + The loss to maximize is given by: + + .. math:: max_{\theta} L(\theta) = max_{\theta} \mathcal{E}[L^{CLIP}(\theta) - c_1 L^{VF}(\theta) + c_2 H(\theta)] + + with: + * Clip loss: + :math:`L^{CLIP}(\theta) = \mathbb{E}[ \min(r_t(\theta) A_t, clip(r_t(\theta), 1-\epsilon, 1+\epsilon) A_t) ]` + * Value function loss: :math:`L^{VF}(\theta) = (V_{target} - V_\theta(s))^2` + * Entropy loss: :math:`H(\theta) = - \int p(\theta) \log(p(\theta)) d\theta` + and where :math:`c_1` and :math:`c_2` are coefficients. + + This algorithm uses an actor-critic model, and a GAE estimator. + + + Properties + ---------- + + pros: + * this is currently one of the most efficient model-free on-policy search algorithm. + cons: + * because it is a gradient-based and an on-policy method, it requires several samples. + + UML: RLAlgo <-- GradientRLAlgo <-- PPO + + + Pseudo-algo + ----------- + + Pseudo-algorithm (taken from [3] and reproduce here for completeness):: + 1. Input: initial policy parameters :math:`\theta_0`, initial value function parameters :math:`\phi_0` + 2. for k=0,1,...,num_episodes do + 3. Exploration: Collect set of trajectories :math:`D_k=\{\tau_i\}` by running policy :math:`\pi_{\theta_k}` + in the environment. + 4. Evaluation: Compute rewards-to-go :math:`\hat{R}_t = \sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1})`, + evaluate advantage estimates :math:`\hat{A}_t` (using any method of advantage estimation) based on the + current value function :math:`V_{\phi_k}`. + 5. Update: + - Update the policy by maximizing the PPO-Clip objective (using e.g. gradient ascent): + :math:`\theta_{k+1} = \argmax_\theta \frac{1}{|D_k|T} \sum_{\tau \in D_k} \sum_{t=0}^T + \min(\frac{\pi_{\theta}(a_t|s_t)}{\pi_{\theta_k}(a_t | s_t)} A^{\pi_{\theta_k}(s_t, a_t), }` + - Update the value function by regression on the mean-squared error (using e.g. gradient descent): + :math:`\phi_{k+1} = \argmin_\phi \frac{1}{|D_k|T} \sum_{\tau \in D_k} \sum_{t=0}^T + (V_{\phi_k}(s_t) - \hat{R}_t)^2` + + + References: + [1] "Proximal Policy Optimization Algorithms", Schulman et al., 2017 + [2] "PyTorch Implementations of Reinforcement Learning Algorithms", Kostrikov, 2018 + [3] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/ppo.html + [4] "Policy Gradient Algorithms": + https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html + + Implementations: + - pytorch-a2c-ppo-acktr: https://github.com/ikostrikov/pytorch-a2c-ppo-acktr + - pytorch-rl: https://github.com/khushhallchandra/pytorch-rl + - DeepRL: https://github.com/ShangtongZhang/DeepRL + """ + + def __init__(self, task, approximators, gamma=0.99, tau=0.95, clip=0.2, lr=5e-4, l2_coeff=0.5, entropy_coeff=0.01, + num_workers=1, storage=None): + """ + Initialize the PPO algorithm. + + Args: + task (RLTask, Env): RL task/env to run + approximators (ActorCritic, [Policy, Value]): approximators to optimize + gamma (float): discount factor (which is a bias-variance tradeoff). This parameter describes how much + importance has the future rewards we get. + tau (float): trace-decay parameter (which is a bias-variance tradeoff). If :math:`\tau=1`, this results + in a Monte Carlo method, while :math:`\tau=0` results in a one-step TD methods. + clip (float): clip parameter + lr (float): learning rate + l2_coeff (float): coefficient for squared-error loss between the target and approximated value functions. + entropy_coeff (float): coefficient for entropy loss. + num_workers (int): number of workers (useful when parallelizing the code) + """ + logger.debug('creating PPO algorithm') + + # create actor critic + actor_critic = approximators + if isinstance(approximators, (tuple, list)): + policy, value = None, None + for approximator in approximators: + if isinstance(approximator, Policy): + policy = approximator + elif isinstance(approximator, ValueApproximator): + value = approximator + actor_critic = ActorCritic(policy, value) + if not isinstance(actor_critic, ActorCritic): + raise TypeError("Expecting 'actor_critic' to be an instance of ActorCritic") + + # get policy + policy = actor_critic.actor + + # create exploration strategy (wrap the original policy and specify how to explore) + # By default, for discrete actions it will use a Categorical distribution and for continuous actions, it will + # use a Gaussian with a diagonal covariance matrix. + logger.debug('creating the action exploration strategies for each action') + exploration = ActionExploration(policy) + + # create storage and estimator + states, actions = policy.states, policy.actions + logger.debug('create rollout storage') + storage = RolloutStorage(num_steps=1000, observation_shapes=states.merged_shape, + action_shapes=actions.merged_shape, num_processes=num_workers) + logger.debug('create return estimator (GAE)') + estimator = GAE(storage, gamma=gamma, tau=tau) + logger.debug('create storage sampler') + sampler = StorageSampler(storage) + + # create loss + logger.debug('create loss') + loss = CLIPLoss(clip=clip) + l2_coeff * ValueLoss() + entropy_coeff * EntropyLoss() + + # create optimizer + logger.debug('create Adam optimizer') + optimizer = Adam(learning_rate=lr) + + # define the 3 main steps in RL: explore, evaluate, and update + logger.debug('create explorer, evaluator, and updater') + explorer = Explorer(task, exploration, storage, num_workers=num_workers) + evaluator = Evaluator(estimator) + updater = Updater(policy, sampler, loss, optimizer) + + # initialize RL algorithm + super(PPO, self).__init__(explorer, evaluator, updater) diff --git a/pyrobolearn/algos/reinforce.py b/pyrobolearn/algos/reinforce.py new file mode 100755 index 0000000..2df271d --- /dev/null +++ b/pyrobolearn/algos/reinforce.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python +"""Provide the REINFORCE/VPG algorithm. + +The REINFORCE/VPG is a model-free, on-policy, gradient policy-based algorithm. +""" + +# from pyrobolearn.envs import Env +from pyrobolearn.policies import Policy +# from pyrobolearn.tasks import RLTask +from pyrobolearn.algos.rl_algo import GradientRLAlgo, Explorer, Evaluator, Updater + +from pyrobolearn.values import ValueApproximator +from pyrobolearn.actorcritics import ActorCritic + +from pyrobolearn.exploration import ActionExploration + +from pyrobolearn.storages import RolloutStorage +from pyrobolearn.samplers import StorageSampler +from pyrobolearn.estimators import ActionRewardEstimator +from pyrobolearn.losses import PGLoss, ValueLoss +from pyrobolearn.optimizers import Adam + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class REINFORCE(GradientRLAlgo): + r"""REINFORCE: REward Increment = Nonnegative Factor * Offset Reinforcement * Characteristic Eligibility + + Type:: model-free, on-policy, gradient-based policy search algorithm for discrete or continuous action spaces. + + This class implements the REINFORCE (aka Vanilla Policy Gradient (VPG)) algorithm. This was the first + policy-gradient method invented that uses the likelihood ratio trick. + "The key idea underlying policy gradients is to push up the probabilities of actions that lead to higher return, + and push down the probabilities of actions that lead to lower return, until you arrive at the optimal policy." [5] + + + Mathematics + ----------- + + The goal in reinforcement learning is to maximize the expected return over all the possible trajectories: + + .. math:: + + J(\theta) &= \int_{\mathbb{T}} p_{\theta}(\tau) R(\tau) d\tau \\ + &= \mathbb{E}_{\tau \sim p_{\theta}(\tau)}[R(\tau)] + + with + + .. math:: + + p_{\theta}(\tau) = p(s_0) \prod_{t=0}^{T} p(s_{t+1} | s_t, a_t) \pi_{\theta}(a_t | s_t) + + R_{\tau} = \frac{1}{T} \sum_{t=0}^{T} c_t r(s_t, a_t, s_{t+1}) + + where :math:`\pi` represents the policy (i.e. :math:`p(a_t|s_t)`) over which we have control, + :math:`\theta` represents the policy parameters (that can be learned/optimized), + :math:`\tau` is a trajectory (i.e. :math:`tau = (s_0, a_0, s_1,..., a_{T-1}, s_T)`, + :math:`\mathbb{T}` represents the set of all possible trajectories, + :math:`p(s_{t+1} | s_t, a_t)` is the dynamic model (aka the transition probability) which is determined + by the environment (and thus, we don't have any control over it), + :math:`R(\tau)` is the reward associated with the trajectory, :math:`r` is the instantaneous reward, and + :math:`c_t` is a weighting coefficient. + + By taking the gradient of this objective with respect to the parameters :math:`\theta` and using the likelihood + ratio trick :math:`\nabla_{\theta} p_{\theta}(\tau) = p_{\theta}(\tau) \nabla_{\theta} \log p_{\theta}(\tau)`, we + can write: + + .. math:: + + \nabla_{\theta} J(\theta) = \int_{\mathbb{T}} p_{\theta}(\tau) \nabla_{\theta} \log p_{\theta}(\tau)R(\tau)d\tau + = \mathbb{E}_{\tau \sim p_{\theta}(\tau)} [ \nabla_{\theta}p_{\theta}(\tau) R(\tau) ] + = \mathbb{E}[ \sum_{t=0}^{T} \nabla_{\theta} \log \pi_{\theta}(a_t|s_t) R(\tau) ] + + We can reduce the variance of the above gradient without introducing any biases by first noticing that actions + at a certain step do not affect previous rewards. Second, removing . Thus, the gradient for REINFORCE can be + rewritten as: + + ..math:: + + g = \mathbb{E}[ (\sum_{t=0}^{T} \nabla_{\theta} \log \pi_{\theta}(a_t|s_t)) + (\sum_{t'=t}^{T} c_t r(s_t, a_t, s_{t+1}) - b) ] + + where :math:`b` is the baseline. The optimal baseline can be computed using the value function :math:`V(s)`. + + Because the integral over all the trajectories is impractical in practice, we instead make use of Monte Carlo + expectations. + + + Properties + ---------- + + Properties: + - the expectation is guaranteed to converge to the true gradient [2] + Pros: + - easy to implement + - basic baseline that can be used when comparing with other algorithms + Cons: + - requires several samples to estimate correctly the gradient + - the learning can be quite unstable; the algorithm is pretty sensible to the value of the learning rate. This + can result in a policy that can vary a lot during the training. + + Notes:: + * We can use off-policy exploration using importance sampling. + + + Pseudo-algo + ----------- + + Pseudo-algorithm (taken from [5] and reproduced here for completeness):: + 1. Input: initial policy parameters :math:`\theta_0`, initial value function parameters :math:`\phi_0` + 2. for k=0,1,...,num_episodes do + 3. Exploration: Collect set of trajectories :math:`D_k=\{\tau_i\}` by running policy :math:`\pi_{\theta_k}` + in the environment. + 4. Evaluation: Compute rewards-to-go :math:`\hat{R}_t = \sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1})`, + evaluate advantage estimates :math:`\hat{A}_t` (using any method of advantage estimation) based on the + current value function :math:`V_{\phi_k}`. + 5. Update: + - Update the policy by maximizing the PG objective (using e.g. gradient ascent): + :math:`\theta_{k+1} = \argmax_\theta \frac{1}{|D_k|T} \sum_{\tau \in D_k} \sum_{t=0}^T + \pi_{\theta_k}(a_t | s_t) A^{\pi_{\theta_k}(s_t, a_t)}` + - Update the value function by regression on the mean-squared error (using e.g. gradient descent): + :math:`\phi_{k+1} = \argmin_\phi \frac{1}{|D_k|T} \sum_{\tau \in D_k} \sum_{t=0}^T + (V_{\phi_k}(s_t) - \hat{R}_t)^2` + + + References: + [1] "Simple Statistical Gradient-Following Algorithms for Connectionist Reinforcement Learning", Williams, 1992 + [2] "Policy Gradient Methods", Peters, 2010 (Scholarpedia) + [3] "A Survey on Policy Search for Robotics", Deisenroth et al., 2013 + [4] PyTorch Reinforce: https://github.com/pytorch/examples/blob/master/reinforcement_learning/reinforce.py + [5] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/vpg.html + [6] "Policy Gradient Algorithms": + https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html + + Other implementations: + - https://github.com/rll/rllab/blob/master/rllab/algos/vpg.py + - https://github.com/pytorch/examples/blob/master/reinforcement_learning/reinforce.py + - https://github.com/JamesChuanggg/pytorch-REINFORCE + - schulman's presentation + - SLM-LAB: https://github.com/kengz/SLM-Lab + - https://github.com/rlcode/reinforcement-learning/blob/master/2-cartpole/3-reinforce/cartpole_reinforce.py + """ + + def __init__(self, task, approximators, gamma=0.99, lr=3e-4, num_workers=1): + """ + Initialize the REINFORCE on-policy RL algorithm. + + Args: + task (RLTask, Env): RL task/env to run + approximators (Policy, [Policy, Value], ActorCritic): approximators to optimize + gamma (float): discount factor (which is a bias-variance tradeoff). This parameter describes how much + importance has the future rewards we get. + lr (float): learning rate + num_workers (int): number of processes / workers to run in parallel + """ + + # check approximators + policy, value, actor_critic = None, None, None + if isinstance(approximators, Policy): + policy = approximators + if not policy.is_parametric(): + raise ValueError("The policy should be parametric.") + elif isinstance(approximators, (tuple, list)): + for approximator in approximators: + if isinstance(approximator, Policy): + policy = approximator + elif isinstance(approximator, ValueApproximator): + value = approximator + actor_critic = ActorCritic(policy, value) + elif isinstance(approximators, ActorCritic): + policy = approximators.actor + value = approximators.critic + actor_critic = approximators + else: + raise TypeError("Expecting the approximators to be an instance of `Policy`, or `ActorCritic`, instead got:" + " {}".format(type(approximators))) + + # create exploration strategy + exploration = ActionExploration(policy) + + # create storage + states, actions = policy.states, policy.actions + storage = RolloutStorage(num_steps=1000, observation_shapes=states.shape, action_shapes=actions.shape, + num_processes=num_workers) + sampler = StorageSampler(storage) + + # create estimator + estimator = ActionRewardEstimator(storage, gamma=gamma) + + # create loss for policy + loss = PGLoss() + + # create optimizer for policy (and possibly value function) + optimizer = Adam(learning_rate=lr) + + # if value function, create its loss + if value is not None: + approximators = [policy, value] + value_loss = ValueLoss() + loss = [loss, value_loss] + else: + approximators = policy + + # define the 3 main steps in RL: explore, evaluate, and update + explorer = Explorer(task, exploration, storage, num_workers=num_workers) + evaluator = Evaluator(estimator) + updater = Updater(approximators, sampler, loss, optimizer) + + # initialize RL algorithm + super(REINFORCE, self).__init__(explorer, evaluator, updater) + + +# alias +VPG = REINFORCE diff --git a/pyrobolearn/algos/updater.py b/pyrobolearn/algos/updater.py index e131330..d8abbf7 100644 --- a/pyrobolearn/algos/updater.py +++ b/pyrobolearn/algos/updater.py @@ -41,7 +41,7 @@ class Updater(object): This class focuses on the third step of RL algorithms. """ - def __init__(self, approximators, sampler, losses, optimizers): + def __init__(self, approximators, sampler, losses, optimizers, updaters=None): """ Initialize the update phase. @@ -51,6 +51,7 @@ class Updater(object): losses (Loss, list/dict of losses): losses. If dict: key=approximator, value=loss. optimizers (Optimizer, or list/dict of optimizers): optimizer to use. If dict: key=approximator, value=optimizer. + updaters (None, dictionary, list of tuple): list of parameter updaters to run at the end. """ self.approximators = approximators self.sampler = sampler