update algos and exploration (need to finish cleaning)

pyrobolearn/__init__.py

@@ -1,6 +1,17 @@
 
 import sys
 
+# logging
+import logging
+
+# create logger
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+handler.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(name)s (%(levelname)s): %(message)s')
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
 # import simulators
 from . import simulators
 
@@ -10,6 +21,9 @@ from . import robots
 # import worlds
 from . import worlds
 
+# import physics randomizer
+from . import physics
+
 # import states
 from . import states
 
@@ -32,21 +46,31 @@ from . import approximators
 from . import policies
 
 # import values
+from . import values
 
 # import actor-critics
+from . import actorcritics
 
 # import dynamical models
+from . import dynamics
 
 # import tools (interfaces and bridges)
-# from . import tools  # uncommenting this will oblige the user to install a bunch of libraries which are not straightforward to install...
+from . import tools
+
+# import recorders
+from . import recorders
 
 # import tasks
 from . import tasks
 
 # import metrics
+from . import metrics
+
+# import losses
+from . import losses
 
 # import optimizers
-# from . import optimizers
+from . import optimizers
 
 # import algos
 from . import algos

pyrobolearn/algos/evaluator.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+"""Provide the Evaluator class used in the second step of RL algorithms
+
+The evaluator assesses the quality of the actions/trajectories performed by the policy using the given estimators.
+It is the step performed after the exploration phase, and before the update step.
+"""
+
+from pyrobolearn.estimators import Estimator
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class Evaluator(object):
+    r"""Evaluator
+
+    (Model-free) reinforcement learning algorithms requires 3 steps:
+    1. Explore: Explore and collect samples in the environment using the policy. The samples are stored in the
+                given memory/storage unit.
+    2. Evaluate: Assess the quality of the actions/trajectories using the estimators.
+    3. Update: Update the policy (and/or value function) parameters based on the loss
+
+    This class focuses on the second step of RL algorithms.
+    """
+
+    def __init__(self, estimator):
+        """
+        Initialize the Evaluation phase.
+
+        Args:
+            estimator (Estimator): estimator used to evaluate the actions performed by the policy.
+        """
+        self.estimator = estimator
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def estimator(self):
+        """Return the estimator used to evaluate the policy."""
+        return self._estimator
+
+    @estimator.setter
+    def estimator(self, estimator):
+        """Set the estimator."""
+        if not isinstance(estimator, Estimator):
+            raise TypeError("Expecting estimator to be an instance of `Estimator`, instead got: "
+                            "{}".format(type(estimator)))
+        self._estimator = estimator
+
+    @property
+    def storage(self):
+        """Return the storage unit."""
+        return self.estimator.storage
+
+    ###########
+    # Methods #
+    ###########
+
+    def evaluate(self):  # , storage):
+        """
+        Evaluate the actions.
+        """
+        self.estimator.evaluate(self.storage)
+
+    #############
+    # Operators #
+    #############
+
+    def __repr__(self):
+        """Return the representation string."""
+        return self.__class__.__name__
+
+    def __str__(self):
+        """Return the class string."""
+        return self.__class__.__name__
+
+    def __call__(self):
+        """Evaluate the estimator on the storage."""
+        self.evaluate()

pyrobolearn/algos/explorer.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+"""Provide the Explorer class used in the first step of RL algorithms
+
+It consists to explore and collect samples in the environment using the policy. The samples are stored in the
+given memory/storage unit which will be used to evaluate the policy, and then update its parameters.
+"""
+
+import inspect
+
+from pyrobolearn.tasks import RLTask
+from pyrobolearn.envs import Env
+from pyrobolearn.policies import Policy
+from pyrobolearn.exploration import Exploration
+from pyrobolearn.storages import RolloutStorage
+
+from pyrobolearn import logger
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class Explorer(object):
+    r"""Explorer
+
+    (Model-free) reinforcement learning algorithms requires 3 steps:
+    1. Explore: Explore and collect samples in the environment using the policy. The samples are stored in the
+                given memory/storage unit.
+    2. Evaluate: Assess the quality of the actions/trajectories using the estimators.
+    3. Update: Update the policy (and/or value function) parameters based on the loss
+
+    This class focuses on the first step of RL algorithms. It accepts the environment, and the exploration strategy
+    which wraps the policy.
+    """
+
+    def __init__(self, task, explorer, storage, num_workers=1):
+        """
+        Initialize the exploration phase.
+
+        Args:
+            task (Task, Env, tuple of Env and Policy): RL task or environment.
+            explorer (Exploration): policies.
+            storage (RolloutStorage): Rollout storage unit (=replay memory). It will save the rollouts in the storage
+                while exploring.
+            num_workers (int): number of processes / workers to run in parallel.
+        """
+        self.task = task
+        self.explorer = explorer
+        self.storage = storage
+        self.num_workers = int(num_workers)
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def task(self):
+        """Return the RL task."""
+        return self._task
+
+    @task.setter
+    def task(self, task):
+        """Set the RL task."""
+        if isinstance(task, (tuple, list)):
+            env, policy = None, None
+            for t in task:
+                if isinstance(t, Env):
+                    env = t
+                if isinstance(t, Policy):  # TODO if multiple policies
+                    policy = t
+            if env is None or policy is None:
+                raise ValueError("Expecting the task to be an instance of `RLTask` or a list/tuple of an environment "
+                                 "and policy.")
+            task = RLTask(env, policy)
+        if not isinstance(task, RLTask):
+            raise TypeError("Expecting the task to be an instance of `RLTask`, instead got: {}".format(type(task)))
+        self._task = task
+
+    @property
+    def policy(self):
+        """Return the policy."""
+        return self.task.policy
+
+    @property
+    def env(self):
+        """Return the environment."""
+        return self.task.environment
+
+    @property
+    def explorer(self):
+        """Return the exploration strategy."""
+        return self._explorer
+
+    @explorer.setter
+    def explorer(self, explorer):
+        """Set the exploration strategy."""
+        if inspect.isclass(explorer):  # if it is a class
+            explorer = explorer(self.policy)
+        if not isinstance(explorer, Exploration):
+            raise TypeError("Expecting explorer to be an instance of Exploration")
+        self._explorer = explorer
+
+    @property
+    def storage(self):
+        """Return the storage unit."""
+        return self._storage
+
+    @storage.setter
+    def storage(self, storage):
+        """Set the storage unit."""
+        if not isinstance(storage, RolloutStorage):
+            raise TypeError("Expecting the storage to be an instance of `RolloutStorage`, instead got: "
+                            "{}".format(type(storage)))
+        self._storage = storage
+
+    ###########
+    # Methods #
+    ###########
+
+    def explore(self, num_steps, deterministic=False):
+        """
+        Explore the environment.
+
+        Args:
+            num_steps (int): number of steps
+            deterministic (bool): if deterministic is True, then it does not explore in the environment.
+
+        Returns:
+            Rollout: memory storage
+        """
+        # reset environment
+        obs = self.env.reset()
+        print("\nExplorer - initial state: {}".format(obs))
+
+        # reset explorer
+        self.explorer.reset()
+
+        # run RL task for T steps
+        for step in range(num_steps):
+            # get action and corresponding distribution from policy
+            act, dist = self.explorer.act(obs, deterministic)
+
+            # perform one step in the environment
+            next_obs, reward, done, info = self.env.step(act)
+
+            # insert in storage
+            # print("\nExplorer:")
+            # print("1. Observation data: {}".format(obs.merged_torch_data))
+            # print("2. Action data: {}".format(act))
+            # print("3. Next observation data: {}".format(next_obs.merged_torch_data))
+            # print("4. Reward: {}".format(reward))
+            # print("5. \\pi(.|s): {}".format(dist))
+            # print("6. log \\pi(a|s): {}".format(dist.log_prob(act)))
+            self.storage.insert(obs.merged_torch_data, act.merged_torch_data, next_obs.data, reward, dist)
+
+            obs = next_obs
+            if done:
+                break
+
+        # # clear explorer
+        # self.explorer.clear()
+
+        return self.storage
+
+    #############
+    # Operators #
+    #############
+
+    def __repr__(self):
+        """Return a representation string about the class."""
+        return self.__class__.__name__
+
+    def __str__(self):
+        """Return a string describing the class."""
+        return self.__class__.__name__
+
+    def __call__(self, num_steps):
+        """Explore in the environment with the specified number of time steps."""
+        self.explore(num_steps)

pyrobolearn/algos/rl_algo.py

@@ -0,0 +1,401 @@
+#!/usr/bin/env python
+"""Provide the basic components for RL algorithms
+
+Dependencies:
+- `pyrobolearn.tasks`
+   - `pyrobolearn.approximators` (e.g. `pyrobolean.policies`, `pyrobolearn.values`, `pyrobolearn.models`,...)
+   - `pyrobolearn.envs`
+"""
+
+import numpy as np
+# from pathos.multiprocessing import Pool
+
+from pyrobolearn.algos.explorer import Explorer
+from pyrobolearn.algos.evaluator import Evaluator
+from pyrobolearn.algos.updater import Updater
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class RLAlgo(object):  # Algo):
+    r"""Reinforcement Learning Algorithm.
+
+    The RL algorithm takes as input a task (i.e. environment + policy), an optimizer, some hyperparameters (such
+    as the number of episodes, rollouts, and timesteps). Optional arguments include a learnable dynamic/transition
+    model, a value function approximator, a memory, and an exploration strategy.
+
+    RL problem
+    ----------
+
+    The RL problem can be depicted as:
+
+            s_t   \pi(a_t|s_t;\theta)  a_t
+               --->  Agent's policy ----
+              |                         |
+              |                         |
+              |                         |
+      s_{t+1}, ------  Environment  <---
+         r_t       p(s_{t+1}|s_t,a_t)
+
+    where :math:`s_t` is the state/observation, :math:`a_t` is the action, :math:`\pi_{\theta}` is the policy
+    parametrized by the parameters :math:`\theta`, :math:`r_t` is the reward returend by the environment,
+    and :math:`p(s_{t+1}|s_t,a_t)` is the dynamic model of the environment.
+
+    Note that normally there is a clear distinction between the observation and the state. Also, there is a difference
+    between the notation used in optimal control and model-based RL.
+
+
+    Markov Decision Process (MDP)
+    ----------------------------
+
+    A RL problem can be formally formulated as a Markov Decision Process (MDP) which is given as a 6-tuple
+    :math:`\{\rho_0, S, A, R, P, \gamma\}`, where :math:`\rho_0` is the initial state distribution, :math:`S` is
+    the set of of all valide states, :math:`A` is the set of all valid actions,
+    :math:`R: S \times A \times S \rightarrow \mathcal{R}` is the reward function with
+    :math:`r_t = R(s_t, a_t, s_{t+1})`, :math:`P: S \times A \rightarrow P(S)` is the state transition probability
+    function which describes the dynamic model of the environment, with :math:`p(s_{t+1} | s_t, a_t)` being the
+    probability, and :math:`\gamma \in [0,1]` is the discount factor used to calculate the return.
+
+
+    Goal of RL
+    ----------
+
+    The goal of RL is to maximize the expected return given by:
+
+    .. math::
+
+        \max_{\theta} J(\theta) &= \max_{\theta} \int_{\mathcal{T}} p_{\theta}(\tau) R(\tau) d\tau \\
+                                &= \mathcal{E}_{\pi_\theta}[]
+
+    where :math:`\mathcal{T}` represents the set of all possible trajectories covered by the policy :math:`\pi_\theta`,
+    :math:`p(\tau)` is the probability distribution over the trajectories :math:`\tau = (s_0, a_0, s_1, a_1, ...)`,
+    and :math:`R(\tau)` is the total return associated with the trajectory.
+
+    .. math::
+
+        p(\tau) &= p(s_0, a_0, ..., s_{T-1}, a_{T-1}, s_T) \\
+                &= p(s_0) \prod_{t=0}^{T-1} p(s_{t+1} | s_t, a_t) \pi_\theta(a_t | s_t)
+
+    where we used the product/chain rule of probability and the Markov property. :math:`p(s_{t+1}|s_t,a_t)`
+    represents the dynamics of the environment over which we have no control, and
+    :math:`\pi_{\theta}(a_t | s_t) = \pi(a_t | s_t; \theta)` is the policy that is parametrized by :math:`\theta`
+    on which we have control over it.
+
+
+    Taxonomy
+    --------
+
+    RL problems can be classified into different categories. We follow the taxonomy described in [3].
+
+    * Model-based vs Model-free
+        * Model-based: it knows about the dynamics of the environment (i.e. the transition function :math:`P`), i.e.
+            it knows :math:`p(s_{t+1} | s_t, a_t)`. This transition dynamics probability could have been computed
+            using mathematical equations or learned from data.
+        * Model-free: The dynamic model is not known. Model-free policy search consists of 3 main steps: explore,
+            evaluate, and update.
+    * Value-based <-- Actor-Critic --> Policy-based
+        * Value-based: Value-based means that it determines how good it is to be in a certain state. The policy is
+            then inferred from this knowledge.
+        * Policy-based: Policy-based (aka Policy search) directly optimizes the agent's policy
+            :math:`\pi_\theta(a | s)` which maps the state to action. Policy-based methods can further be subdivided
+            into 3 categories: Policy Gradient (PG) vs Expectation-Maximization (EM) vs Information Theory (Inf.Th.)
+        * Actor-critic: Actor-Critic combines both previous approaches.
+    * On-policy vs Off-policy
+        * On-policy: the collected data and the data on which we train the policy is the one collected by the same
+            policy.
+        * Off-policy: the policy that is being optimized and the policy that explores in the environment are different.
+            The former is called the target policy while the latter is the behavior policy which collects the data.
+            Off-policy tends to be a little bit slower than on-policy methods.
+    * Step-based vs Episode-based
+        * Step-based: the exploration is performed in the action space
+        * Episode-based: the exploration is performed in the parameter space
+
+
+    Pseudo-algo
+    -----------
+
+    The basic steps of RL algorithms are:
+    1. explore in the environment with the current policy and generate samples
+    1.5 if model-based, learn a model of the environment
+    2. evaluate the policy performance
+    3. Update the policy
+
+
+    Open Problems
+    -------------
+
+    - hierarchical
+    - exploration
+    - algorithms
+    - sample efficiency
+    - simulation vs reality
+
+
+    References (tutorials):
+        [1] "Reinforcement Learning: An Introduction", Sutton and Barto, 2018
+        [2] "Reinforcement Learning", Silver, UC London, 2015
+        [3] "A Survey on Policy Search for Robotics", Deisenroth et al., 2013
+        [4] "CS294: Deep Reinforcement Learning", Levine et al., UC Berkeley, 2017
+        [5] OpenAI - Spinning Up: https://spinningup.openai.com/
+    """
+
+    # def __init__(self, rlTask, exploration_strategy, storage, hyperparameters, optimizer=None, dynamic_model=None,
+    #              num_workers=1):
+    def __init__(self, explorer, evaluator, updater, hyperparameters={}, dynamic_model=None):  # , num_workers=1):
+        """
+        Initialize the reinforcement learning algorithm.
+
+        Args:
+            explorer (Explorer): explorer that specifies how to explore in the environment
+            evaluator (Evaluator): evaluate the actions
+            updater (Updater): update the approximators (rl, value-functions,...)
+            hyperparameters (dict): dictionary containing the hyperparameters
+            dynamic_model (None): dynamical model
+            num_workers (int): number of workers (useful when parallelizing the code)
+        """
+
+        super(RLAlgo, self).__init__()
+        # TODO: think about multiple agents/rewards
+
+        self.explorer = explorer
+        self.evaluator = evaluator
+        self.updater = updater
+
+        self.env = self.environment
+        self.dynamic_model = dynamic_model
+
+        # self.episodes = hyperparameters.get('episodes', 1) # nb of episodes
+        # self.rollouts = hyperparameters.get('rollouts', 1) # nb of rollouts per episode
+        # self.timesteps = hyperparameters.get('timesteps', 1000) # nb of timesteps per rollout
+        # TODO: define the number of iterations
+
+        self.best_reward = -np.infty
+        self.best_parameters = None
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def explorer(self):
+        """Return the explorer instance."""
+        return self._explorer
+
+    @explorer.setter
+    def explorer(self, explorer):
+        """Set the exploration phase."""
+        if not isinstance(explorer, Explorer):
+            raise TypeError("Expecting the explorer to be an instance of `Explorer`, instead got: "
+                            "{}".format(type(explorer)))
+        self._explorer = explorer
+
+    @property
+    def evaluator(self):
+        """Return the evaluator used to evaluate the actions taken by the policy."""
+        return self._evaluator
+
+    @evaluator.setter
+    def evaluator(self, evaluator):
+        """Set the evaluator for the 2nd phase of RL algorithms."""
+        if not isinstance(evaluator, Evaluator):
+            raise TypeError("Expecting the evaluator to be an instance of `Evaluator`, instead got: "
+                            "{}".format(type(evaluator)))
+        self._evaluator = evaluator
+
+    @property
+    def updater(self):
+        """Return the updater instance that is used to update the various approximator parameters."""
+        return self._updater
+
+    @updater.setter
+    def updater(self, updater):
+        """Set the updater for the 3rd phase of RL algorithms."""
+        if not isinstance(updater, Updater):
+            raise TypeError("Expecting the updater to be an instance of `Updater`, instead got: "
+                            "{}".format(type(updater)))
+        self._updater = updater
+
+    @property
+    def task(self):
+        """Return the RL task."""
+        return self.explorer.task
+
+    @property
+    def environment(self):
+        return self.task.environment
+
+    @property
+    def policy(self):
+        """Return the policy."""
+        return self.task.policy
+
+    @property
+    def exploration_strategy(self):
+        """Return the exploration strategy."""
+        return self.explorer.explorer
+
+    @property
+    def estimator(self):
+        """Return the estimator."""
+        return self.evaluator.estimator
+
+    @property
+    def storage(self):
+        """Return the storage unit."""
+        return self.explorer.storage
+
+    @property
+    def optimizers(self):
+        """Return the optimizers."""
+        return self.updater.optimizers
+
+    @property
+    def losses(self):
+        """Return the losses."""
+        return self.updater.losses
+
+    ###########
+    # Methods #
+    ###########
+
+    def init(self, explorer, evaluator, updater):
+        """Initialize the RL algo."""
+        self.explorer = explorer
+        self.evaluator = evaluator
+        self.updater = updater
+
+    def rollout(self, deterministic=True):
+        """
+        Run the policy in the environment.
+        """
+        # Reset the environment
+        state = self.env.reset()
+
+        if deterministic:
+            self.explorer.disable()
+
+        # Run policy in environment for T time steps
+        total_reward = 0
+        for _ in range(self.timesteps):
+            # run policy given the state
+            prev_state = state
+            action = self.policy.act(state, self.exploration)
+
+            # run one step in the environment
+            state, reward, done, info = self.env.step(action)
+            total_reward += reward
+
+            # save (s,a,s',r) in storage
+            self.storage.add(prev_state, action, state, reward)
+
+            # if episode is done
+            if done:
+                break
+
+        return total_reward
+
+    def train(self, num_steps, num_rollouts=1, num_episodes=1, verbose=False, seed=None):
+        """
+        Train the policy in the provided environment.
+
+        Args:
+            num_steps (int): number of step per rollout
+            num_rollouts (int): number of rollouts per episode (default: 1)
+            num_episodes (int): number of episodes (default: 1)
+            verbose(bool): if True, print details about the optimization process
+            seed (int): random seed
+
+        Returns:
+            dict: history
+        """
+        history = {}
+
+        # set the policy in training mode
+        self.policy.train(mode=True)
+
+        # for each episode
+        for ep in range(num_episodes):
+
+            # for each rollout
+            for rollout in range(num_rollouts):
+                # TODO: consider to learn the dynamic model if provided
+
+                # Explore, evaluate, and update
+                self.explorer(num_steps)
+                self.evaluator()
+                loss = self.updater()
+
+                # add the loss in the history
+                history.setdefault('loss', []).append(loss)
+
+        # set the policy in test mode
+        self.policy.train(mode=False)
+
+        return history
+
+    def test(self, num_steps, dt=0., use_terminating_condition=False, render=True):  # , storage):
+        """
+        Test the policy in the environment.
+
+        Args:
+            num_steps (int): number of steps
+            dt (float): time step
+            use_terminating_condition (bool): if we should use the terminating condition to end preemptively the
+                task if the policy succeeded or failed this last one
+            render (bool): render the test phase (default: True)
+
+        Returns:
+            list: list of results for each policy at each time step
+        """
+        results = self.task.run(self, num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition,
+                                render=render)
+        return results
+
+    #############
+    # Operators #
+    #############
+
+    def __repr__(self):
+        return self.__class__.__name__
+
+    def __str__(self):
+        return self.__class__.__name__
+
+
+class GradientRLAlgo(RLAlgo):
+    r"""Gradient based reinforcement learning algorithm.
+
+    These methods optimizes directly the policy by computing the gradient of the expected reward.
+
+    .. math:: g = \mathbb{E}[ \sum_{t=0}^\infty \psi_t \nabla_\theta \log \pi_\theta(a_t | s_t) ]
+
+    where:
+    * :math:`\pi_\theta(a_t | s_t)` is the policy parametrized by the vector :math:`\theta`. The policy predicts the
+        action :math:`a_t` given the state :math:`s_t`.
+    * :math:`\psi_t` is a type of return function (such as total reward, value function, advantage function,
+        TD residual,...)
+    """
+
+    # def __init__(self, task, exploration_strategy, memory, hyperparameters):
+    #    super(GradientRLAlgo, self).__init__(task, exploration_strategy, memory, hyperparameters)
+    def __init__(self, explorer, evaluator, updater, hyperparameters=None, dynamic_model=None):  # , num_workers=1):
+        super(GradientRLAlgo, self).__init__(explorer, evaluator, updater, hyperparameters, dynamic_model)
+        # , num_workers)
+        # self.optimizer = hyperparameters.get('optimizer') #TODO
+
+
+class EMRLAlgo(RLAlgo):
+    r"""Expectation-Maximization reinforcement learning algorithm.
+
+    """
+
+    def __init__(self, task, exploration_strategy, storage, hyperparameters):
+        super(EMRLAlgo, self).__init__(task, exploration_strategy, storage, hyperparameters)

pyrobolearn/algos/updater.py

@@ -0,0 +1,512 @@
+#!/usr/bin/env python
+"""Provide the Updater class used in the third and final step of RL algorithms
+
+The updater update the approximator (such as the policy and/or value function) parameters based on the loss, and
+using the specified optmizer.
+"""
+
+# TODO: makes the 5 following classes inherit from the same Parent class
+from pyrobolearn.approximators import Approximator
+from pyrobolearn.policies import Policy
+from pyrobolearn.values import Value
+from pyrobolearn.dynamics import DynamicModel
+from pyrobolearn.actorcritics import ActorCritic
+from pyrobolearn.exploration import Exploration  # TODO change that name to Explorer instead
+
+from pyrobolearn.losses import Loss
+from pyrobolearn.optimizers import Optimizer
+from pyrobolearn.storages import Storage, RolloutStorage, Batch
+from pyrobolearn.samplers import StorageSampler
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class Updater(object):
+    r"""Updater
+
+    (Model-free) reinforcement learning algorithms requires 3 steps:
+    1. Explore: Explore and collect samples in the environment using the policy. The samples are stored in the
+                given memory/storage unit.
+    2. Evaluate: Assess the quality of the actions/trajectories using the estimators.
+    3. Update: Update the policy (and/or value function) parameters based on the loss
+
+    This class focuses on the third step of RL algorithms.
+    """
+
+    def __init__(self, approximators, sampler, losses, optimizers):
+        """
+        Initialize the update phase.
+
+        Args:
+            approximators (list of Policy, Value, ActorCritic,...): approximators to update based on the given losses.
+            sampler (StorageSampler): sampler associated with the storage.
+            losses (Loss, list/dict of losses): losses. If dict: key=approximator, value=loss.
+            optimizers (Optimizer, or list/dict of optimizers): optimizer to use. If dict: key=approximator,
+                value=optimizer.
+        """
+        self.approximators = approximators
+        self.sampler = sampler
+        self.losses = losses
+        self.optimizers = optimizers
+        self._evaluator = ApproximatorEvaluator(approximators)
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def approximators(self):
+        """Return the list of approximators to update."""
+        return self._approximators
+
+    @approximators.setter
+    def approximators(self, approximators):
+        """Set the approximator instances."""
+        if not isinstance(approximators, list):
+            approximators = [approximators]
+        for approximator in approximators:
+            if not isinstance(approximator, (Approximator, Policy, Value, ActorCritic, DynamicModel, Exploration)):
+                raise TypeError("Expecting the approximator to be an instance of `Approximator`, `Policy`, `Value`, "
+                                "`ActorCritic`, `DynamicModel`, or `Exploration`. Instead got: "
+                                "{}".format(type(approximator)))
+        self._approximators = approximators
+        self._evaluator = ApproximatorEvaluator(self._approximators)
+
+    @property
+    def sampler(self):
+        """Return the sampler instance."""
+        return self._sampler
+
+    @sampler.setter
+    def sampler(self, sampler):
+        """Set the sampler."""
+        if not isinstance(sampler, StorageSampler):
+            raise TypeError("Expecting the sampler to be an instance of `StorageSampler`, instead got: "
+                            "{}".format(type(sampler)))
+        self._sampler = sampler
+        # self.storage = self._sampler.storage
+
+    @property
+    def storage(self):
+        """Return the storage unit."""
+        # return self._storage
+        return self.sampler.storage
+
+    @storage.setter
+    def storage(self, storage):
+        """Set the storage unit."""
+        self.sampler.storage = storage
+        # if not isinstance(storage, RolloutStorage):
+        #     raise TypeError("Expecting the storage to be an instance of `Storage`, instead got: "
+        #                     "{}".format(type(storage)))
+        # self._storage = storage
+
+    @property
+    def losses(self):
+        """Return the losses (one for each approximator)."""
+        return self._losses
+
+    @losses.setter
+    def losses(self, losses):
+        """Set the losses."""
+        # check that the losses are the correct data type
+        if not isinstance(losses, list):
+            losses = [losses]
+        for loss in losses:
+            if not isinstance(loss, Loss):
+                raise TypeError("Expecting the loss to be an instance of `Loss`, instead got: {}".format(type(loss)))
+
+        # check that the number of losses matches the number of approximators
+        if len(losses) != len(self.approximators):
+            raise ValueError("The number of losses does not match up with the number of approximators to update.")
+
+        # set the losses
+        self._losses = losses
+
+    @property
+    def optimizers(self):
+        """Return the optimizers used to optimize the parameters of the approximators."""
+        return self._optimizers
+
+    @optimizers.setter
+    def optimizers(self, optimizers):
+        """Set the optimizers."""
+        # check that the optimizers are the correct data type
+        if not isinstance(optimizers, list):
+            optimizers = [optimizers]
+        for optimizer in optimizers:
+            if not isinstance(optimizer, Optimizer):
+                raise TypeError("Expecting optimizer to be an instance of `Optimizer`, instead got: "
+                                "{}".format(type(optimizer)))
+
+        # check that the number of optimizers match the number of approximators / losses
+        if len(optimizers) != len(self.approximators):
+            if len(optimizers) == 1:
+                optimizers = optimizers * len(self.approximators)
+            else:
+                raise ValueError("Expecting the number of optimizers (={}) to match up with the number of "
+                                 "approximators / losses (={})".format(len(optimizers), len(self.approximators)))
+
+        # set the optimizers
+        self._optimizers = optimizers
+
+    @property
+    def evaluator(self):
+        """Return the evaluator instance which evaluates the approximators on the given batch.."""
+        return self._evaluator
+
+    ###########
+    # Methods #
+    ###########
+
+    def update(self, num_batches=10):
+        """
+        Update the given approximators (policies, value functions, etc).
+
+        Args:
+            num_batches (int): number of batches
+
+        Returns:
+            list: list of losses
+        """
+        # set the number of batches
+        self.sampler.num_batches = num_batches
+
+        # for each batch
+        for batch in self.sampler:
+
+            # evaluation with the current parameters
+            self.evaluator.evaluate(batch)
+
+            # update each approximator based on the loss on which it is evaluated and using the specified optimizer
+            for approximator, loss, optimizer in zip(self.approximators, self.losses, self.optimizers):
+
+                # compute loss on the data (the loss knows what to do)
+                loss = loss.compute(batch)
+
+                # update parameters
+                optimizer.optimize(approximator.parameters(), loss)
+
+        return self.losses
+
+    #############
+    # Operators #
+    #############
+
+    def __repr__(self):
+        """Return a representation string."""
+        return self.__class__.__name__
+
+    def __str__(self):
+        """Return a string describing the class."""
+        return self.__class__.__name__
+
+    def __call__(self, num_batches=10):  # , storage, losses):
+        """Update the approximators."""
+        self.update(num_batches=num_batches)
+
+
+class ApproximatorEvaluator(object):
+    r"""Approximators evaluator
+
+    Approximators evaluator used mostly during the update phase. Evaluate the various approximators on the given batch.
+
+    This consists:
+    - for policies, to compute :math:`\pi_{\theta}(a|s)` and :math:`\pi_{\theta}(.|s)` if possible.
+    - for value functions, to compute :math:`V_{\phi}(s)`, :math:`Q_{\phi}(s,a)`, and/or :math:`A_{\phi}`(s,a)
+    - for dynamic models, to compute :math:``
+    """
+
+    def __init__(self, approximators):
+        """
+        Initialize the evaluator for the approximators.
+
+        Args:
+            approximators ((list of) Approximator): approximators
+        """
+        self.approximators = approximators
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def approximators(self):
+        """Return the list of approximators to update."""
+        return self._approximators
+
+    @approximators.setter
+    def approximators(self, approximators):
+        """Set the list of approximators to update."""
+        if not isinstance(approximators, list):
+            approximators = [approximators]
+        for approximator in approximators:
+            if not isinstance(approximator, (Approximator, Policy, Value, ActorCritic, DynamicModel, Exploration)):
+                raise TypeError("Expecting the approximator to be an instance of `Approximator`, `Policy`, `Value`, "
+                                "`ActorCritic`, `DynamicModel`, or `Exploration`. Instead got: "
+                                "{}".format(type(approximator)))
+        self._approximators = approximators
+
+    ###########
+    # Methods #
+    ###########
+
+    def evaluate(self, batch):
+        """Evaluate the various approximators."""
+        if not isinstance(batch, Batch):
+            raise TypeError("Expecting the given batch storage to be an instance of `Batch`, instead got: "
+                            "{}".format(type(batch)))
+        # sub-evaluation with the current parameter
+        for approximator in self.approximators:
+            if isinstance(approximator, (Policy, Exploration)):
+                actions, action_distributions = approximator.evaluate(batch['observations'])
+                batch.current['actions'] = actions
+                batch.current['action_distributions'] = action_distributions
+            elif isinstance(approximator, Value):
+                values = approximator.evaluate(batch['observations'], batch['actions'])
+                batch.current['values'] = values
+            elif isinstance(approximator, ActorCritic):
+                actions, action_distributions, values = approximator.evaluate(batch['observations'], batch['actions'])
+                batch.current['actions'] = actions
+                batch.current['action_distributions'] = action_distributions
+                batch.current['values'] = values
+            elif isinstance(approximator, DynamicModel):
+                next_states, state_distributions = approximator.evaluate(batch['observations'], batch['actions'])
+                batch.current['next_states'] = next_states
+                batch.current['state_distributions'] = state_distributions
+            else:
+                raise TypeError("Expecting the approximator to be an instance of `Policy`, `Value`, `ActorCritic`, or "
+                                "`DynamicModel`, instead got: {}".format(type(approximator)))
+        return batch
+
+
+class PolicyEvaluator(object):
+    r"""Policy evaluator
+
+    Evaluate a policy by computing :math:`\pi_{\theta}(a|s)` and if possible the distribution :math:`\pi(.|s)`. The
+    policy is evaluated on a batch.
+    """
+
+    def __init__(self, policy, batch=None):
+        """Initialize the policy evaluator.
+
+        policy (Policy): policy to evaluate.
+        batch (None, Batch): initial batch.
+        """
+        self.policy = policy
+        self.batch = batch
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def policy(self):
+        """Return the policy instance."""
+        return self._policy
+
+    @policy.setter
+    def policy(self, policy):
+        """Set the policy."""
+        if not isinstance(policy, Policy):
+            raise TypeError("Expecting the given policy to be an instance of `Policy`, instead got: "
+                            "{}".format(type(policy)))
+        self._policy = policy
+
+    ###########
+    # Methods #
+    ###########
+
+    def evaluate(self, batch=None):
+        """Evaluate the policy on the given batch. If None, it will evaluate on the previous batch."""
+        # check batch
+        if batch is None:
+            batch = self.batch
+            if batch is None:
+                raise ValueError("Expecting a batch to be given.")
+
+        # evaluate policy
+        actions, action_distributions = self.policy.evaluate(batch['observations'])
+
+        # put them in the batch
+        batch.current['actions'] = actions
+        batch.current['action_distributions'] = action_distributions
+
+        # return batch
+        return batch
+
+
+class ValueEvaluator(object):
+    r"""Value evaluator
+
+    Evaluate a value by computing :math:`V_{\phi}(s)`, :math:`Q_{\phi}(s,a)`, and / or :math:`A_{\phi}(s,a)`.
+    The value is evaluated on a batch.
+    """
+
+    def __init__(self, value, batch=None):
+        """Initialize the value evaluator.
+
+        value (Value): value to evaluate.
+        batch (None, Batch): initial batch.
+        """
+        self.value = value
+        self.batch = batch
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def value(self):
+        """Return the value instance."""
+        return self._value
+
+    @value.setter
+    def value(self, value):
+        """Set the value function approximator."""
+        if not isinstance(value, Value):
+            raise TypeError("Expecting the given value to be an instance of `Value`, instead got: "
+                            "{}".format(type(value)))
+        self._value = value
+
+    ###########
+    # Methods #
+    ###########
+
+    def evaluate(self, batch=None):
+        """Evaluate the value on the given batch. If None, it will evaluate on the previous batch."""
+        # check batch
+        if batch is None:
+            batch = self.batch
+            if batch is None:
+                raise ValueError("Expecting a batch to be given.")
+
+        # evaluate value
+        values = self.value.evaluate(batch['observations'], batch['actions'])
+
+        # put them in the batch
+        batch.current['values'] = values
+
+        # return batch
+        return batch
+
+
+class ActorCriticEvaluator(object):
+    r"""ActorCritic evaluator
+
+    Evaluate an action by computing :math:`\pi_{\theta}(a|s)` and if possible the distribution :math:`\pi(.|s)`. It
+    also evaluates the value by computing :math:`V_{\phi}(s)`, :math:`Q_{\phi}(s,a)`, and / or :math:`A_{\phi}(s,a)`.
+    Both are evaluated on a batch.
+    """
+
+    def __init__(self, actorcritic, batch=None):
+        """Initialize the actorcritic evaluator.
+
+        actorcritic (ActorCritic): actorcritic to evaluate.
+        batch (None, Batch): initial batch.
+        """
+        self.actorcritic = actorcritic
+        self.batch = batch
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def actorcritic(self):
+        """Return the actor-critic instance."""
+        return self._actorcritic
+
+    @actorcritic.setter
+    def actorcritic(self, actorcritic):
+        """Set the actor-critic."""
+        if not isinstance(actorcritic, ActorCritic):
+            raise TypeError("Expecting the given actorcritic to be an instance of `ActorCritic`, instead got: "
+                            "{}".format(type(actorcritic)))
+        self._actorcritic = actorcritic
+
+    ###########
+    # Methods #
+    ###########
+
+    def evaluate(self, batch=None):
+        """Evaluate the actorcritic on the given batch. If None, it will evaluate on the previous batch."""
+        # check batch
+        if batch is None:
+            batch = self.batch
+            if batch is None:
+                raise ValueError("Expecting a batch to be given.")
+
+        # evaluate actorcritic
+        actions, action_distributions, values = self.actorcritic.evaluate(batch['observations'])  # , batch['actions'])
+
+        # put them in the batch
+        batch.current['actions'] = actions
+        batch.current['action_distributions'] = action_distributions
+        batch.current['values'] = values
+
+        # return batch
+        return batch
+
+
+class DynamicModelEvaluator(object):
+    r"""Dynamic model evaluator
+
+    Evaluate the next state given the current state and action.
+    """
+
+    def __init__(self, dynamic_model, batch=None):
+        """Initialize the dynamic_model evaluator.
+
+        dynamic_model (ActorCritic): dynamic_model to evaluate.
+        batch (None, Batch): initial batch.
+        """
+        self.dynamic_model = dynamic_model
+        self.batch = batch
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def dynamic_model(self):
+        """Return the dynamic_model instance."""
+        return self._dynamic_model
+
+    @dynamic_model.setter
+    def dynamic_model(self, dynamic_model):
+        """Set the dynamic model."""
+        if not isinstance(dynamic_model, DynamicModel):
+            raise TypeError("Expecting the given dynamic_model to be an instance of `ActorCritic`, instead got: "
+                            "{}".format(type(dynamic_model)))
+        self._dynamic_model = dynamic_model
+
+    ###########
+    # Methods #
+    ###########
+
+    def evaluate(self, batch=None):
+        """Evaluate the dynamic_model on the given batch. If None, it will evaluate on the previous batch."""
+        # check batch
+        if batch is None:
+            batch = self.batch
+            if batch is None:
+                raise ValueError("Expecting a batch to be given.")
+
+        # evaluate dynamic_model
+        next_states, state_distributions = self.dynamic_model.evaluate(batch['observations'], batch['actions'])
+
+        # put them in the batch
+        batch.current['next_states'] = next_states
+        batch.current['state_distributions'] = state_distributions
+
+        # return batch
+        return batch

pyrobolearn/estimators/estimator.py

@@ -142,7 +142,7 @@ class Estimator(object):
 
 
 class TotalRewardEstimator(Estimator):
-    r"""Total reward Estimator (aka finite-horizon undiscounted return)
+    r"""Total reward Estimator (aka (finite-horizon) discounted return)
 
     Return the total reward of the trajectory given by:
 

pyrobolearn/exploration/README.md

@@ -0,0 +1,9 @@
+## Exploration strategies
+
+This folder provides the various exploration strategies used in reinforcement learning. It basically wraps the policy, 
+and defines how the policy should explore in the environment. Exploration is mainly performed in the action space or 
+the (hyper-)parameter space.
+
+## what to look/check next?
+
+Check `distributions`, and `policies` folders.

pyrobolearn/exploration/__init__.py

@@ -0,0 +1,19 @@
+
+import logging
+
+# import exploration
+from .exploration import *
+
+# import action exploration
+from .actions import *
+
+# import parameter exploration
+from .parameters import *
+
+# create logger
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+handler.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(name)s (%(levelname)s): %(message)s')
+handler.setFormatter(formatter)
+logger.addHandler(handler)

pyrobolearn/exploration/actions/__init__.py

@@ -0,0 +1,12 @@
+
+# import action exploration
+from .action_exploration import *
+
+# import discrete action exploration
+from .discrete import *
+from .eps_greedy import *
+from .boltzmann import *
+
+# import continuous action exploration
+from .continuous import *
+from .gaussian import *

pyrobolearn/exploration/actions/action_exploration.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+r"""Provide the action exploration strategies.
+
+Action exploration is used in reinforcement learning algorithms and describe how the policy explores in the
+environment. Note that the policy is the only (probability) function that we have control over; we do not control the
+dynamic transition (probability) function nor the reward function. In action exploration, a probability distribution
+is put on the outputted action space :math:`a_t \sim \pi_{\theta}(\cdot|s_t)`. There are mainly two categories:
+exploration for discrete actions (which uses discrete probability distribution) and exploration for continuous action
+(which uses continuous probability distribution).
+
+Note that action exploration is a step-based exploration strategy where at each time step of an episode, an action is
+sampled based on the specified distribution.
+
+Action exploration might change a bit the structure of the policy while running.
+
+References:
+    [1] "Reinforcement Learning: An Introduction", Sutton and Barto, 2018
+"""
+
+import collections
+import torch
+
+from pyrobolearn.actions import Action
+import pyrobolearn as prl
+from pyrobolearn.exploration import Exploration
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class ActionExploration(Exploration):
+    r"""Action Exploration (aka Step-based RL)
+
+    Explore in the action space of the policy. At each time step, the action is sampled from the policy based
+    on the given distribution (hence the name 'step-based' RL).
+
+    Assume a policy is denoted by :math:`\pi_{\theta}(a|s)` which maps states :math:`s` to action :math`a`, and
+    is parametrized by :math:`\theta` which are the parameters that can be learned/optimized/trained. In action
+    space exploration, the actions :math:`a` are sampled from a probability distribution, such as a Gaussian
+    distribution such that :math:`a \sim \mathcal{N}(\pi_{\theta}(a|s), \Sigma)`.
+
+    This way of exploring is notably used in:
+    - several reinforcement learning algorithms (REINFORCE, TRPO, PPO, etc)
+    """
+
+    def __init__(self, policy, action=None, explorations=None):
+        """
+        Initialize the action exploration strategy.
+
+        Args:
+            policy (Policy): Policy to wrap.
+            action (Action, None): action space to explore.
+            explorations ((list of) ActionExploration, None): (list of) action exploration strategies. Each action
+                exploration strategy describes how the policy explores in the specified (discrete or continuous)
+                action space.
+        """
+        super(ActionExploration, self).__init__(policy)
+
+        # check action
+        if action is not None and not isinstance(action, Action):
+            raise TypeError("Expecting the given action to be an instance of `Action`, instead got: "
+                            "{}".format(type(action)))
+        self._action = action
+
+        # check exploration strategies
+
+        # if no exploration strategies set
+        if explorations is None:
+            explorations = []
+
+            # if no action has been defined
+            if self.action is None:
+                # go over each action of the policy, and add the corresponding exploration strategy based on the
+                # action type
+                for action in self.policy.actions:
+                    if action.is_discrete():
+                        prl.logger.debug('creating a Boltzmann action exploration with action of size: %d',
+                                         action.space[0].n)
+                        exploration = prl.exploration.actions.BoltzmannActionExploration(self.policy, action)
+                    elif action.is_continuous():
+                        exploration = prl.exploration.actions.GaussianActionExploration(self.policy, action)
+                    else:
+                        raise ValueError("Expecting an action to be discrete or continuous.")
+                    explorations.append(exploration)
+
+        else:
+            # check if an action has already been set
+            if self.action is not None:
+                raise ValueError("Expecting to be given an action or a list of explorations, not both.")
+
+            # transform the explorations to a list if not iterable
+            if not isinstance(explorations, collections.Iterable):
+                explorations = [explorations]
+
+            # check the length of exploration strategies and the number of actions in the policy
+            if len(explorations) != len(self.policy.actions):
+                raise ValueError("Expecting the number of actions (={}) to be the same as the number of exploration "
+                                 "strategy (={}).".format(len(self.policy.actions), len(explorations)))
+
+            actions = set([exploration.action for exploration in explorations])
+
+            # check that each exploration is set for each action
+            for action in self.policy.actions:
+                if action not in actions:
+                    raise ValueError("Expecting for each action in the policy to have its corresponding exploration "
+                                     "strategy. The following action was not found in the exploration strategies: "
+                                     "{}".format(action))
+
+        # set the exploration strategies
+        self._explorations = explorations
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def action(self):
+        """Return the specific action to explore. This might return None."""
+        return self._action
+
+    @property
+    def explorations(self):
+        """Return the list of exploration strategies."""
+        return self._explorations
+
+    ###########
+    # Methods #
+    ###########
+
+    def act(self,  state=None, deterministic=True, to_numpy=True, return_logits=False, apply_action=True):
+        r"""
+        Act/Explore in the environment given the states.
+
+        Args:
+            state (State): current state
+            deterministic (bool): True by default. It can only be set to False, if the policy is stochastic.
+            to_numpy (bool): If True, it will convert the data (torch.Tensors) to numpy arrays.
+            return_logits (bool): If True, in the case of discrete outputs, it will return the logits.
+            apply_action (bool): If True, it will call and execute the action.
+
+        Returns:
+            (list of) torch.Tensor: action(s)
+            (list of) torch.distributions.Distribution: policy distribution(s) :math:`\pi_{\theta}(\cdot | s)`
+        """
+        # TODO: finish to clean
+        print(state)
+        actions = self.policy.act(state, to_numpy=False, return_logits=True)
+
+        if deterministic:
+            return actions, None
+
+        # From deterministic output into stochastic outputs
+        # print("Actions before dist: {}").format(actions.train_data)
+        print("Exploration strategy - actions: {}".format(actions))
+        self.dist = self.distribution(actions)
+        actions = self.dist.sample()
+        print("Exploration strategy - sampled action: {}".format(actions))
+        if isinstance(actions, torch.Tensor):
+            if actions.requires_grad:
+                self.policy.actions.data = actions.detach().numpy()
+            else:
+                self.policy.actions.data = actions.numpy()
+        else:
+            self.policy.actions.data = actions
+
+        return actions, self.dist
+
+    def mode(self):
+        """Return the mode of the distributions."""
+        actions = self.dist.mode()
+        return actions
+
+    def sample(self):
+        """Sample an action from the distribution."""
+        actions = self.dist.sample()
+        return actions
+
+    def action_log_prob(self, actions):
+        """Return the log probability evaluated at the given actions."""
+        return self.dist.log_probs(actions)
+
+    def action_prob(self, actions):
+        """Return the probability evaluated at the given actions."""
+        return torch.exp(self.dist.log_probs(actions))
+
+    def entropy(self):
+        """Return the entropy of the distribution."""
+        return self.dist.entropy().mean()

pyrobolearn/exploration/actions/boltzmann.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+r"""Provide the discrete Boltzmann action exploration.
+
+The Boltzmann exploration strategy consists to explore in the discrete action space of policies by using a
+categorical distribution on action probabilities. The probabilities are often computed using a softmax function
+which maps the values of the logits to correct probabilities (i.e. each probability is between 0 and 1, and the
+sum of them sums to 1). That is, compared to epsilon-greedy which selects another action uniformly, Boltzmann
+exploration selects an action based on its weight which is outputted by the policy.
+"""
+
+
+import torch
+
+from pyrobolearn.distributions.modules import CategoricalModule, IdentityModule
+from pyrobolearn.exploration.actions.discrete import DiscreteActionExploration
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class BoltzmannActionExploration(DiscreteActionExploration):
+    r"""Boltzmann action exploration.
+
+    The Boltzmann exploration strategy consists to explore in the discrete action space of policies by using a
+    categorical distribution on action probabilities. The probabilities are often computed using a softmax function
+    which maps the values of the logits to correct probabilities (i.e. each probability is between 0 and 1, and the
+    sum of them sums to 1). That is, compared to epsilon-greedy which selects another action uniformly, Boltzmann
+    exploration selects an action based on its weight which is outputted by the policy.
+    """
+
+    def __init__(self, policy, action):
+        """
+        Initialize the Boltzmann action exploration strategy.
+
+        Args:
+            policy (Policy): policy to wrap.
+            action (Action): discrete actions.
+        """
+        super(BoltzmannActionExploration, self).__init__(policy, action=action)
+
+        # create Categorical module
+        logits = IdentityModule()
+        self._module = CategoricalModule(logits=logits)

pyrobolearn/exploration/actions/continuous.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+r"""Provide the continuous action exploration strategies.
+
+Action exploration is used in reinforcement learning algorithms and describe how the policy explores in the
+environment. Note that the policy is the only (probability) function that we have control over; we do not control the
+dynamic transition (probability) function nor the reward function. In action exploration, a probability distribution
+is put on the outputted action space :math:`a_t \sim \pi_{\theta}(.|s_t)`. There are mainly two categories:
+exploration for discrete actions (which uses discrete probability distribution) and exploration for continuous action
+(which uses continuous probability distribution).
+
+Note that action exploration is a step-based exploration strategy where at each time step of an episode, an action is
+sampled based on the specified distribution.
+
+Action exploration might change a bit the structure of the policy while running.
+
+References:
+    [1] "Reinforcement Learning: An Introduction", Sutton and Barto, 2018
+"""
+
+from pyrobolearn.exploration.actions.action_exploration import ActionExploration
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class ContinuousActionExploration(ActionExploration):
+    r"""Continuous action exploration.
+
+    Continuous action exploration strategies use continuous probability distributions on the (continuous) actions.
+    """
+
+    def __init__(self, policy, action):
+        """
+        Initialize the continuous action exploration strategy.
+
+        Args:
+            policy (Policy): policy to wrap.
+            action (action): continuous action.
+        """
+        super(ContinuousActionExploration, self).__init__(policy, action=action)
+        if not self.action.is_continuous():
+            raise ValueError("Expecting the given action to be continuous.")

pyrobolearn/exploration/actions/discrete.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+r"""Provide the discrete action exploration strategies.
+
+Action exploration is used in reinforcement learning algorithms and describe how the policy explores in the
+environment. Note that the policy is the only (probability) function that we have control over; we do not control the
+dynamic transition (probability) function nor the reward function. In action exploration, a probability distribution
+is put on the outputted action space :math:`a_t \sim \pi_{\theta}(.|s_t)`. There are mainly two categories:
+exploration for discrete actions (which uses discrete probability distribution) and exploration for continuous action
+(which uses continuous probability distribution).
+
+Note that action exploration is a step-based exploration strategy where at each time step of an episode, an action is
+sampled based on the specified distribution.
+
+Action exploration might change a bit the structure of the policy while running.
+
+References:
+    [1] "Reinforcement Learning: An Introduction", Sutton and Barto, 2018
+"""
+
+from pyrobolearn.exploration.actions.action_exploration import ActionExploration
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class DiscreteActionExploration(ActionExploration):
+    r"""Discrete action exploration
+
+    Discrete action exploration strategies use discrete probability distributions on the (discrete) actions.
+    """
+
+    def __init__(self, policy, action):
+        """
+        Initialize the discrete action exploration strategy.
+
+        Args:
+            policy (Policy): policy to wrap.
+            action (action): discrete action.
+        """
+        super(DiscreteActionExploration, self).__init__(policy, action=action)
+        if not self.action.is_discrete():
+            raise ValueError("Expecting the given action to be discrete.")

pyrobolearn/exploration/actions/eps_greedy.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+r"""Provide the discrete epsilon-greedy action exploration.
+
+The epsilon-greedy exploration strategy consists to explore in the discrete action space of policies. Specifically,
+it selects the best action :math:`a*` with probability :math:`p = (1 - \epsilon)`, or another action
+:math:`a \in A\{a*}` randomly (based on uniform distribution) with probability :math:`p = \frac{\epsilon}{|A|-1}`.
+"""
+
+import torch
+
+from pyrobolearn.exploration.actions.discrete import DiscreteActionExploration
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class EpsilonGreedyActionExploration(DiscreteActionExploration):
+    r"""Epsilon-greedy action exploration.
+
+    The epsilon-greedy exploration strategy consists to explore in the discrete action space of policies. Specifically,
+    it selects the best action :math:`a*` with probability :math:`p = (1 - \epsilon)`, or another action
+    :math:`a \in A\{a*}` randomly (based on uniform distribution) with probability :math:`p = \frac{\epsilon}{|A|-1}`.
+    """
+
+    def __init__(self, policy, action):
+        """
+        Initialize the epsilon-greedy action exploration strategy.
+
+        Args:
+            policy (Policy): policy to wrap.
+            action (Action): discrete actions.
+        """
+        super(EpsilonGreedyActionExploration, self).__init__(policy, action=action)

pyrobolearn/exploration/actions/gaussian.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+"""Provide the continuous Gaussian action exploration strategies.
+
+The Gaussian exploration strategy consists to explore in the continuous action space of policies by using a
+gaussian distribution on action probabilities.
+"""
+
+import torch
+
+from pyrobolearn.distributions.modules import *
+from pyrobolearn.exploration.actions.continuous import ContinuousActionExploration
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class GaussianActionExploration(ContinuousActionExploration):
+    r"""Gaussian continuous action exploration.
+
+    The Gaussian exploration strategy consists to explore in the continuous action space of policies by using a
+    gaussian distribution on action probabilities.
+    """
+
+    def __init__(self, policy, action, module=None):
+        """
+        Initialize the Gaussian action exploration strategy.
+
+        Args:
+            policy (Policy): policy to wrap.
+            action (Action): continuous actions.
+            module (None, GaussianModule): Gaussian module.
+        """
+        super(GaussianActionExploration, self).__init__(policy, action=action)
+
+        # create Gaussian module if necessary
+        if module is None:
+            mean = IdentityModule()
+            covariance = DiagonalCovarianceModule(num_inputs=self.policy.base_output.size(-1),
+                                                  num_outputs=action.size)
+            module = GaussianModule(mean=mean, covariance=covariance)
+
+        # check that the module is a Gaussian Module
+        if not isinstance(module, GaussianModule):
+            raise TypeError("Expecting the given 'module' to be an instance of `GaussianModule`, instead got: "
+                            "{}".format(type(module)))
+
+        self._module = module

pyrobolearn/exploration/exploration.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+"""Provide the various exploration strategies.
+
+Specifically, this file provides the main abstract `Exploration` class from which all the other exploration strategies
+inherit from. Exploration is mainly useful in reinforcement learning, and can be carried out in the action or parameter
+space.
+
+References:
+    [1] "Reinforcement Learning: An Introduction", Sutton and Barto, 2018
+"""
+
+import torch
+
+from pyrobolearn.policies import Policy
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class Exploration(object):
+    r"""Exploration class
+
+    Exploration strategy: it wraps the policy, and defines how the policy should explore in the environment.
+
+    There are 2 main exploration strategies:
+    1. exploration in the action space of the policy
+    2. exploration in the parameter space of the policy
+
+    Exploration can be, for instance, carried out based on the uncertainty of an action, a dynamic model (which
+    predicts the next state given the current state and action), a value fct evaluated at the given state, and so on.
+    This is also called "curiosity" and is linked to the notion of entropy (as the entropy is related to the notion
+    of uncertainty).
+
+    Exploration is crucial because it defines a stochastic policy, and thus a probability distribution. This in turn
+    enables the use of probability concepts such as 'maximizing the likelihood or marginal likelihood'.
+    Policy search algorithms currently only works with stochastic policies.
+    """
+
+    def __init__(self, policy):
+        """
+        Initialize the Exploration strategist.
+
+        Args:
+            policy (Policy): Policy to wrap.
+        """
+        self.policy = policy
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def policy(self):
+        """Return the policy instance"""
+        return self._policy
+
+    @policy.setter
+    def policy(self, policy):
+        """Set the policy instance."""
+        if not isinstance(policy, Policy):
+            raise TypeError("Expecting policy to be an instance of `Policy`, instead got: {}.".format(type(policy)))
+        self._policy = policy
+
+    ###########
+    # Methods #
+    ###########
+
+    def reset(self):
+        """Reset the exploration strategy, which can be useful at the beginning of an episode."""
+        self.policy.reset()
+
+    def _act(self, state=None, to_numpy=True, return_logits=False, apply_action=True):
+        """Perform the exploratory action."""
+        raise NotImplementedError
+
+    def act(self, state=None, deterministic=False, to_numpy=True, return_logits=False, apply_action=True):
+        """Perform the action given the state.
+
+        Args:
+            state (State, list of np.array, list of torch.Tensor): current state.
+            deterministic (bool): If True, it will return a deterministic action. If False, it will explore.
+            to_numpy (bool): If True, it will convert the data (torch.Tensors) to numpy arrays.
+            return_logits (bool): If True, in the case of discrete outputs, it will return the logits.
+            apply_action (bool): If True, it will call and execute the action.
+
+        Returns:
+            (list of) np.array / torch.Tensor: action data
+        """
+        if deterministic:
+            return self.policy.act(state, deterministic=True, to_numpy=to_numpy, return_logits=return_logits,
+                                   apply_action=apply_action)
+        else:  # explore using the distribution
+            return self._act(state, to_numpy=to_numpy, return_logits=return_logits, apply_action=apply_action)
+
+    # def step(self, states):
+    #     """Perform one step using the policy with the corresponding exploration strategy."""
+    #     pass
+
+    # def clear(self):
+    #     """Called at the end of an episode to reset a policy or clear whatever has been done."""
+    #     pass
+
+
+# class ModelUncertaintyExploration(Exploration):
+#     r"""Model Uncertainty Exploration
+#
+#     Exploration based on the uncertainty of a learned dynamic model.
+#
+#     References:
+#     [1]
+#     """
+#
+#     def __init__(self, policy):
+#         super(ModelUncertaintyExploration, self).__init__(policy)

pyrobolearn/exploration/parameters/gaussian.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""Provide the continuous Gaussian parameter exploration strategies.
+
+The Gaussian parameter exploration strategy consists to explore in the parameter space of policies by using a gaussian
+distribution on the parameters.
+"""
+
+import torch
+
+# from pyrobolearn.distributions.modules import FixedMeanModule, FixedCovarianceModule, GaussianModule
+from pyrobolearn.distributions.gaussian import Gaussian
+from pyrobolearn.exploration.parameters.parameter_exploration import ParameterExploration
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class GaussianParameterExploration(ParameterExploration):
+    r"""Gaussian continuous parameter exploration.
+
+    The Gaussian exploration strategy consists to explore in the continuous parameter space of policies by using a
+    gaussian distribution on parameters.
+    """
+
+    def __init__(self, policy, variance=1., module=None):
+        """
+        Initialize the Gaussian action exploration strategy.
+
+        Args:
+            policy (Policy): policy to wrap.
+            variance (float): variance parameter for the covariance.
+            module (None, GaussianModule): Gaussian module.
+        """
+        super(GaussianParameterExploration, self).__init__(policy)
+
+        # create Gaussian module if necessary
+        if module is None:
+            mean = torch.tensor(self.parameters, requires_grad=True)
+            # mean = FixedMeanModule(mean=mean)
+            covariance = variance * torch.eye(self.size, requires_grad=True)
+            # covariance = FixedCovarianceModule(covariance=covariance)
+            # module = GaussianModule(mean=mean, covariance=covariance)
+            module = Gaussian(mean=mean, covariance=covariance)
+
+        # check that the module is a Gaussian Module
+        if not isinstance(module, GaussianModule):
+            raise TypeError("Expecting the given 'module' to be an instance of `GaussianModule`, instead got: "
+                            "{}".format(type(module)))
+
+        self._module = module
+
+    @property
+    def module(self):
+        """Return the module instance."""
+        return self._module
+
+    def sample(self):
+        """Sample the parameters from the """
+        parameters = self.module.rsample((1,))  # rsample allows to get the gradients
+        self.policy.set_vectorized_parameters(vector=parameters)
+

pyrobolearn/exploration/parameters/parameter_exploration.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+"""Provide the parameter exploration strategies.
+
+Parameter exploration is used in reinforcement learning algorithms, and describes how the policy explores in the
+environment. Note that the policy is the only (probability) function that we have control over; we do not control the
+dynamic transition (probability) function nor the reward function. In parameter exploration, we explore the parameter
+space of the policy.
+
+Note that parameter exploration is an episode-based exploration strategy where the parameters of the policy are only
+perturbed at the beginning of an episode, and unchanged during that particular episode.
+
+Parameter exploration might change a bit the structure of the policy while running.
+
+References:
+    [1] "Evolution strategies as a scalable alternative to reinforcement learning", Salimans et al., 2017
+    [2] "Parameter Space Noise for Exploration", Plappert et al., 2018
+"""
+
+import torch
+
+from pyrobolearn.exploration import Exploration
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class ParameterExploration(Exploration):
+    r"""Parameter Exploration (aka Episode-based RL)
+
+    Explore in the parameter space of the policy. At each episode, we apply a small variation on the parameters
+    of the policy and keep it fixed during the whole episode (hence the name 'episode-based' RL).
+
+    Assume a policy is denoted by :math:`\pi_{\theta}(a|s)` which maps states :math:`s` to action :math`a`, and
+    is parametrized by :math:`\theta` which are the parameters that can be learned/optimized/trained. In parameter
+    space exploration, the parameters :math:`\theta` are sampled from a probability distribution, such as a
+    Gaussian distribution such that :math:`\theta \sim \mathcal{N}(\theta_k, \Sigma)`.
+
+    This way of exploring is notably used in:
+    - population-based algorithms such as evolutionary algorithms (ES, NEAT). Note that in `NEAT`, the topology
+    of the learning model (i.e. neural network) is also explored along with its weights.
+    - reinforcement learning algorithms like PoWER, and others.
+
+    Pros:
+    - when sampling several parameters (and thus policies), each one of them can be evaluated in a parallel manner.
+    These policies are thus independent.
+    - giving the same state to a policy results in the same action (in contrast to action exploration).
+
+    References:
+        [1] "Parameter Space Noise for Exploration", Plappert et al., 2018
+    """
+
+    def __init__(self, policy):
+        super(ParameterExploration, self).__init__(policy)
+        self._parameters = policy.get_vectorized_parameters(to_numpy=False)
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def parameters(self):
+        """Returns the parameters."""
+        return self._parameters
+
+    @property
+    def size(self):
+        """Returns the dimension of the parameters."""
+        return self.parameters.size(-1)
+
+    ###########
+    # Methods #
+    ###########
+
+    def reset(self):
+        # sample new set of parameters for policy
+        pass
+
+    def act(self, state=None, deterministic=True, to_numpy=True, return_logits=False, apply_action=True):
+        """Perform the action given the state."""
+        actions = self.policy.act(state)
+
+
+# class ModelUncertaintyExploration(Exploration):
+#     r"""Model Uncertainty Exploration
+#
+#     Exploration based on the uncertainty of a learned dynamic model.
+#
+#     References:
+#     [1]
+#     """
+#
+#     def __init__(self, policy):
+#         super(ModelUncertaintyExploration, self).__init__(policy)

pyrobolearn/optimizers/__init__.py

@@ -1,6 +1,6 @@
 
 # import optimizer
-from optimizer import Optimizer
+from .optimizer import Optimizer
 
 # import scipy optimizer
 # from scipy_optimizer import Scipy
@@ -21,7 +21,7 @@ from optimizer import Optimizer
 # from gpyopt_optimizer import BayesianOptimizer
 
 # import torch optimizers
-from torch_optimizer import *
+from .torch_optimizer import *
 
 # import Contact-Invariant Optimizer
 # from cio import CIO

pyrobolearn/optimizers/torch_optimizer.py

@@ -12,7 +12,7 @@ References:
 import torch.nn as nn
 import torch.optim as optim
 
-from optimizer import Optimizer
+from pyrobolearn.optimizers.optimizer import Optimizer
 
 
 __author__ = "Brian Delhaisse"

pyrobolearn/policies/policy.py

@@ -512,13 +512,19 @@ class Policy(object):
 
     def train(self, mode=True):
         """
-        Set the policy to train mode.
+        Set the policy in training mode.
 
         Args:
             mode (bool): if True, set the policy in train mode.
         """
         self.train_mode = mode
 
+    def eval(self):
+        """
+        Set the policy in evaluation mode.
+        """
+        self.train(mode=False)
+
     def reset(self, reset_processors=False, *args, **kwargs):
         """
         Reset the policy.

pyrobolearn/rewards/reward.py

@@ -169,30 +169,36 @@ class Reward(object):
 
     @property
     def state(self):
+        """Return the state instance."""
         return self._state
 
     @state.setter
     def state(self, state):
+        """Set the state."""
         if state is not None and not isinstance(state, State):
             raise TypeError("Expecting state to be None or an instance of State.")
         self._state = state
 
     @property
     def action(self):
+        """Return the action instance."""
         return self._action
 
     @action.setter
     def action(self, action):
+        """Set the action."""
         if action is not None and not isinstance(action, Action):
             raise TypeError("Expecting action to be None or an instance of Action.")
         self._action = action
 
     @property
     def rewards(self):
+        """Return the inner rewards."""
         return self._rewards
 
     @rewards.setter
     def rewards(self, rewards):
+        """Set the inner rewards."""
         if rewards is None:
             rewards = []
         elif isinstance(rewards, collections.Iterable):
@@ -210,17 +216,29 @@ class Reward(object):
     ###########
 
     def has_rewards(self):
+        """Check if there are inner rewards."""
         return len(self._rewards) > 0
 
     @staticmethod
     def is_maximized():
+        """Check if it is maximized."""
         return True
 
     def reset(self):
+        """Reset the rewards."""
         for reward in self.rewards:
             reward.reset()
 
-    def compute(self):
+    def compute(self):  # **kwargs):
+        """Compute the reward and return the scalar value
+
+        Warnings: by default, *args and **kwargs are disabled as it could lead to several problems:
+        1. As more and more rewards will become available, there might share the same argument name if the programmer
+        is not careful, which could lead to bugs that are difficult to detect.
+        2. It is better to provide the arguments during the initialization of the reward class. If the user has
+        a variable that might change, create a class for that variable and in the corresponding reward's compute
+        method, check what is its value.
+        """
         pass
 
     #############
@@ -234,8 +252,8 @@ class Reward(object):
             lst = [reward.__repr__() for reward in self.rewards]
             return ' + '.join(lst)
 
-    def __call__(self, *args, **kwargs):
-        return self.compute()
+    def __call__(self):  # **kwargs):
+        return self.compute()  # **kwargs)
 
     # for unary and binary operators, see `__init__()` method.
 

pyrobolearn/storages/storage.py

@@ -16,6 +16,8 @@ import numpy as np
 import torch
 from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler
 
+from pyrobolearn import logger
+
 
 __author__ = "Brian Delhaisse"
 __copyright__ = "Copyright 2018, PyRoboLearn"
@@ -114,7 +116,7 @@ class PyTorchStorage(Storage):
     def _to(self, item, device=None, dtype=None):
         """Send the item to the specified device and convert it to the specified data type."""
         if isinstance(item, torch.Tensor):
-            print('setting {} to {} with dtype={}'.format(item, device, dtype))
+            logger.debug('setting tensor of size {} to {} with dtype={}'.format(item.size(), device, dtype))
             item = item.to(device=device, dtype=dtype)
         elif isinstance(item, dict):
             for key, value in item.items():
@@ -328,10 +330,10 @@ class DictStorage(dict, PyTorchStorage):
         """Get the attribute using the key name. That is, instead of `D['name']`, you can do `D.name`."""
         return self[name]
 
-    def __setattr__(self, key, value):
-        """Set the attribute using the given key and value. That is, instead of `D[key] = value`, you can do
-        `D.key = value`"""
-        self[key] = value
+    # def __setattr__(self, key, value):
+    #     """Set the attribute using the given key and value. That is, instead of `D[key] = value`, you can do
+    #     `D.key = value`"""
+    #     self[key] = value
 
 
 # alias
@@ -382,13 +384,12 @@ class RolloutStorage(DictStorage):
         # recurrent_hidden_state_size (int): size of the internal state
         print("\nStorage: observation shape: {}".format(observation_shapes))
         print("Storage: action shape: {}".format(action_shapes))
-
+        super(RolloutStorage, self).__init__()
         self._step = 0
         self._num_steps = int(num_steps)
         self._num_processes = int(num_processes)
         self._shifts = {}  # dictionary that maps the key to the time shift; this is add to the current time step
         self.init(self.num_steps, observation_shapes, action_shapes, self.num_processes)
-        super(RolloutStorage, self).__init__()
 
     ##############
     # Properties #
@@ -502,20 +503,25 @@ class RolloutStorage(DictStorage):
         self._num_processes = int(num_processes)
 
         # allocate space for observations / states
+        logger.debug('creating space for observations of shapes: {}'.format(observation_shapes))
         if not isinstance(observation_shapes, list):
             observation_shapes = [observation_shapes]
         self.create_new_entry('observations', shapes=observation_shapes, num_steps=self.num_steps+1)
 
         # allocate space for actions
+        logger.debug('creating space for actions of shapes: {}'.format(action_shapes))
         if not isinstance(action_shapes, list):
             action_shapes = [action_shapes]
         self.create_new_entry('actions', shapes=action_shapes, num_steps=self.num_steps)
 
         # allocate space for rewards
+        logger.debug('creating space for rewards')
         self.create_new_entry('rewards', shapes=1, num_steps=self.num_steps)
 
         # allocate space for the returns and masks
+        logger.debug('creating space for returns')
         self.create_new_entry('returns', shapes=1, num_steps=self.num_steps + 1)
+        logger.debug('creating space for masks')
         self.create_new_entry('masks', shapes=1, num_steps=self.num_steps + 1)
 
         # space for log probabilities on policy, distributions, scalar values from value functions,
@@ -680,13 +686,13 @@ class RolloutStorage(DictStorage):
                             "{}".format(key, type(key)))
         super(RolloutStorage, self).__setitem__(key, value)
 
-    def __setattr__(self, key, value):
-        """Set the attribute using the given key and value. That is, instead of `D[key] = value`, you can do
-        `D.key = value`. By default, this creates a tensor with shape (num_steps + 1, self.num_processes, 1).
-
-        Warnings: avoid to use this.
-        """
-        self.create_new_entry(key, shapes=1, num_steps=self.num_steps + 1)
+    # def __setattr__(self, key, value):
+    #     """Set the attribute using the given key and value. That is, instead of `D[key] = value`, you can do
+    #     `D.key = value`. By default, this creates a tensor with shape (num_steps + 1, self.num_processes, 1).
+    #
+    #     Warnings: avoid to use this.
+    #     """
+    #     self.create_new_entry(key, shapes=1, num_steps=self.num_steps + 1)
 
 
 # Tests