update algos

pyrobolearn/algos/bo.py

@@ -84,6 +84,7 @@ class BO(object):
         self.num_rollouts = 1
         self.verbose = False
         self.episode = 0
+        self.render = False
 
         self.domain = domain
 
@@ -130,7 +131,8 @@ class BO(object):
         # run a number of rollouts
         reward = []
         for rollout in range(self.num_rollouts):
-            rew = self.task.run(num_steps=self.num_steps, dt=1./240, use_terminating_condition=True, render=False)
+            rew = self.task.run(num_steps=self.num_steps, dt=self.dt, use_terminating_condition=True,
+                                render=self.render)
             reward.append(rew)
         reward = np.mean(reward)
 
@@ -143,13 +145,31 @@ class BO(object):
 
         return reward
 
-    def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None, max_time=3600):
+    def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, render=False, seed=None,
+              max_time=3600, dt=0):
+        """
+        Train the policy.
+
+        Args:
+            num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment
+                proceeds.
+            num_rollouts (int): number of rollouts per episode to average the results.
+            num_episodes (int): number of episodes.
+            verbose (bool): If True, it will print information about the training process.
+            seed (int): random seed.
+
+        Returns:
+            list of float: average rewards per episode.
+            list of float: maximum reward obtained per episode.
+        """
         # set few variables
         self.num_steps = num_steps
         self.num_rollouts = num_rollouts
         self.episode = 0
         self.verbose = verbose
         self.rewards = []
+        self.dt = dt
+        self.render = render
 
         # set seed if specified
         if seed is not None:
@@ -177,7 +197,7 @@ class BO(object):
         # print(opt.model.kernel.name)
 
         # Run the optimization
-        max_iter = num_episodes  # evaluation budget (min=4), nb_eval = 4 + max_iter
+        max_iter = num_episodes if num_episodes < 5 else num_episodes - 5  # evaluation budget (min=5)
         max_time = max_time  # time budget
         eps = 10e-6  # Minimum allows distance between the last two observations
 
@@ -205,6 +225,18 @@ class BO(object):
         return self.rewards
 
     def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True):
+        """
+        Test the policy in the environment.
+
+        Args:
+            num_steps (int): number of steps to run the episode.
+            dt (float): time to sleep before the next step.
+            use_terminating_condition (bool): If True, it will use the terminal condition to end the environment.
+            render (bool): If True, it will render the environment.
+
+        Returns:
+            float: obtained reward
+        """
         return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition,
                              render=render)
 

pyrobolearn/algos/cem.py

@@ -6,7 +6,6 @@ This CEM is an evolutionary algorithm that explores in the parameter space of th
 
 import numpy as np
 import torch
-# from pathos.multiprocessing import Pool
 
 from pyrobolearn.envs import Env
 from pyrobolearn.tasks import RLTask
@@ -79,26 +78,47 @@ class CEM(object):  # RLAlgo
         self.best_reward = -np.infty
         self.best_parameters = None
 
-    def get_vectorized_parameters(self, to_numpy=True):
-        parameters = self.policy.parameters
-        vector = torch.cat([parameter.reshape(-1) for parameter in parameters])  # np.concatenate = torch.cat
-        if to_numpy:
-            return vector.detach().numpy()
-        return vector
+    ##############
+    # Properties #
+    ##############
 
-    def set_vectorized_parameters(self, vector):
-        # convert the vector to torch array
-        if isinstance(vector, np.ndarray):
-            vector = torch.from_numpy(vector).float()
+    @property
+    def population_size(self):
+        """Return the population size."""
+        return self._population_size
 
-        # set the parameters from the vectorized one
-        idx = 0
-        for parameter in self.policy.parameters:
-            size = parameter.nelement()
-            parameter.data = vector[idx:idx+size].reshape(parameter.shape)
-            idx += size
+    @population_size.setter
+    def population_size(self, size):
+        """Set the population size."""
+        # check argument
+        if not isinstance(size, int):
+            raise TypeError("Expecting the population size to be an integer.")
+        if size < 1:
+            raise ValueError("Expecting the population size to be an integer bigger than 0.")
+
+        # set population size
+        self._population_size = size
+
+    ###########
+    # Methods #
+    ###########
 
     def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None):
+        """
+        Train the policy.
+
+        Args:
+            num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment
+                proceeds.
+            num_rollouts (int): number of rollouts per episode to average the results.
+            num_episodes (int): number of episodes.
+            verbose (bool): If True, it will print information about the training process.
+            seed (int): random seed.
+
+        Returns:
+            list of float: average rewards per episode.
+            list of float: maximum reward obtained per episode.
+        """
         # set seed
         if seed is not None:
             np.random.seed(seed)
@@ -107,7 +127,7 @@ class CEM(object):  # RLAlgo
         max_rewards, avg_rewards = [], []
 
         # init
-        theta_mean = self.get_vectorized_parameters(to_numpy=True)
+        theta_mean = self.policy.get_vectorized_parameters(to_numpy=True)
         theta_std = np.ones(len(theta_mean))
         # pool = Pool(self.num_workers)
 
@@ -125,7 +145,7 @@ class CEM(object):  # RLAlgo
             rewards = []
             for i, theta in enumerate(thetas):
                 # set policy parameters
-                self.set_vectorized_parameters(theta)
+                self.policy.set_vectorized_parameters(theta)
 
                 # run a number of rollouts
                 reward = []
@@ -171,11 +191,23 @@ class CEM(object):  # RLAlgo
             print("\nBest reward found: {}".format(self.best_reward))
 
         # set the best parameters
-        self.set_vectorized_parameters(self.best_parameters)
+        self.policy.set_vectorized_parameters(self.best_parameters)
 
         return avg_rewards, max_rewards
 
-    def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True):
+    def test(self, num_steps=1000, dt=0., use_terminating_condition=False, render=True):
+        """
+        Test the policy in the environment.
+
+        Args:
+            num_steps (int): number of steps to run the episode.
+            dt (float): time to sleep before the next step.
+            use_terminating_condition (bool): If True, it will use the terminal condition to end the environment.
+            render (bool): If True, it will render the environment.
+
+        Returns:
+            float: obtained reward
+        """
         return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition,
                              render=render)
 

pyrobolearn/algos/cmaes.py

@@ -19,6 +19,7 @@ except ImportError as e:
 from pyrobolearn.envs import Env
 from pyrobolearn.tasks import RLTask
 
+# from pyrobolearn.algos.rl_algos import *
 
 __author__ = "Brian Delhaisse"
 __copyright__ = "Copyright 2018, PyRoboLearn"
@@ -88,10 +89,12 @@ class CMAES(object):  # Algo):
 
     @property
     def population_size(self):
+        """Return the population size."""
         return self._population_size
 
     @population_size.setter
     def population_size(self, size):
+        """Set the population size."""
         # check argument
         if not isinstance(size, int):
             raise TypeError("Expecting the population size to be an integer.")
@@ -120,6 +123,21 @@ class CMAES(object):  # Algo):
         return -reward
 
     def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None):
+        """
+        Train the policy.
+
+        Args:
+            num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment
+                proceeds.
+            num_rollouts (int): number of rollouts per episode to average the results.
+            num_episodes (int): number of episodes.
+            verbose (bool): If True, it will print information about the training process.
+            seed (int): random seed.
+
+        Returns:
+            list of float: average rewards per episode.
+            list of float: maximum reward obtained per episode.
+        """
         # create CMA-ES
         self.es = cma.CMAEvolutionStrategy(self.policy.get_vectorized_parameters(), sigma0=self.sigma,
                                            inopts={'popsize': self.population_size})  # {'bounds': [-np.inf, np.inf]}
@@ -180,5 +198,17 @@ class CMAES(object):  # Algo):
         return avg_rewards, max_rewards
 
     def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True):
+        """
+        Test the policy in the environment.
+
+        Args:
+            num_steps (int): number of steps to run the episode.
+            dt (float): time to sleep before the next step.
+            use_terminating_condition (bool): If True, it will use the terminal condition to end the environment.
+            render (bool): If True, it will render the environment.
+
+        Returns:
+            float: obtained reward
+        """
         return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition,
                              render=render)

pyrobolearn/algos/fd.py

@@ -9,6 +9,7 @@ import torch
 
 from pyrobolearn.envs import Env
 from pyrobolearn.tasks import RLTask
+
 # from pyrobolearn.algos.rl_algo import GradientRLAlgo
 
 __author__ = "Brian Delhaisse"
@@ -24,7 +25,11 @@ __status__ = "Development"
 class FD(object):  # GradientRLAlgo):
     r"""Finite-Difference Policy Gradient Method.
 
-    Type: policy gradient based (on-policy be definition) with exploration in the parameter space
+    Type:: policy gradient based (on-policy by definition) with exploration in the parameter space
+
+
+    Description
+    -----------
 
     The goal of RL is to maximize the expected return:
 
@@ -37,22 +42,41 @@ class FD(object):  # GradientRLAlgo):
 
     .. math:: `g_{FD} = (\Delta\Theta^\top \Delta\Theta)^{-1} \Delta\Theta^\top \Delta J`
 
-    which is used to to perform a gradient ascent step: :math:`\theta_{i+1} = \theta_{i} + \eta g_{FD}`,
-    where :math:`\eta` is the learning rate coefficient.
+    which is used to to perform a gradient ascent step: :math:`\theta_{i+1} = \theta_{i} + \eta g_{FD}`, where
+    :math:`\eta` is the learning rate coefficient.
+
+
+    Properties
+    ----------
 
     Properties:
         * Exploration is performed in the parameter space of the policy
     Pros:
         * Easy to implement and test
-        * work with deterministic and stochastic rl
+        * work with deterministic and stochastic policies
         * highly efficient in simulation
     Cons:
         * the perturbation of the parameters is hard (especially with systems that can go unstable)
         * O(M^3) for the time complexity (because of the matrix inversion), where M is the number of parameters
 
-    Pseudo-algo:
 
-    References:
+    Pseudo-algo
+    -----------
+
+    Pseudo-algorithm (taken from [1] with some modification, and reproduce here for completeness)::
+        1. Input: initial policy parameters :math:`\theta_0`
+        2. for k=0,1,...,num_episodes do
+        3.     Exploration: generate policy variation :math:`\Delta \theta_k`, and collect set of trajectories
+                :math:`D_k=\{\tau_i\}` by running policy :math:`\pi_{\theta_k + \Delta \theta_k}` and
+                :math:`\pi_{\theta_k - \Delta \theta_k}` in the environment.
+        4.     Evaluation: compute total rewards
+                :math:`J_{k+} = \mathbb{E}_{\theta_k + \Delta \theta_k}[\sum_{t=0}^T \gamma^t r_t]`,
+                :math:`J_{k-} = \mathbb{E}_{\theta_k - \Delta \theta_k}[\sum_{t=0}^T \gamma^t r_t]`, and difference
+                gradient estimator :math:`\Delta J = J_{k+} - J_{k-}`
+        5.     Update: compute gradient :math:`g_{FD} = (\Delta \Theta ^\trsp \Delta \Theta)^{-1} \Delta\Theta
+                \Delta\hat{J}` and update policy parameters using :math:`\theta_{k+1} = \theta_k + \alpha_k g_{FD}`
+
+    References::
         [1] "Policy Gradient Methods" (http://www.scholarpedia.org/article/Policy_gradient_methods), Peters, 2010
     """
 
@@ -125,6 +149,20 @@ class FD(object):  # GradientRLAlgo):
         return reward
 
     def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None):
+        """
+        Train the policy.
+
+        Args:
+            num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment
+                proceeds.
+            num_rollouts (int): number of rollouts per episode to average the results.
+            num_episodes (int): number of episodes.
+            verbose (bool): If True, it will print information about the training process.
+            seed (int): random seed.
+
+        Returns:
+            list of float: average rewards per episode.
+        """
         # set seed
         if seed is not None:
             np.random.seed(seed)
@@ -178,11 +216,23 @@ class FD(object):  # GradientRLAlgo):
             if self.normalize_grad:
                 grad /= np.linalg.norm(grad)
             params = params + self.lr * grad  # TODO: allows the user to choose the optimizer
-            # self.optimizer.optimize(self.policy.get_params(), grad)
+            # self.optimizer.optimize(self.policy.list_parameters(), grad)
             self.policy.set_vectorized_parameters(params)
 
         return rewards
 
     def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True):
+        """
+        Test the policy in the environment.
+
+        Args:
+            num_steps (int): number of steps to run the episode.
+            dt (float): time to sleep before the next step.
+            use_terminating_condition (bool): If True, it will use the terminal condition to end the environment.
+            render (bool): If True, it will render the environment.
+
+        Returns:
+            float: obtained reward
+        """
         return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition,
                              render=render)

pyrobolearn/algos/neat_algo.py

@@ -8,17 +8,17 @@ works with neural networks and is thus tightly coupled with its associated polic
 
 import numpy as np
 
+from pyrobolearn.envs import Env
+from pyrobolearn.tasks import RLTask
+from pyrobolearn.policies import NEATPolicy
+# from pyrobolearn.algos.rl_algo import RLAlgo
+
 try:
     import neat
     # from neat import nn, population, config, statistics
 except ImportError as e:
     raise ImportError(e.__str__() + "\n HINT: you can install NEAT directly via 'pip install neat-python'.")
 
-from pyrobolearn.envs import Env
-from pyrobolearn.tasks import RLTask
-from pyrobolearn.policies import NEATPolicy
-# from pyrobolearn.algos.rl_algo import RLAlgo
-
 
 __author__ = "Brian Delhaisse"
 __copyright__ = "Copyright 2018, PyRoboLearn"

pyrobolearn/algos/power.py

@@ -5,8 +5,8 @@ This Policy learning by Weighting Exploration with the Returns (PoWER) algorithm
 Expectation-Maximization (EM) algorithm. The exploration is carried out in the parameter space.
 """
 
-# import sys
 import numpy as np
+import sys
 
 from pyrobolearn.envs import Env
 from pyrobolearn.tasks import RLTask
@@ -65,9 +65,9 @@ class PoWER(object):  # EMRLAlgo):
         Initialize the PoWER algorithm.
 
         Args:
-            task (RLTask, Env): RL task/env to run
-            policy (Policy): specify the policy (model) to optimize
-            std_params (float):
+            task (RLTask, Env): RL task/env to run.
+            policy (Policy): specify the policy (model) to optimize.
+            std_params (float): standard deviation of the parameters.
         """
         # create explorer
         # create evaluator
@@ -104,10 +104,12 @@ class PoWER(object):  # EMRLAlgo):
 
     @property
     def std_params(self):
+        """Return the standard deviation of the parameters."""
         return self._std_params
 
     @std_params.setter
     def std_params(self, std_params):
+        """Set the standard deviation of the parameters."""
         if std_params < 0.:
             std_params = 1.
         self._std_params = std_params
@@ -133,6 +135,20 @@ class PoWER(object):  # EMRLAlgo):
 
     # def train(self, num_episodes=1, num_steps=100, std_params=1., task=None, seed=None, debug=False):
     def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None):
+        """
+        Train the policy.
+
+        Args:
+            num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment
+                proceeds.
+            num_rollouts (int): number of rollouts per episode to average the results.
+            num_episodes (int): number of episodes.
+            verbose (bool): If True, it will print information about the training process.
+            seed (int): random seed.
+
+        Returns:
+            list of float: average rewards per episode.
+        """
         # check parameters
         if num_episodes < 1:
             num_episodes = 1
@@ -227,5 +243,17 @@ class PoWER(object):  # EMRLAlgo):
         return rewards
 
     def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True):
+        """
+        Test the policy in the environment.
+
+        Args:
+            num_steps (int): number of steps to run the episode.
+            dt (float): time to sleep before the next step.
+            use_terminating_condition (bool): If True, it will use the terminal condition to end the environment.
+            render (bool): If True, it will render the environment.
+
+        Returns:
+            float: obtained reward
+        """
         return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition,
                              render=render)