From 31cdc46c524b03498b15fb3d69854c7fb4109b2f Mon Sep 17 00:00:00 2001 From: Brian Delhaisse Date: Tue, 26 Mar 2019 19:55:26 +0100 Subject: [PATCH] update algos --- pyrobolearn/algos/bo.py | 38 +++++++++++++++-- pyrobolearn/algos/cem.py | 74 ++++++++++++++++++++++++---------- pyrobolearn/algos/cmaes.py | 30 ++++++++++++++ pyrobolearn/algos/fd.py | 64 +++++++++++++++++++++++++---- pyrobolearn/algos/neat_algo.py | 10 ++--- pyrobolearn/algos/power.py | 36 +++++++++++++++-- 6 files changed, 212 insertions(+), 40 deletions(-) diff --git a/pyrobolearn/algos/bo.py b/pyrobolearn/algos/bo.py index 2e01faa..3ce1118 100644 --- a/pyrobolearn/algos/bo.py +++ b/pyrobolearn/algos/bo.py @@ -84,6 +84,7 @@ class BO(object): self.num_rollouts = 1 self.verbose = False self.episode = 0 + self.render = False self.domain = domain @@ -130,7 +131,8 @@ class BO(object): # run a number of rollouts reward = [] for rollout in range(self.num_rollouts): - rew = self.task.run(num_steps=self.num_steps, dt=1./240, use_terminating_condition=True, render=False) + rew = self.task.run(num_steps=self.num_steps, dt=self.dt, use_terminating_condition=True, + render=self.render) reward.append(rew) reward = np.mean(reward) @@ -143,13 +145,31 @@ class BO(object): return reward - def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None, max_time=3600): + def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, render=False, seed=None, + max_time=3600, dt=0): + """ + Train the policy. + + Args: + num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment + proceeds. + num_rollouts (int): number of rollouts per episode to average the results. + num_episodes (int): number of episodes. + verbose (bool): If True, it will print information about the training process. + seed (int): random seed. + + Returns: + list of float: average rewards per episode. + list of float: maximum reward obtained per episode. + """ # set few variables self.num_steps = num_steps self.num_rollouts = num_rollouts self.episode = 0 self.verbose = verbose self.rewards = [] + self.dt = dt + self.render = render # set seed if specified if seed is not None: @@ -177,7 +197,7 @@ class BO(object): # print(opt.model.kernel.name) # Run the optimization - max_iter = num_episodes # evaluation budget (min=4), nb_eval = 4 + max_iter + max_iter = num_episodes if num_episodes < 5 else num_episodes - 5 # evaluation budget (min=5) max_time = max_time # time budget eps = 10e-6 # Minimum allows distance between the last two observations @@ -205,6 +225,18 @@ class BO(object): return self.rewards def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True): + """ + Test the policy in the environment. + + Args: + num_steps (int): number of steps to run the episode. + dt (float): time to sleep before the next step. + use_terminating_condition (bool): If True, it will use the terminal condition to end the environment. + render (bool): If True, it will render the environment. + + Returns: + float: obtained reward + """ return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition, render=render) diff --git a/pyrobolearn/algos/cem.py b/pyrobolearn/algos/cem.py index 8648f47..c45e608 100755 --- a/pyrobolearn/algos/cem.py +++ b/pyrobolearn/algos/cem.py @@ -6,7 +6,6 @@ This CEM is an evolutionary algorithm that explores in the parameter space of th import numpy as np import torch -# from pathos.multiprocessing import Pool from pyrobolearn.envs import Env from pyrobolearn.tasks import RLTask @@ -79,26 +78,47 @@ class CEM(object): # RLAlgo self.best_reward = -np.infty self.best_parameters = None - def get_vectorized_parameters(self, to_numpy=True): - parameters = self.policy.parameters - vector = torch.cat([parameter.reshape(-1) for parameter in parameters]) # np.concatenate = torch.cat - if to_numpy: - return vector.detach().numpy() - return vector + ############## + # Properties # + ############## - def set_vectorized_parameters(self, vector): - # convert the vector to torch array - if isinstance(vector, np.ndarray): - vector = torch.from_numpy(vector).float() + @property + def population_size(self): + """Return the population size.""" + return self._population_size - # set the parameters from the vectorized one - idx = 0 - for parameter in self.policy.parameters: - size = parameter.nelement() - parameter.data = vector[idx:idx+size].reshape(parameter.shape) - idx += size + @population_size.setter + def population_size(self, size): + """Set the population size.""" + # check argument + if not isinstance(size, int): + raise TypeError("Expecting the population size to be an integer.") + if size < 1: + raise ValueError("Expecting the population size to be an integer bigger than 0.") + + # set population size + self._population_size = size + + ########### + # Methods # + ########### def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None): + """ + Train the policy. + + Args: + num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment + proceeds. + num_rollouts (int): number of rollouts per episode to average the results. + num_episodes (int): number of episodes. + verbose (bool): If True, it will print information about the training process. + seed (int): random seed. + + Returns: + list of float: average rewards per episode. + list of float: maximum reward obtained per episode. + """ # set seed if seed is not None: np.random.seed(seed) @@ -107,7 +127,7 @@ class CEM(object): # RLAlgo max_rewards, avg_rewards = [], [] # init - theta_mean = self.get_vectorized_parameters(to_numpy=True) + theta_mean = self.policy.get_vectorized_parameters(to_numpy=True) theta_std = np.ones(len(theta_mean)) # pool = Pool(self.num_workers) @@ -125,7 +145,7 @@ class CEM(object): # RLAlgo rewards = [] for i, theta in enumerate(thetas): # set policy parameters - self.set_vectorized_parameters(theta) + self.policy.set_vectorized_parameters(theta) # run a number of rollouts reward = [] @@ -171,11 +191,23 @@ class CEM(object): # RLAlgo print("\nBest reward found: {}".format(self.best_reward)) # set the best parameters - self.set_vectorized_parameters(self.best_parameters) + self.policy.set_vectorized_parameters(self.best_parameters) return avg_rewards, max_rewards - def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True): + def test(self, num_steps=1000, dt=0., use_terminating_condition=False, render=True): + """ + Test the policy in the environment. + + Args: + num_steps (int): number of steps to run the episode. + dt (float): time to sleep before the next step. + use_terminating_condition (bool): If True, it will use the terminal condition to end the environment. + render (bool): If True, it will render the environment. + + Returns: + float: obtained reward + """ return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition, render=render) diff --git a/pyrobolearn/algos/cmaes.py b/pyrobolearn/algos/cmaes.py index ca14cd0..3293efd 100755 --- a/pyrobolearn/algos/cmaes.py +++ b/pyrobolearn/algos/cmaes.py @@ -19,6 +19,7 @@ except ImportError as e: from pyrobolearn.envs import Env from pyrobolearn.tasks import RLTask +# from pyrobolearn.algos.rl_algos import * __author__ = "Brian Delhaisse" __copyright__ = "Copyright 2018, PyRoboLearn" @@ -88,10 +89,12 @@ class CMAES(object): # Algo): @property def population_size(self): + """Return the population size.""" return self._population_size @population_size.setter def population_size(self, size): + """Set the population size.""" # check argument if not isinstance(size, int): raise TypeError("Expecting the population size to be an integer.") @@ -120,6 +123,21 @@ class CMAES(object): # Algo): return -reward def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None): + """ + Train the policy. + + Args: + num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment + proceeds. + num_rollouts (int): number of rollouts per episode to average the results. + num_episodes (int): number of episodes. + verbose (bool): If True, it will print information about the training process. + seed (int): random seed. + + Returns: + list of float: average rewards per episode. + list of float: maximum reward obtained per episode. + """ # create CMA-ES self.es = cma.CMAEvolutionStrategy(self.policy.get_vectorized_parameters(), sigma0=self.sigma, inopts={'popsize': self.population_size}) # {'bounds': [-np.inf, np.inf]} @@ -180,5 +198,17 @@ class CMAES(object): # Algo): return avg_rewards, max_rewards def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True): + """ + Test the policy in the environment. + + Args: + num_steps (int): number of steps to run the episode. + dt (float): time to sleep before the next step. + use_terminating_condition (bool): If True, it will use the terminal condition to end the environment. + render (bool): If True, it will render the environment. + + Returns: + float: obtained reward + """ return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition, render=render) diff --git a/pyrobolearn/algos/fd.py b/pyrobolearn/algos/fd.py index 9c0e67d..da7a22e 100755 --- a/pyrobolearn/algos/fd.py +++ b/pyrobolearn/algos/fd.py @@ -9,6 +9,7 @@ import torch from pyrobolearn.envs import Env from pyrobolearn.tasks import RLTask + # from pyrobolearn.algos.rl_algo import GradientRLAlgo __author__ = "Brian Delhaisse" @@ -24,7 +25,11 @@ __status__ = "Development" class FD(object): # GradientRLAlgo): r"""Finite-Difference Policy Gradient Method. - Type: policy gradient based (on-policy be definition) with exploration in the parameter space + Type:: policy gradient based (on-policy by definition) with exploration in the parameter space + + + Description + ----------- The goal of RL is to maximize the expected return: @@ -37,22 +42,41 @@ class FD(object): # GradientRLAlgo): .. math:: `g_{FD} = (\Delta\Theta^\top \Delta\Theta)^{-1} \Delta\Theta^\top \Delta J` - which is used to to perform a gradient ascent step: :math:`\theta_{i+1} = \theta_{i} + \eta g_{FD}`, - where :math:`\eta` is the learning rate coefficient. + which is used to to perform a gradient ascent step: :math:`\theta_{i+1} = \theta_{i} + \eta g_{FD}`, where + :math:`\eta` is the learning rate coefficient. + + + Properties + ---------- Properties: * Exploration is performed in the parameter space of the policy Pros: * Easy to implement and test - * work with deterministic and stochastic rl + * work with deterministic and stochastic policies * highly efficient in simulation Cons: * the perturbation of the parameters is hard (especially with systems that can go unstable) * O(M^3) for the time complexity (because of the matrix inversion), where M is the number of parameters - Pseudo-algo: - References: + Pseudo-algo + ----------- + + Pseudo-algorithm (taken from [1] with some modification, and reproduce here for completeness):: + 1. Input: initial policy parameters :math:`\theta_0` + 2. for k=0,1,...,num_episodes do + 3. Exploration: generate policy variation :math:`\Delta \theta_k`, and collect set of trajectories + :math:`D_k=\{\tau_i\}` by running policy :math:`\pi_{\theta_k + \Delta \theta_k}` and + :math:`\pi_{\theta_k - \Delta \theta_k}` in the environment. + 4. Evaluation: compute total rewards + :math:`J_{k+} = \mathbb{E}_{\theta_k + \Delta \theta_k}[\sum_{t=0}^T \gamma^t r_t]`, + :math:`J_{k-} = \mathbb{E}_{\theta_k - \Delta \theta_k}[\sum_{t=0}^T \gamma^t r_t]`, and difference + gradient estimator :math:`\Delta J = J_{k+} - J_{k-}` + 5. Update: compute gradient :math:`g_{FD} = (\Delta \Theta ^\trsp \Delta \Theta)^{-1} \Delta\Theta + \Delta\hat{J}` and update policy parameters using :math:`\theta_{k+1} = \theta_k + \alpha_k g_{FD}` + + References:: [1] "Policy Gradient Methods" (http://www.scholarpedia.org/article/Policy_gradient_methods), Peters, 2010 """ @@ -125,6 +149,20 @@ class FD(object): # GradientRLAlgo): return reward def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None): + """ + Train the policy. + + Args: + num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment + proceeds. + num_rollouts (int): number of rollouts per episode to average the results. + num_episodes (int): number of episodes. + verbose (bool): If True, it will print information about the training process. + seed (int): random seed. + + Returns: + list of float: average rewards per episode. + """ # set seed if seed is not None: np.random.seed(seed) @@ -178,11 +216,23 @@ class FD(object): # GradientRLAlgo): if self.normalize_grad: grad /= np.linalg.norm(grad) params = params + self.lr * grad # TODO: allows the user to choose the optimizer - # self.optimizer.optimize(self.policy.get_params(), grad) + # self.optimizer.optimize(self.policy.list_parameters(), grad) self.policy.set_vectorized_parameters(params) return rewards def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True): + """ + Test the policy in the environment. + + Args: + num_steps (int): number of steps to run the episode. + dt (float): time to sleep before the next step. + use_terminating_condition (bool): If True, it will use the terminal condition to end the environment. + render (bool): If True, it will render the environment. + + Returns: + float: obtained reward + """ return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition, render=render) diff --git a/pyrobolearn/algos/neat_algo.py b/pyrobolearn/algos/neat_algo.py index d4aaec7..7342ee8 100755 --- a/pyrobolearn/algos/neat_algo.py +++ b/pyrobolearn/algos/neat_algo.py @@ -8,17 +8,17 @@ works with neural networks and is thus tightly coupled with its associated polic import numpy as np +from pyrobolearn.envs import Env +from pyrobolearn.tasks import RLTask +from pyrobolearn.policies import NEATPolicy +# from pyrobolearn.algos.rl_algo import RLAlgo + try: import neat # from neat import nn, population, config, statistics except ImportError as e: raise ImportError(e.__str__() + "\n HINT: you can install NEAT directly via 'pip install neat-python'.") -from pyrobolearn.envs import Env -from pyrobolearn.tasks import RLTask -from pyrobolearn.policies import NEATPolicy -# from pyrobolearn.algos.rl_algo import RLAlgo - __author__ = "Brian Delhaisse" __copyright__ = "Copyright 2018, PyRoboLearn" diff --git a/pyrobolearn/algos/power.py b/pyrobolearn/algos/power.py index 71f543f..678a5a3 100755 --- a/pyrobolearn/algos/power.py +++ b/pyrobolearn/algos/power.py @@ -5,8 +5,8 @@ This Policy learning by Weighting Exploration with the Returns (PoWER) algorithm Expectation-Maximization (EM) algorithm. The exploration is carried out in the parameter space. """ -# import sys import numpy as np +import sys from pyrobolearn.envs import Env from pyrobolearn.tasks import RLTask @@ -65,9 +65,9 @@ class PoWER(object): # EMRLAlgo): Initialize the PoWER algorithm. Args: - task (RLTask, Env): RL task/env to run - policy (Policy): specify the policy (model) to optimize - std_params (float): + task (RLTask, Env): RL task/env to run. + policy (Policy): specify the policy (model) to optimize. + std_params (float): standard deviation of the parameters. """ # create explorer # create evaluator @@ -104,10 +104,12 @@ class PoWER(object): # EMRLAlgo): @property def std_params(self): + """Return the standard deviation of the parameters.""" return self._std_params @std_params.setter def std_params(self, std_params): + """Set the standard deviation of the parameters.""" if std_params < 0.: std_params = 1. self._std_params = std_params @@ -133,6 +135,20 @@ class PoWER(object): # EMRLAlgo): # def train(self, num_episodes=1, num_steps=100, std_params=1., task=None, seed=None, debug=False): def train(self, num_steps=1000, num_rollouts=1, num_episodes=1, verbose=False, seed=None): + """ + Train the policy. + + Args: + num_steps (int): number of steps per rollout / episode. In one episode, how many steps does the environment + proceeds. + num_rollouts (int): number of rollouts per episode to average the results. + num_episodes (int): number of episodes. + verbose (bool): If True, it will print information about the training process. + seed (int): random seed. + + Returns: + list of float: average rewards per episode. + """ # check parameters if num_episodes < 1: num_episodes = 1 @@ -227,5 +243,17 @@ class PoWER(object): # EMRLAlgo): return rewards def test(self, num_steps=1000, dt=0, use_terminating_condition=False, render=True): + """ + Test the policy in the environment. + + Args: + num_steps (int): number of steps to run the episode. + dt (float): time to sleep before the next step. + use_terminating_condition (bool): If True, it will use the terminal condition to end the environment. + render (bool): If True, it will render the environment. + + Returns: + float: obtained reward + """ return self.task.run(num_steps=num_steps, dt=dt, use_terminating_condition=use_terminating_condition, render=render)