From 26d562e371b4c548aaa1d43e480a3e10e2f4ec24 Mon Sep 17 00:00:00 2001 From: Brian Delhaisse Date: Wed, 10 Apr 2019 04:31:55 +0200 Subject: [PATCH] update gym environment wrapper + correct approximators --- pyrobolearn/approximators/approximator.py | 8 + pyrobolearn/approximators/nn_approximator.py | 13 +- pyrobolearn/controllers/README.md | 1 + pyrobolearn/envs/env.py | 3 +- pyrobolearn/envs/gym_wrapper.py | 261 +++++++++++++++---- pyrobolearn/models/basics/linear.py | 8 + pyrobolearn/optimizers/README.md | 2 +- 7 files changed, 231 insertions(+), 65 deletions(-) diff --git a/pyrobolearn/approximators/approximator.py b/pyrobolearn/approximators/approximator.py index 5f4c1c4..4f66f66 100644 --- a/pyrobolearn/approximators/approximator.py +++ b/pyrobolearn/approximators/approximator.py @@ -329,6 +329,14 @@ class Approximator(object): size = x return size + def train(self): + """Set the inner model into training mode.""" + self.model.train() + + def eval(self): + """Set the inner model into testing mode.""" + self.model.eval() + def parameters(self): """Return an iterator over the approximator parameters.""" return self.model.parameters() diff --git a/pyrobolearn/approximators/nn_approximator.py b/pyrobolearn/approximators/nn_approximator.py index 8de908a..5dfd3d0 100644 --- a/pyrobolearn/approximators/nn_approximator.py +++ b/pyrobolearn/approximators/nn_approximator.py @@ -63,7 +63,7 @@ class MLPApproximator(NNApproximator): """ def __init__(self, inputs, outputs, hidden_units=(), - activation_fct='Linear', last_activation_fct=None, dropout_prob=None, + activation='Linear', last_activation=None, dropout=None, preprocessors=None, postprocessors=None): """ Initialize the Multi-Layer Perceptron approximator. @@ -72,9 +72,9 @@ class MLPApproximator(NNApproximator): inputs (State, Action, np.array, torch.Tensor): inputs of the inner models (instance of State/Action) outputs (State, Action, np.array, torch.Tensor): outputs of the inner models (instance of Action/State) hidden_units (tuple, list of int): number of hidden units in each layer - activation_fct (str): activation function to apply on each layer - last_activation_fct (str, None): activation function to apply on the last layer - dropout_prob (None, float): dropout probability + activation (str): activation function to apply on each layer + last_activation (str, None): activation function to apply on the last layer + dropout (None, float): dropout probability preprocessors (None, Processor, list of Processor): the inputs are first given to the preprocessors then to the model. postprocessors (None, Processor, list of Processor): the predicted outputs by the model are given to the @@ -93,9 +93,8 @@ class MLPApproximator(NNApproximator): output_size = self._size(outputs) # create model - num_units = [input_size] + list(hidden_units) + [output_size] - model = MLP(num_units=num_units, activation_fct=activation_fct, last_activation_fct=last_activation_fct, - dropout_prob=dropout_prob) + units = [input_size] + list(hidden_units) + [output_size] + model = MLP(units=units, activation=activation, last_activation=last_activation, dropout=dropout) # call superclass super(MLPApproximator, self).__init__(inputs, outputs, model, preprocessors=preprocessors, diff --git a/pyrobolearn/controllers/README.md b/pyrobolearn/controllers/README.md index 8bea6ee..2c7a35f 100644 --- a/pyrobolearn/controllers/README.md +++ b/pyrobolearn/controllers/README.md @@ -3,5 +3,6 @@ Controllers are basically policies that do not possess any (hyper-)parameters to optimize. They are manually coded by the user. Planning (TODO): +- [ ] add (kinematic and dynamic) whole body controllers - [ ] add some finite state machines - [ ] add some classical locomotion controllers (based on ZMP, optimization, etc) diff --git a/pyrobolearn/envs/env.py b/pyrobolearn/envs/env.py index 5c0bd05..a2b2016 100644 --- a/pyrobolearn/envs/env.py +++ b/pyrobolearn/envs/env.py @@ -214,7 +214,8 @@ class Env(object): # gym.Env): # Methods # ########### - def _convert_state_to_data(self, states, convert=True): + @staticmethod + def _convert_state_to_data(states, convert=True): """Convert a `State` to a list of numpy arrays or a numpy array.""" if convert: data = [] diff --git a/pyrobolearn/envs/gym_wrapper.py b/pyrobolearn/envs/gym_wrapper.py index be59002..dfc9303 100644 --- a/pyrobolearn/envs/gym_wrapper.py +++ b/pyrobolearn/envs/gym_wrapper.py @@ -11,13 +11,16 @@ import functools import numpy as np import torch import gym +import baselines from gym import * import warnings warnings.simplefilter("ignore") -from pyrobolearn.states.gym_states import GymState +from pyrobolearn.states.gym_states import GymState, State from pyrobolearn.actions.gym_actions import GymAction, Action -# from pyrobolearn.rewards import GymReward +from pyrobolearn.states.processors import StateProcessor +from pyrobolearn.rewards.processors import RewardProcessor +from pyrobolearn.rewards import GymReward # from terminating_condition import GymTerminatingCondition @@ -31,17 +34,38 @@ __email__ = "briandelhaisse@gmail.com" __status__ = "Development" -def make(env_id): - """Create the OpenAI Gym environment and return a wrapped version of it.""" - env = gym.make(env_id) - env = GymEnvWrapper(env) +def from_gym(env, state_processors=None, reward_processors=None): + """wraps a gym environment. + + Args: + env (gym.Env): gym environment. + state_processors ((a list of) StateProcessor types/classes, None): State processors. + reward_processors ((a list of) RewardProcessor types/classes, None): Reward processors. + """ + env = GymEnvWrapper(env, state_processors, reward_processors) return env -def create(env_id): - """Create the OpenAI Gym environment and return a wrapped version of it with the associated state and action.""" - env = gym.make(env_id) - env = GymEnvWrapper(env) +def make(env_id, state_processors=None, reward_processors=None): + """Create the OpenAI Gym environment and return a wrapped version of it. + + Args: + env_id (str): gym environment string. + state_processors ((a list of) StateProcessor types/classes, None): State processors. + reward_processors ((a list of) RewardProcessor types/classes, None): Reward processors. + """ + return from_gym(gym.make(env_id), state_processors, reward_processors) + + +def create(env_id, state_processors=None, reward_processors=None): + """Create the OpenAI Gym environment and return a wrapped version of it with the associated state and action. + + Args: + env_id (str): gym environment string. + state_processors ((a list of) StateProcessor types/classes, None): State processors. + reward_processors ((a list of) RewardProcessor types/classes, None): Reward processors. + """ + env = from_gym(gym.make(env_id), state_processors, reward_processors) return env, env.state, env.action @@ -51,50 +75,202 @@ class GymEnvWrapper(gym.Env): This update the data of the GymState, GymAction, and GymReward when performing a step in a gym environment. """ - def __init__(self, env, state=None, action=None): + def __init__(self, env, state_processors=None, reward_processors=None): + """ + Initialize the gym Env wrapper. + + Args: + env (gym.Env): gym environment. + state_processors ((a list of) StateProcessor types/classes): State processors. + reward_processors ((a list of) RewardProcessor types/classes): Reward processors. + """ # set environment self.env = env - # set state and action - self.state = state - self.action = action + # set state, action, and reward + self._state = GymState(self.env) + self._action = GymAction(self.env) + self._reward = GymReward(0.) + # self.done = GymTerminatingCondition(done=False) # define the observation and action space self.observation_space = self.env.observation_space self.action_space = self.env.action_space - # self.reward = GymReward(1) - # self.done = GymTerminatingCondition(done=False) + # set the processors + self.state_processors = state_processors + self.reward_processors = reward_processors + + ############## + # Properties # + ############## @property def env(self): + """Return the environment instance.""" return self._env @env.setter def env(self, environment): - if not isinstance(environment, gym.Env): + """Set the environment instance.""" + if not isinstance(environment, gym.Env): # , baselines.common.vec_env.VecEnv)): raise TypeError("Expecting the given environment to be an instance of `gym.Env`") self._env = environment + self._state = GymState(self.env) + self._action = GymAction(self.env) @property def state(self): + """Return the state instance.""" return self._state - @state.setter - def state(self, s): - if s is None: - s = GymState(self.env) - self._state = s + # @state.setter + # def state(self, s): + # """Set the state; if None, wraps the `gym.Env` to get the state.""" + # if s is None: + # s = + # elif not isinstance(s, GymState): + # raise TypeError("Expecting the given state to be None or an instance of `GymState`, instead got: " + # "{}".format(type(s))) + # self._state = s @property def action(self): + """Return the action instance.""" return self._action - @action.setter - def action(self, a): - if a is None: - a = GymAction(self.env) - self._action = a + # @action.setter + # def action(self, a): + # """Set the action; wraps the `gym.Env` to get the original action.""" + # if a is None: + # a = GymAction(self.env) + # elif not isinstance(a, GymAction): + # raise TypeError("Expecting the given action to be None or an instance of `GymAction`, instead got: " + # "{}".format(type(s))) + # self._action = a + + @property + def reward(self): + """Return the reward instance.""" + return self._reward + + @property + def state_processors(self): + """Return the processors that have to be applied on the state.""" + return self._state_processors + + @state_processors.setter + def state_processors(self, processors): + """Set the state processors.""" + if processors is None: + processors = [] + elif issubclass(processors, StateProcessor): + processors = [processors] + elif isinstance(processors, (list, tuple, set)): + for idx, processor in enumerate(processors): + if not issubclass(processor, StateProcessor): + raise TypeError("Expecting the {} item to be a subclass of `StateProcessor`, instead got: " + "{}".format(idx, type(processor))) + else: + raise TypeError("Expecting the given state processors to be None, or a (list of) subclass of " + "`StateProcessor`, instead got: {}".format(type(processors))) + + state = self.state + for processor in processors: + state = processor(state) + self._state_processors = state + + @property + def reward_processors(self): + """Return the processors that have to be applied on the reward.""" + return self._reward_processors + + @reward_processors.setter + def reward_processors(self, processors): + """Set the reward processors.""" + if processors is None: + processors = [] + elif issubclass(processors, RewardProcessor): + processors = [processors] + elif isinstance(processors, (list, tuple, set)): + for idx, processor in enumerate(processors): + if not issubclass(processor, RewardProcessor): + raise TypeError("Expecting the {} item to be a subclass of `RewardProcessor`, instead got: " + "{}".format(idx, type(processor))) + else: + raise TypeError("Expecting the given reward processors to be None, or a (list of) subclass of " + "`RewardProcessor`, instead got: {}".format(type(processors))) + + reward = self.reward + for processor in processors: + reward = processor(reward) + self._reward_processors = reward + + ########### + # Methods # + ########### + + @staticmethod + def _convert_state_to_data(states, convert=True): + """Convert a `State` to a list of numpy arrays or a numpy array.""" + if convert: + data = [] + for state in states: + if isinstance(state, State): + state = state.merged_data + if isinstance(state, list) and len(state) == 1: + state = state[0] + data.append(state) + if len(data) == 1: + data = data[0] + return data + return states + + def reset(self): + observation = self.env.reset() + self.state.data = observation + self.reward_processors.reset() + return self._convert_state_to_data(self.state) + + def step(self, actions): + """perform a step in the environment and set the data for the GymState and GymAction""" + if isinstance(actions, Action): # convert from `Action` to numpy + actions = actions.merged_data[0] + elif isinstance(actions, torch.Tensor): # convert from torch to numpy + if actions.requires_grad: + actions = actions.detach().numpy() + else: + actions = actions.numpy() + if isinstance(self.action_space, gym.spaces.Discrete) and isinstance(actions, np.ndarray): + actions = actions[0] # float + + # perform a step with the original gym environment + observations, reward, done, info = self.env.step(actions) + + # set the data + self.state.data = observations + self.action.data = actions + self.reward.value = reward + # self.done.done = done + + # process the data + observations = self.state_processors() + reward = self.reward_processors() + + observations = self._convert_state_to_data(observations) + + # return the data + return observations, reward, done, info + + def render(self, mode='human'): + self.env.render(mode) + + def hide(self): + pass + + ############# + # Operators # + ############# def __getattr__(self, name): """Get the functions from the Gym Environment""" @@ -103,39 +279,12 @@ class GymEnvWrapper(gym.Env): attribute = functools.partial(attribute) return attribute - def step(self, actions): - """perform a step in the environment and set the data for the GymState and GymAction""" - if isinstance(actions, Action): - actions = actions.data[0] - elif isinstance(actions, torch.Tensor): - if actions.requires_grad: - actions = actions.detach().numpy() - else: - actions = actions.numpy() - if isinstance(self.action_space, gym.spaces.Discrete) and isinstance(actions, np.ndarray): - actions = actions[0] - observations, reward, done, info = self.env.step(actions) - self.state.data = observations - self.action.data = actions - # self.reward.value = reward - # self.done.done = done - return self.state, reward, done, info - - def reset(self): - observation = self.env.reset() - self.state.data = observation - return self.state - - def render(self, mode='human'): - self.env.render(mode) - - def hide(self): - pass - def __repr__(self): + """Return a representing object.""" return self.env.__repr__() def __str__(self): + """Return a string describing the class.""" return self.env.__str__() diff --git a/pyrobolearn/models/basics/linear.py b/pyrobolearn/models/basics/linear.py index a9d7f77..b5459e3 100644 --- a/pyrobolearn/models/basics/linear.py +++ b/pyrobolearn/models/basics/linear.py @@ -150,6 +150,14 @@ class Linear(object): # Methods # ########### + def train(self): + """Set into training mode.""" + self.model.train() + + def eval(self): + """Set into eval mode.""" + self.model.eval() + def copy_parameters(self, parameters): """Copy the given parameters. diff --git a/pyrobolearn/optimizers/README.md b/pyrobolearn/optimizers/README.md index 1a4db0b..c2b5e06 100644 --- a/pyrobolearn/optimizers/README.md +++ b/pyrobolearn/optimizers/README.md @@ -1,4 +1,4 @@ -## Optimizers +## Optimizers / Solvers This folder provides the various optimizers. It mostly wraps already existing optimizers but provides a common API.