From 84cf074737c05ea93a835939ecc5cc91c67ecb6a Mon Sep 17 00:00:00 2001 From: Brian Delhaisse Date: Fri, 29 Mar 2019 16:26:57 +0100 Subject: [PATCH] update model, approximators, actorcritics --- pyrobolearn/actorcritics/README.md | 7 + pyrobolearn/actorcritics/__init__.py | 9 ++ pyrobolearn/actorcritics/actorcritic.py | 126 ++++++++++++++++++ pyrobolearn/actorcritics/basic_actorcritic.py | 53 ++++++++ pyrobolearn/actorcritics/nn_actorcritic.py | 67 ++++++++++ pyrobolearn/approximators/approximator.py | 27 +++- pyrobolearn/models/nn/dnn.py | 22 +++ 7 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 pyrobolearn/actorcritics/README.md create mode 100644 pyrobolearn/actorcritics/__init__.py create mode 100644 pyrobolearn/actorcritics/actorcritic.py create mode 100644 pyrobolearn/actorcritics/basic_actorcritic.py create mode 100644 pyrobolearn/actorcritics/nn_actorcritic.py diff --git a/pyrobolearn/actorcritics/README.md b/pyrobolearn/actorcritics/README.md new file mode 100644 index 0000000..ac34b03 --- /dev/null +++ b/pyrobolearn/actorcritics/README.md @@ -0,0 +1,7 @@ +## Actor-critics + +This folder contains actor-critic approximators. + +## What to look/check next? + +Check the `policies` and `values` folders. diff --git a/pyrobolearn/actorcritics/__init__.py b/pyrobolearn/actorcritics/__init__.py new file mode 100644 index 0000000..1b04667 --- /dev/null +++ b/pyrobolearn/actorcritics/__init__.py @@ -0,0 +1,9 @@ + +# import actor-critic +from .actorcritic import * + +# import basic actor-critic (such as linear actor-critic) +from .basic_actorcritic import * + +# import +from .nn_actorcritic import * diff --git a/pyrobolearn/actorcritics/actorcritic.py b/pyrobolearn/actorcritics/actorcritic.py new file mode 100644 index 0000000..34a9426 --- /dev/null +++ b/pyrobolearn/actorcritics/actorcritic.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +"""Defines the various actor-critic models which combine a policy and value function. + +Note that actor-critic methods can share their parameters. + +Dependencies: +- `pyrobolearn.policies` +- `pyrobolearn.values` +""" + +import itertools +import torch + +from pyrobolearn.policies import Policy +from pyrobolearn.values import ValueApproximator + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class ActorCritic(object): + r"""Actor-Critic Methods + + These methods are between value-based and policy-based reinforcement learning approaches. + Both a policy (called the actor) and a (state) value function approximator (called the critic) are being optimized. + """ + + def __init__(self, policy, value): + """ + Initialize the actor critic. + + Args: + policy (Policy): policy approximator + value (Value): value function approximator + """ + self.actor = policy + self.critic = value + + ############## + # Properties # + ############## + + @property + def actor(self): + """Return the actor.""" + return self._actor + + @actor.setter + def actor(self, actor): + """Set the actor.""" + if not isinstance(actor, (Policy, torch.nn.Module)): + raise TypeError("Expecting the actor to be an instance of 'Policy' or 'torch.nn.Module', " + "instead got {}".format(type(actor))) + self._actor = actor + + @property + def critic(self): + """Return the critic.""" + return self._critic + + @critic.setter + def critic(self, critic): + """Set the critic.""" + if not isinstance(critic, (ValueApproximator, torch.nn.Module)): + raise TypeError("Expecting the critic to be an instance of 'ValueApproximator' or 'torch.nn.Module', " + "instead got {}".format(type(critic))) + self._critic = critic + + @property + def states(self): + """Return the states.""" + return self.actor.states + + @property + def actions(self): + """Return the actions.""" + return self.actor.actions + + ########### + # Methods # + ########### + + def parameters(self): + """Return the parameters of first the actor then the critic.""" + generator = itertools.chain(self.actor.parameters(), self.critic.parameters()) + return generator + + def value(self, x): + """Compute the value function.""" + return self.evaluate(x) + + def action(self, x): + """Compute the action.""" + return self.act(x) + + def act(self, states=None, deterministic=True): + """Evaluate the given input states.""" + return self.actor.act(states, deterministic=deterministic) + + def evaluate(self, states=None): + """Evaluate the given input states.""" + return self.critic.compute(states) + + def act_and_evaluate(self, states=None): + """Act and evaluate the given input states.""" + return self.act(states), self.evaluate(states) + + +class SharedActorCritic(object): + r"""Shared Actor Critic + + This class described the actor critic with shared parameters. + From the policy :math:`\pi_{\theta}(a_t | s_t)` + """ + + def __init__(self, states, actions, model=None, rate=1, preprocessors=None, postprocessors=None): + # super(SharedActorCritic, self).__init__() + # add a linear output node to the policy for the critic value + # TODO + pass diff --git a/pyrobolearn/actorcritics/basic_actorcritic.py b/pyrobolearn/actorcritics/basic_actorcritic.py new file mode 100644 index 0000000..4edd55a --- /dev/null +++ b/pyrobolearn/actorcritics/basic_actorcritic.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +"""Defines basic actor-critic models (such as linear models) which combine a policy and value function. + +Dependencies: +- `pyrobolearn.policies` +- `pyrobolearn.values` +""" + +import itertools +import torch + +from pyrobolearn.policies import LinearPolicy +from pyrobolearn.values import LinearStateValue +from pyrobolearn.actorcritics import ActorCritic, SharedActorCritic + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class LinearActorCritic(ActorCritic): + r"""Linear Actor Critic + """ + + def __init__(self, states, actions, rate=1, preprocessors=None, postprocessors=None): + """Initialize MLP policy. + + Args: + states (State): 1D-states that is feed to the policy (the input dimensions will be inferred from the + states) + actions (Action): 1D-actions outputted by the policy and will be applied in the simulator (the output + dimensions will be inferred from the actions) + rate (int, float): rate (float) at which the policy operates if we are operating in real-time. If we are + stepping deterministically in the simulator, it represents the number of ticks (int) to sleep before + executing the model. + preprocessors (Processor, list of Processor, None): pre-processors to be applied to the given input + postprocessors (Processor, list of Processor, None): post-processors to be applied to the policy's output + """ + policy = LinearPolicy(states, actions, rate=rate, preprocessors=preprocessors, postprocessors=postprocessors) + value = LinearStateValue(states, preprocessors=preprocessors) + super(LinearActorCritic, self).__init__(policy, value) + + +class LinearSharedActorCritic(SharedActorCritic): + r"""Linear Shared Actor Critic + """ + pass + diff --git a/pyrobolearn/actorcritics/nn_actorcritic.py b/pyrobolearn/actorcritics/nn_actorcritic.py new file mode 100644 index 0000000..c92f9c0 --- /dev/null +++ b/pyrobolearn/actorcritics/nn_actorcritic.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +"""Defines the neural network actor-critic models which combine a policy and value function. + +Note that actor-critic methods can share their parameters. + +Dependencies: +- `pyrobolearn.policies` +- `pyrobolearn.values` +""" + +import itertools +import torch + +from pyrobolearn.policies import MLPPolicy +from pyrobolearn.values import MLPStateValue +from pyrobolearn.actorcritics import ActorCritic, SharedActorCritic + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "MIT" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class MLPActorCritic(ActorCritic): + r"""Multi-Layer Perceptron Actor Critic + """ + + def __init__(self, states, actions, hidden_units=(), activation_fct='linear', last_activation_fct=None, + dropout_prob=None, rate=1, preprocessors=None, postprocessors=None): + """Initialize MLP policy. + + Args: + states (State): 1D-states that is feed to the policy (the input dimensions will be inferred from the + states) + actions (Action): 1D-actions outputted by the policy and will be applied in the simulator (the output + dimensions will be inferred from the actions) + hidden_units (list/tuple of int): number of hidden units in the corresponding layer + activation_fct (None, str, or list/tuple of str/None): activation function to be applied after each layer. + If list/tuple, then it has to match the + last_activation_fct (None or str): last activation function to be applied. If not specified, it will check + if it is in the list/tuple of activation functions provided for the + previous argument. + dropout_prob (None, float, or list/tuple of float/None): dropout probability. + rate (int, float): rate (float) at which the policy operates if we are operating in real-time. If we are + stepping deterministically in the simulator, it represents the number of ticks (int) to sleep before + executing the model. + preprocessors (Processor, list of Processor, None): pre-processors to be applied to the given input + postprocessors (Processor, list of Processor, None): post-processors to be applied to the policy's output + """ + policy = MLPPolicy(states, actions, hidden_units=hidden_units, activation_fct=activation_fct, + last_activation_fct=last_activation_fct, dropout_prob=dropout_prob, rate=rate, + preprocessors=preprocessors, postprocessors=postprocessors) + value = MLPStateValue(states, hidden_units=hidden_units, activation_fct=activation_fct, + last_activation_fct=last_activation_fct, dropout_prob=dropout_prob, + preprocessors=preprocessors) + super(MLPActorCritic, self).__init__(policy, value) + + +class MLPSharedActorCritic(SharedActorCritic): + r"""Multi-Layer Perceptron Shared Actor Critic + """ + pass + diff --git a/pyrobolearn/approximators/approximator.py b/pyrobolearn/approximators/approximator.py index cf14f3d..5f4c1c4 100644 --- a/pyrobolearn/approximators/approximator.py +++ b/pyrobolearn/approximators/approximator.py @@ -60,7 +60,8 @@ class Approximator(object): r"""Initialize the outer model. Args: - inputs (State, Action, np.array, torch.Tensor): inputs of the inner models (instance of State/Action) + inputs ((list of) State, Action, np.array, torch.Tensor): inputs of the inner models (instance of + State / Action) outputs (State, Action, np.array, torch.Tensor): outputs of the inner models (instance of Action/State) model (Model, None): inner model which will be wrapped if not an instance of Model preprocessors (None, Processor, list of Processor): the inputs are first given to the preprocessors then @@ -98,8 +99,14 @@ class Approximator(object): if inputs is not None: if isinstance(inputs, (int, float)): inputs = np.array([inputs]) + elif isinstance(inputs, list): + for x in inputs: + if not isinstance(x, (State, Action, torch.Tensor, np.ndarray)): + raise TypeError("Expecting the given input to be an instance of `State`, `Action`, " + "`torch.Tensor`, `np.ndarray`, instead got: {}".format(type(x))) elif not isinstance(inputs, (State, Action, torch.Tensor, np.ndarray)): - raise TypeError("Expecting the inputs to be a State, Action, torch.Tensor, or np.ndarray.") + raise TypeError("Expecting the inputs to be a State, Action, torch.Tensor, np.ndarray, or a list " + "of them.") if self._model is not None: pass # TODO: check that the dimensions agree with the model # set inputs @@ -374,6 +381,22 @@ class Approximator(object): return x.numpy() return x + def merge_inputs(self, x=None, to_numpy=True): + """ + Merge the inputs of the approximator. + + Args: + x (None, (list of) State / Action / np.array / torch.Tensor): input data. If None, it will get the + data from the inputs that were given at the initialization. + to_numpy (bool): If True, it will convert to numpy arrays. + + Returns: + list of np.array / torch.Tensor: input data + """ + # if no input is given, take the provided inputs at the beginning + if x is None: + pass + def predict(self, x=None, to_numpy=True, return_logits=False, set_output_data=True): """Predict the output given the input. diff --git a/pyrobolearn/models/nn/dnn.py b/pyrobolearn/models/nn/dnn.py index ff0ba06..bb6b993 100644 --- a/pyrobolearn/models/nn/dnn.py +++ b/pyrobolearn/models/nn/dnn.py @@ -87,6 +87,8 @@ class NN(object): # Model self._input_shape = input_shape self._output_shape = output_shape + self.base_output = None + # TODO: infer the framework based on the model self.framework = framework @@ -105,6 +107,14 @@ class NN(object): # Model if model is not None: if not (isinstance(model, torch.nn.Module)): # or isinstance(model, keras.models.Model)): raise TypeError("The model should be an instance of torch.nn.Module or keras.models.Model") + + # everytime this model is called it will save the base output, that is the output of the second to last + # layer. + def hook(module, inputs, outputs): + self.base_output = inputs + + model[-1].register_forward_hook(hook) + self._model = model @property @@ -189,6 +199,18 @@ class NN(object): # Model # Methods # ########### + def train(self): + """Set into training mode.""" + self.model.train() + # for param in self.model.parameters(): + # param.requires_grad = True + + def eval(self): + """Set into eval mode.""" + self.model.eval() + # for param in self.model.parameters(): + # param.requires_grad = False + def parameters(self): """Return an iterator over the model parameters.""" return self.model.parameters()