update model, approximators, actorcritics

pyrobolearn/actorcritics/README.md

@@ -0,0 +1,7 @@
+## Actor-critics
+
+This folder contains actor-critic approximators.
+
+## What to look/check next?
+
+Check the `policies` and `values` folders.

pyrobolearn/actorcritics/__init__.py

@@ -0,0 +1,9 @@
+
+# import actor-critic
+from .actorcritic import *
+
+# import basic actor-critic (such as linear actor-critic)
+from .basic_actorcritic import *
+
+# import
+from .nn_actorcritic import *

pyrobolearn/actorcritics/actorcritic.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+"""Defines the various actor-critic models which combine a policy and value function.
+
+Note that actor-critic methods can share their parameters.
+
+Dependencies:
+- `pyrobolearn.policies`
+- `pyrobolearn.values`
+"""
+
+import itertools
+import torch
+
+from pyrobolearn.policies import Policy
+from pyrobolearn.values import ValueApproximator
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class ActorCritic(object):
+    r"""Actor-Critic Methods
+
+    These methods are between value-based and policy-based reinforcement learning approaches.
+    Both a policy (called the actor) and a (state) value function approximator (called the critic) are being optimized.
+    """
+
+    def __init__(self, policy, value):
+        """
+        Initialize the actor critic.
+
+        Args:
+            policy (Policy): policy approximator
+            value (Value): value function approximator
+        """
+        self.actor = policy
+        self.critic = value
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def actor(self):
+        """Return the actor."""
+        return self._actor
+
+    @actor.setter
+    def actor(self, actor):
+        """Set the actor."""
+        if not isinstance(actor, (Policy, torch.nn.Module)):
+            raise TypeError("Expecting the actor to be an instance of 'Policy' or 'torch.nn.Module', "
+                            "instead got {}".format(type(actor)))
+        self._actor = actor
+
+    @property
+    def critic(self):
+        """Return the critic."""
+        return self._critic
+
+    @critic.setter
+    def critic(self, critic):
+        """Set the critic."""
+        if not isinstance(critic, (ValueApproximator, torch.nn.Module)):
+            raise TypeError("Expecting the critic to be an instance of 'ValueApproximator' or 'torch.nn.Module', "
+                            "instead got {}".format(type(critic)))
+        self._critic = critic
+
+    @property
+    def states(self):
+        """Return the states."""
+        return self.actor.states
+
+    @property
+    def actions(self):
+        """Return the actions."""
+        return self.actor.actions
+
+    ###########
+    # Methods #
+    ###########
+
+    def parameters(self):
+        """Return the parameters of first the actor then the critic."""
+        generator = itertools.chain(self.actor.parameters(), self.critic.parameters())
+        return generator
+
+    def value(self, x):
+        """Compute the value function."""
+        return self.evaluate(x)
+
+    def action(self, x):
+        """Compute the action."""
+        return self.act(x)
+
+    def act(self, states=None, deterministic=True):
+        """Evaluate the given input states."""
+        return self.actor.act(states, deterministic=deterministic)
+
+    def evaluate(self, states=None):
+        """Evaluate the given input states."""
+        return self.critic.compute(states)
+
+    def act_and_evaluate(self, states=None):
+        """Act and evaluate the given input states."""
+        return self.act(states), self.evaluate(states)
+
+
+class SharedActorCritic(object):
+    r"""Shared Actor Critic
+
+    This class described the actor critic with shared parameters.
+    From the policy :math:`\pi_{\theta}(a_t | s_t)`
+    """
+
+    def __init__(self, states, actions, model=None, rate=1, preprocessors=None, postprocessors=None):
+        # super(SharedActorCritic, self).__init__()
+        # add a linear output node to the policy for the critic value
+        # TODO
+        pass

pyrobolearn/actorcritics/basic_actorcritic.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""Defines basic actor-critic models (such as linear models) which combine a policy and value function.
+
+Dependencies:
+- `pyrobolearn.policies`
+- `pyrobolearn.values`
+"""
+
+import itertools
+import torch
+
+from pyrobolearn.policies import LinearPolicy
+from pyrobolearn.values import LinearStateValue
+from pyrobolearn.actorcritics import ActorCritic, SharedActorCritic
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class LinearActorCritic(ActorCritic):
+    r"""Linear Actor Critic
+    """
+
+    def __init__(self, states, actions, rate=1, preprocessors=None, postprocessors=None):
+        """Initialize MLP policy.
+
+        Args:
+            states (State): 1D-states that is feed to the policy (the input dimensions will be inferred from the
+                            states)
+            actions (Action): 1D-actions outputted by the policy and will be applied in the simulator (the output
+                              dimensions will be inferred from the actions)
+            rate (int, float): rate (float) at which the policy operates if we are operating in real-time. If we are
+                stepping deterministically in the simulator, it represents the number of ticks (int) to sleep before
+                executing the model.
+            preprocessors (Processor, list of Processor, None): pre-processors to be applied to the given input
+            postprocessors (Processor, list of Processor, None): post-processors to be applied to the policy's output
+        """
+        policy = LinearPolicy(states, actions, rate=rate, preprocessors=preprocessors, postprocessors=postprocessors)
+        value = LinearStateValue(states, preprocessors=preprocessors)
+        super(LinearActorCritic, self).__init__(policy, value)
+
+
+class LinearSharedActorCritic(SharedActorCritic):
+    r"""Linear Shared Actor Critic
+    """
+    pass
+

pyrobolearn/actorcritics/nn_actorcritic.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+"""Defines the neural network actor-critic models which combine a policy and value function.
+
+Note that actor-critic methods can share their parameters.
+
+Dependencies:
+- `pyrobolearn.policies`
+- `pyrobolearn.values`
+"""
+
+import itertools
+import torch
+
+from pyrobolearn.policies import MLPPolicy
+from pyrobolearn.values import MLPStateValue
+from pyrobolearn.actorcritics import ActorCritic, SharedActorCritic
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class MLPActorCritic(ActorCritic):
+    r"""Multi-Layer Perceptron Actor Critic
+    """
+
+    def __init__(self, states, actions, hidden_units=(), activation_fct='linear', last_activation_fct=None,
+                 dropout_prob=None, rate=1, preprocessors=None, postprocessors=None):
+        """Initialize MLP policy.
+
+        Args:
+            states (State): 1D-states that is feed to the policy (the input dimensions will be inferred from the
+                            states)
+            actions (Action): 1D-actions outputted by the policy and will be applied in the simulator (the output
+                              dimensions will be inferred from the actions)
+            hidden_units (list/tuple of int): number of hidden units in the corresponding layer
+            activation_fct (None, str, or list/tuple of str/None): activation function to be applied after each layer.
+                                                                   If list/tuple, then it has to match the
+            last_activation_fct (None or str): last activation function to be applied. If not specified, it will check
+                                               if it is in the list/tuple of activation functions provided for the
+                                               previous argument.
+            dropout_prob (None, float, or list/tuple of float/None): dropout probability.
+            rate (int, float): rate (float) at which the policy operates if we are operating in real-time. If we are
+                stepping deterministically in the simulator, it represents the number of ticks (int) to sleep before
+                executing the model.
+            preprocessors (Processor, list of Processor, None): pre-processors to be applied to the given input
+            postprocessors (Processor, list of Processor, None): post-processors to be applied to the policy's output
+        """
+        policy = MLPPolicy(states, actions, hidden_units=hidden_units, activation_fct=activation_fct,
+                           last_activation_fct=last_activation_fct, dropout_prob=dropout_prob, rate=rate,
+                           preprocessors=preprocessors, postprocessors=postprocessors)
+        value = MLPStateValue(states, hidden_units=hidden_units, activation_fct=activation_fct,
+                              last_activation_fct=last_activation_fct, dropout_prob=dropout_prob,
+                              preprocessors=preprocessors)
+        super(MLPActorCritic, self).__init__(policy, value)
+
+
+class MLPSharedActorCritic(SharedActorCritic):
+    r"""Multi-Layer Perceptron Shared Actor Critic
+    """
+    pass
+

pyrobolearn/approximators/approximator.py

@@ -60,7 +60,8 @@ class Approximator(object):
         r"""Initialize the outer model.
 
         Args:
-            inputs (State, Action, np.array, torch.Tensor): inputs of the inner models (instance of State/Action)
+            inputs ((list of) State, Action, np.array, torch.Tensor): inputs of the inner models (instance of
+                State / Action)
             outputs (State, Action, np.array, torch.Tensor): outputs of the inner models (instance of Action/State)
             model (Model, None): inner model which will be wrapped if not an instance of Model
             preprocessors (None, Processor, list of Processor): the inputs are first given to the preprocessors then
@@ -98,8 +99,14 @@ class Approximator(object):
         if inputs is not None:
             if isinstance(inputs, (int, float)):
                 inputs = np.array([inputs])
+            elif isinstance(inputs, list):
+                for x in inputs:
+                    if not isinstance(x, (State, Action, torch.Tensor, np.ndarray)):
+                        raise TypeError("Expecting the given input to be an instance of `State`, `Action`, "
+                                        "`torch.Tensor`, `np.ndarray`, instead got: {}".format(type(x)))
             elif not isinstance(inputs, (State, Action, torch.Tensor, np.ndarray)):
-                raise TypeError("Expecting the inputs to be a State, Action, torch.Tensor, or np.ndarray.")
+                raise TypeError("Expecting the inputs to be a State, Action, torch.Tensor, np.ndarray, or a list "
+                                "of them.")
             if self._model is not None:
                 pass  # TODO: check that the dimensions agree with the model
         # set inputs
@@ -374,6 +381,22 @@ class Approximator(object):
             return x.numpy()
         return x
 
+    def merge_inputs(self, x=None, to_numpy=True):
+        """
+        Merge the inputs of the approximator.
+
+        Args:
+            x (None, (list of) State / Action / np.array / torch.Tensor): input data. If None, it will get the
+                data from the inputs that were given at the initialization.
+            to_numpy (bool): If True, it will convert to numpy arrays.
+
+        Returns:
+            list of np.array / torch.Tensor: input data
+        """
+        # if no input is given, take the provided inputs at the beginning
+        if x is None:
+            pass
+
     def predict(self, x=None, to_numpy=True, return_logits=False, set_output_data=True):
         """Predict the output given the input.
 

pyrobolearn/models/nn/dnn.py

@@ -87,6 +87,8 @@ class NN(object):  # Model
         self._input_shape = input_shape
         self._output_shape = output_shape
 
+        self.base_output = None
+
         # TODO: infer the framework based on the model
         self.framework = framework
 
@@ -105,6 +107,14 @@ class NN(object):  # Model
         if model is not None:
             if not (isinstance(model, torch.nn.Module)):  # or isinstance(model, keras.models.Model)):
                 raise TypeError("The model should be an instance of torch.nn.Module or keras.models.Model")
+
+            # everytime this model is called it will save the base output, that is the output of the second to last
+            # layer.
+            def hook(module, inputs, outputs):
+                self.base_output = inputs
+
+            model[-1].register_forward_hook(hook)
+
         self._model = model
 
     @property
@@ -189,6 +199,18 @@ class NN(object):  # Model
     # Methods #
     ###########
 
+    def train(self):
+        """Set into training mode."""
+        self.model.train()
+        # for param in self.model.parameters():
+        #     param.requires_grad = True
+
+    def eval(self):
+        """Set into eval mode."""
+        self.model.eval()
+        # for param in self.model.parameters():
+        #     param.requires_grad = False
+
     def parameters(self):
         """Return an iterator over the model parameters."""
         return self.model.parameters()