add approximators, policies, processors, recorders

pyrobolearn/approximators/__init__.py

@@ -0,0 +1,3 @@
+
+# import function approximators
+from approximator import *

pyrobolearn/approximators/approximator.py

@@ -0,0 +1,558 @@
+#!/usr/bin/env python
+"""Define the basic (Function) Approximator class.
+
+This file describes the `Approximator` class that wraps a learning model and connects it with its inputs, and outputs.
+The inputs/outputs can be states, actions, arrays/tensors, etc.
+
+Dependencies:
+- `pyrobolearn.models`
+- `pyrobolearn.states`
+- `pyrobolearn.actions`
+"""
+
+from abc import ABCMeta, abstractmethod
+import numpy as np
+import torch
+
+from pyrobolearn.states import State
+from pyrobolearn.actions import Action
+
+from pyrobolearn.models import Model
+from pyrobolearn.models.linear import Linear
+from pyrobolearn.models.nn import NN, MLP, NEATModel
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class Approximator(object):
+    r"""Function Approximator (abstract) class
+
+    The function approximator is a wrapper around the inner learning base model, and connects a learning model to
+    its state/action inputs and outputs. That is, this class described how to connect a learning model with
+    states/actions, and thus allows the inner models to be independent from the notions of states and actions.
+
+    This class and all the children inheriting from this one will be used internally by several classes such as
+    policies, dynamic models, value estimators, and others. This enables to not duplicate and write the same code
+    for these various different concepts which share similar features.
+    For instance, policies are function approximators where the inputs are states, and the outputs are actions.
+    Dynamic models are extended models where the inputs are states and actions, and the outputs are the next state.
+
+    Often the learning model can be constructed automatically by knowing the input and output dimensions, along with
+    few other optional parameters. This is useful if one does not wish to change the learning model but just to scale
+    it to different input/output sizes.
+
+    This class is used by the following classes:
+    * Policy: approximator mapping states to actions
+    * Value estimator: approximator mapping states (or states and actions) to a value, or mapping states to actions
+                       (in the case of discrete actions)
+    * Actor-Critic:
+    * Dynamic: approximator mapping states and actions to the next states
+    * Transformation mappings: approximator mapping states to states
+
+    Example::
+
+        states = JntPosState(robot) + JntVelState(robot)
+        actions = JntPosAction(robot)
+        model = ...
+    """
+
+    def __init__(self, inputs, outputs, model=None, preprocessors=None, postprocessors=None):
+        r"""Initialize the outer model.
+
+        Args:
+            inputs (State, Action, array): inputs of the inner models (instance of State/Action)
+            outputs (State, Action, array): outputs of the inner models (instance of Action/State)
+            model (Model, None): inner model which will be wrapped if not an instance of Model
+            preprocessors (None, Processor): the inputs are first given to the preprocessors then to the model.
+            postprocessors (None, Processor): the predicted outputs by the model are given to the processors before
+                                              being returned.
+        """
+
+        # Check inputs and outputs, and convert to the correct format
+        self._model = None
+        self.inputs = inputs
+        self.outputs = outputs
+
+        # preprocessors and postprocessors
+        self.preprocessors = preprocessors if preprocessors is not None else lambda x: x
+        self.postprocessors = postprocessors if postprocessors is not None else lambda x: x
+
+        # Check the given model: check if correct input/output sizes wrt the previous arguments, and check
+        # the model type and wrap it if necessary. That is, if the type is from the original module/library,
+        # wrap it with the corresponding inner model
+        self.model = model
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def inputs(self):
+        return self._inputs
+
+    @inputs.setter
+    def inputs(self, inputs):
+        if inputs is not None:
+            if isinstance(inputs, (int, float)):
+                inputs = np.array([inputs])
+            elif not isinstance(inputs, (State, Action, torch.Tensor, np.ndarray)):
+                raise TypeError("Expecting the inputs to be a State, Action, torch.Tensor, or np.ndarray.")
+            if self._model is not None:
+                pass  # TODO: check that the dimensions agree with the model
+        # set inputs
+        self._inputs = inputs
+
+    @property
+    def outputs(self):
+        return self._outputs
+
+    @outputs.setter
+    def outputs(self, outputs):
+        if outputs is not None:
+            if isinstance(outputs, (int, float)):
+                outputs = np.array([outputs])
+            elif not isinstance(outputs, (State, Action, torch.Tensor, np.ndarray)):
+                raise TypeError("Expecting the outputs to be a State, Action, torch.Tensor, or np.ndarray.")
+            if self._model is not None:
+                pass  # TODO: check that the dimensions agree with the model
+        # set outputs
+        self._outputs = outputs
+
+    @property
+    def model(self):
+        return self._model
+
+    @model.setter
+    def model(self, model):
+        if model is not None:
+            # check model type
+            # if not isinstance(model, Model):
+            #     raise TypeError("Expecting the model to be an instance of Model, instead received: "
+            #                     "{}".format(type(model)))
+            # TODO
+
+            # check model input/output shape
+            if self._inputs is None:
+                raise ValueError("Inputs have not been set.")
+            if self._outputs is None:
+                raise ValueError("Outputs have not been set.")
+            shape = model.input_shape
+
+            # TODO
+
+        # set model
+        self._model = model
+
+    @property
+    def num_parameters(self):
+        """Return the total number of parameters"""
+        return self.model.num_parameters
+
+    ###########
+    # Methods #
+    ###########
+
+    def is_parametric(self):
+        """
+        Return True if the model is parametric.
+
+        Returns:
+            bool: True if the model is parametric.
+        """
+        return self.model.is_parametric()
+
+    def is_linear(self):
+        """
+        Return True if the model is linear (wrt the parameters). This can be for instance useful for some learning
+        algorithms (some only works on linear models).
+
+        Returns:
+            bool: True if it is a linear model
+        """
+        return self.model.is_linear()
+
+    def is_recurrent(self):
+        """
+        Return True if the model is recurrent. This can be for instance useful for some learning algorithms which
+        change their behavior when they deal with recurrent learning models.
+
+        Returns:
+            bool: True if it is a recurrent model.
+        """
+        return self.model.is_recurrent()
+
+    def is_deterministic(self):
+        """
+        Return True if the model is deterministic.
+
+        Returns:
+            bool: True if the model is deterministic.
+        """
+        return self.model.is_deterministic()
+
+    def is_probabilistic(self):
+        """
+        Return True if the model is probabilistic/stochastic.
+
+        Returns:
+            bool: True if the model is probabilistic.
+        """
+        return self.model.is_probabilistic()
+
+    # alias
+    is_stochastic = is_probabilistic
+
+    def is_discriminative(self):
+        """
+        Return True if the model is discriminative, that is, if the model estimates the conditional probability
+        :math:`p(y|x)`.
+
+        Returns:
+            bool: True if the model is discriminative.
+        """
+        return self.model.is_discriminative()
+
+    def is_generative(self):
+        """
+        Return True if the model is generative, that is, if the model estimates the joint distribution of the input
+        and output :math:`p(x,y)`. A generative model allows to sample from it.
+
+        Returns:
+            bool: True if the model is generative.
+        """
+        return self.model.is_generative()
+
+    def reset(self):
+        pass
+
+    def predict(self, x, to_numpy=True):
+        # x = self.preprocessors(x)
+        x = self.model(x.data[0])
+        # x = self.postprocessors(x)
+        return x
+
+    def parameters(self):
+        return self.model.parameters()
+
+    def get_params(self):
+        return list(self.parameters())
+
+    def hyperparameters(self):
+        return self.model.hyperparameters()
+
+    def get_hyperparams(self):
+        return list(self.hyperparameters())
+
+    def get_vectorized_parameters(self, to_numpy=True):
+        return self.model.get_vectorized_parameters(to_numpy=to_numpy)
+
+    def set_vectorized_parameters(self, vector):
+        self.model.set_vectorized_parameters(vector=vector)
+
+    def get_input_dims(self):
+        return self.model.input_dims
+
+    def get_output_dims(self):
+        return self.model.output_dims
+
+    def set_exploration(self):
+        pass
+
+    def save(self, filename):
+        self.model.save(filename)
+
+    def load(self, filename):
+        self.model.load(filename)
+
+    #############
+    # Operators #
+    #############
+
+    def __call__(self, x):
+        return self.predict(x)
+
+    def __str__(self):
+        return self.model.__str__()
+
+
+class RandomApproximator(Approximator):
+    r"""Random Approximator
+    """
+
+    class Random(object):
+
+        def __init__(self, num_outputs, seed=None):
+            self.num_outputs = num_outputs
+            if seed is not None:
+                np.random.seed(seed)
+
+    def __init__(self, outputs, preprocessors=None, postprocessors=None):
+        # call parent class
+        model = self.Random(num_outputs=self._size(outputs), seed=None)
+        super(RandomApproximator, self).__init__(inputs=None, outputs=outputs, model=model,
+                                                 preprocessors=preprocessors, postprocessors=postprocessors)
+
+    def _size(self, x):
+        size = 0
+        if isinstance(x, (State, Action)):
+            if x.isDiscrete():
+                size = x.space[0].n
+            else:
+                size = x.totalSize()
+        elif isinstance(x, np.ndarray):
+            size = x.size
+        elif isinstance(x, torch.Tensor):
+            size = x.numel()
+        elif isinstance(x, int):
+            size = x
+        return size
+
+    # def predict(self, x):
+    #     if isinstance(self.outputs, (State, Action)):
+    #         # get the space of each output
+    #         spaces = self.outputs.space
+    #
+    #         # sample from each space
+    #         output_data = [space.sample() for space in spaces]
+    #
+    #         # set the data for each action
+    #         self.outputs.data = output_data
+    #
+    #         return self.outputs
+
+    def predict(self, x):
+        # x = self.preprocessors(x)
+        x = self.model.predict(x.data[0])
+        if isinstance(self.outputs, (State, Action)) and self.outputs.isDiscrete():
+            x = np.argmax(x)
+        # x = self.postprocessors(x)
+        return x
+
+
+class LinearApproximator(Approximator):
+    r"""Linear Function Approximator
+    """
+
+    def __init__(self, inputs, outputs, preprocessors=None, postprocessors=None):
+        # call parent class
+        model = Linear(num_inputs=self._size(inputs), num_outputs=self._size(outputs), add_bias=True)
+        super(LinearApproximator, self).__init__(inputs, outputs, model=model, preprocessors=preprocessors,
+                                                 postprocessors=postprocessors)
+
+    def _size(self, x):
+        size = 0
+        if isinstance(x, (State, Action)):
+            if x.isDiscrete():
+                size = x.space[0].n
+            else:
+                size = x.totalSize()
+        elif isinstance(x, np.ndarray):
+            size = x.size
+        elif isinstance(x, torch.Tensor):
+            size = x.numel()
+        elif isinstance(x, int):
+            size = x
+        return size
+
+    def predict(self, x, to_numpy=True):
+        # x = self.preprocessors(x)
+        x = self.model.predict(x.data[0], to_numpy=to_numpy)
+        if isinstance(self.outputs, (State, Action)) and self.outputs.isDiscrete():
+            if to_numpy:
+                x = np.array([np.argmax(x)])
+            else:
+                x = torch.argmax(x, dim=0, keepdim=True)
+        # x = self.postprocessors(x)
+        return x
+
+
+class NNApproximator(Approximator):
+    r"""Neural Network Function Approximator
+    """
+
+    def __init__(self, inputs, outputs, model, preprocessors=None, postprocessors=None):
+
+        # call parent class
+        super(NNApproximator, self).__init__(inputs, outputs, model, preprocessors=preprocessors,
+                                             postprocessors=postprocessors)
+
+        # convert/wrap the model
+        if not isinstance(model, NN):
+            model = NN(model, input_dims=inputs.shape, output_dims=outputs.shape)  # TODO
+        self.model = model
+
+
+class MLPApproximator(NNApproximator):
+    r"""Multi-Layer Perceptron Function Approximator
+
+    It creates a feed-forward and fully-connected neural network, where linear layers are followed by non-linear
+    activation functions. The input and output dimensions are inferred from the inputs and outputs.
+    """
+
+    def __init__(self, inputs, outputs, hidden_units=(),
+                 activation_fct='Linear', last_activation_fct=None, dropout_prob=None,
+                 preprocessors=None, postprocessors=None):
+
+        # check that the inputs and ouputs are 1D
+        # if not self._check1D(inputs):
+        #     raise ValueError("Length of input shape should be 1! Instead, got {}".format(inputs.shape))
+        # print(outputs)
+        # print(outputs.shape)
+        # if not self._check1D(outputs):
+        #     raise ValueError("Length of output shape should be 1! Instead, got {}".format(outputs.shape))
+
+        input_size = self._size(inputs)
+        output_size = self._size(outputs)
+
+        # create model
+        num_units = [input_size] + list(hidden_units) + [output_size]
+        model = MLP(num_units=num_units, activation_fct=activation_fct, last_activation_fct=last_activation_fct,
+                    dropout_prob=dropout_prob)
+
+        # call superclass
+        super(MLPApproximator, self).__init__(inputs, outputs, model, preprocessors=preprocessors,
+                                              postprocessors=postprocessors)
+
+    def _size(self, x):
+        if isinstance(x, (State, Action)):
+            size = x.totalSize()
+        elif isinstance(x, np.ndarray):
+            size = x.size
+        elif isinstance(x, torch.Tensor):
+            size = x.numel()
+        elif isinstance(x, int):
+            size = x
+        return size
+
+    def _check1D(self, arg):
+        """Check that the given argument is a 1D vector, or simple array"""
+        # if isinstance(arg, np.ndarray):
+        shapes = arg.shape
+        # else:
+        #     shape = arg.shape()
+        for shape in shapes:
+            if not (len(shape) == 1 and isinstance(shape[0], int)):
+                return False
+        return True
+
+    def predict(self, x):
+        # convert given input to torch tensor
+        if isinstance(x, (State, Action)):
+            x = np.concatenate((x. data))
+            x = torch.from_numpy(x).float()
+
+        # feed it to the model and get predicted output
+        x = self.model(x)
+
+        # check output
+        if isinstance(self.outputs, (State, Action)):
+            # data
+            self.outputs.train_data = x
+            # convert back output from torch tensor to np array
+            x = x.detach().numpy()
+            self.outputs.data = x
+
+            return self.outputs
+
+        return x
+
+
+class NEATApproximator(Approximator):
+    r"""NEAT Approximator
+
+    See Also: `neat_model.py`, `neat_policy`, `neat_algo.py`
+    """
+
+    def __init__(self, inputs, outputs, num_hidden=0, activation_fct='relu', network_type='feedforward',
+                 aggregation='sum', weights_limits=(-20, 20), bias_limits=(-20, 20),
+                 preprocessors=None, postprocessors=None):
+
+        # call parent class
+        model = NEATModel(num_inputs=self._size(inputs), num_outputs=self._size(outputs), num_hidden=num_hidden,
+                          activation_fct=activation_fct, network_type=network_type, aggregation=aggregation,
+                          weights_limits=weights_limits, bias_limits=bias_limits)
+        super(NEATApproximator, self).__init__(inputs, outputs, model=model, preprocessors=preprocessors,
+                                               postprocessors=postprocessors)
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def config(self):
+        """Return the config object"""
+        return self.model.config
+
+    @config.setter
+    def config(self, config):
+        """Set the config file (str) or object."""
+        self.model.config = config
+
+    @property
+    def genome(self):
+        return self.model.genome
+
+    @genome.setter
+    def genome(self, genome):
+        self.model.genome = genome
+
+    @property
+    def network(self):
+        return self.model.network
+
+    @property
+    def population(self):
+        return self.model.population
+
+    ###########
+    # Methods #
+    ###########
+
+    def _size(self, x):
+        size = 0
+        if isinstance(x, (State, Action)):
+            if x.isDiscrete():
+                size = x.space[0].n
+            else:
+                size = x.totalSize()
+        elif isinstance(x, np.ndarray):
+            size = x.size
+        elif isinstance(x, torch.Tensor):
+            size = x.numel()
+        elif isinstance(x, int):
+            size = x
+        return size
+
+    def predict(self, x, to_numpy=True):
+        # x = self.preprocessors(x)
+        x = self.model.predict(x.merged_data[0])
+
+        if isinstance(self.outputs, (State, Action)):
+            if self.outputs.isDiscrete():
+                x = np.argmax(x)
+            elif self.outputs.isContinuous():
+                x = 2 * np.array(x) - 1
+            else:
+                raise NotImplementedError("The outputs are not discrete or continuous...")
+            self.outputs.data = x
+        # x = self.postprocessors(x)
+        # return x
+        return self.outputs
+
+    def set_network(self, genome=None, config=None):
+        self.model.set_network(genome, config)
+
+    def update_config(self, config):
+        self.model.update_config(config)
+
+
+# Tests
+if __name__ == '__main__':
+    pass

pyrobolearn/approximators/processor.py

@@ -0,0 +1,33 @@
+
+
+class Processor(object):
+    r"""Processor
+
+    This class describes pre- and post- processors. Specifically, it describes how to process the data before and
+    after the policy.
+    """
+    def __init__(self, inputs, outputs):
+        self.inputs = inputs
+        self.outputs = outputs
+
+
+class PreProcessor(Processor):
+    r"""Preprocessor
+
+    It processes the input data before giving it to the policy/controller. For instance, it can be a state estimator
+    such as a kalman filter.
+    """
+
+    def __init__(self, inputs, outputs):
+        super(PreProcessor, self).__init__(inputs, outputs)
+
+
+class PostProcessor(Processor):
+    r"""Postprocessor
+
+    It processes the data outputted by the policy. For instance, the policy could output cartesian positions,
+    and we could process this data using an inverse kinematic scheme to output joint data.
+    """
+
+    def __init__(self, inputs, outputs):
+        super(PostProcessor, self).__init__(inputs, outputs)

pyrobolearn/metrics/metric.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""Defines the various metrics used in different learning paradigms.
+
+Dependencies:
+- `pyrobolearn.tasks`
+"""
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class Metric(object):
+    r"""Metric (abstract) class
+
+    The metric class contains the various metrics used to evaluate a certain learning paradigm (e.g. imitation
+    learning, reinforcement learning, transfer learning, active learning, and so on).
+
+    It notably contains the functionalities to evaluate a certain task using the metric, and different to plot them.
+    """
+
+    def __init__(self):
+        pass
+
+
+class ILMetric(Metric):
+    r"""Imitation Learning Metric
+
+    Metrics used in imitation learning.
+
+    References:
+        [1] "Learning from Humans", Billard et al., 2016
+    """
+
+    def __init__(self):
+        super(ILMetric, self).__init__()
+
+
+class RLMetric(Metric):
+    r"""Reinforcement Learning Metric
+
+    Metrics used in reinforcement learning.
+
+    References:
+        [1] "Deep Reinforcement Learning that Matters", Henderson et al., 2018
+    """
+
+    def __init__(self):
+        super(RLMetric, self).__init__()
+
+
+class TLMetric(Metric):
+    r"""Transfer Learning Metric
+
+    Metrics used in transfer learning.
+
+    References:
+        [1] "A Survey on Transfer Learning", Pan et al., 2010
+        [2] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
+    """
+
+    def __init__(self):
+        super(TLMetric, self).__init__()

pyrobolearn/policies/__init__.py

@@ -0,0 +1,18 @@
+
+# import general policy
+from policy import Policy
+
+# import basic policies
+from basic_policy import *
+
+# import nn policies
+from nn_policy import *
+
+# import dmp policies
+from dmp_policy import *
+
+# import cpg policies
+from cpg_policy import *
+
+# import neat policies
+from neat_policy import *

pyrobolearn/policies/basic_policy.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+"""Define various basic policies.
+
+Define the various basic policies such as the random policy, linear policy, policies based on value functions, etc.
+"""
+
+import numpy as np
+from policy import Policy
+from pyrobolearn.approximators import LinearApproximator
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class RandomPolicy(Policy):
+    """Random policy
+    """
+
+    def __init__(self, states, actions, seed=None, *args, **kwargs):
+        super(RandomPolicy, self).__init__(states, actions, *args, **kwargs)
+        if seed is not None:
+            np.random.seed(seed)
+
+    def act(self, state=None, deterministic=False, to_numpy=True):
+        # get the space of each action
+        spaces = self.actions.space
+
+        # sample from each space
+        action_data = [space.sample() for space in spaces]
+
+        # set the data for each action
+        self.actions.data = action_data
+
+        return self.actions
+
+    def sample(self, state):
+        return self.act(state)
+
+    def reset(self, seed=None):
+        if seed is not None:
+            np.random.seed(seed)
+
+
+class LinearPolicy(Policy):
+    """Linear Policy
+    """
+
+    def __init__(self, states, actions, *args, **kwargs):
+        model = LinearApproximator(states, actions)
+        super(LinearPolicy, self).__init__(states, actions, model, *args, **kwargs)
+
+    def act(self, state, deterministic=True, to_numpy=True):
+        return self.model.predict(state, to_numpy=to_numpy)
+
+    def sample(self, state):
+        pass
+
+
+class PolicyFromValue(Policy):
+    r"""Policy From Value Function Approximator
+
+    .. math::
+
+        a = argmax_a Q^\pi(s,a)
+
+    .. seealso::
+
+        * `value.py`
+    """
+
+    def __init__(self, states, actions, *args, **kwargs):
+        super(PolicyFromValue, self).__init__(states, actions, *args, **kwargs)
+
+    def act(self, state=None, deterministic=True, to_numpy=True):
+        pass
+
+    def sample(self, state):
+        pass

pyrobolearn/policies/cpg_policy.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+"""Define the Central Pattern Generator (CPG) Policy.
+
+Define the various CPG policies that can be used.
+"""
+
+import numpy as np
+import torch
+
+from pyrobolearn.models import CPGNetwork
+from policy import Policy
+from pyrobolearn.states import State
+from pyrobolearn.actions import Action, JointPositionAction
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class CPGPolicy(Policy):
+    r"""Central Pattern Generator (CPG) Network policy
+    """
+
+    def __init__(self, states, actions, timesteps=100, rate=1, *args, **kwargs):
+        super(CPGPolicy, self).__init__(states, actions, rate=rate, *args, **kwargs)
+
+        # check actions
+        if not isinstance(actions, JointPositionAction):
+            raise TypeError("Expecting the actions to be an instance of JointPositionAction, instead got: "
+                            "{}".format(type(actions)))
+
+        # create CPG network based on the robot kinematic structures
+
+        # get specified legs
+        robot = actions.robot
+        joints = set(actions.joints)
+        legs = []
+        for robot_leg in robot.legs:
+            leg = []
+            for joint in robot_leg:
+                if joint in joints:
+                    leg.append(joint)
+            legs.append(leg)
+
+        num_legs = len(legs)
+
+        # define few variables to initialize the CPG nodes
+        # variables for the node
+        init_phi = 0.
+        offset = 0.
+        amplitude = 1.
+        freq = 1.
+        # variables for coupling the nodes
+        weight_legs = 1. / len(legs)
+        if len(legs) > 0 and len(legs[0]) > 0:
+            weight_leg = 1. / len(legs[0])
+        else:
+            weight_leg = 0.
+        bias = 0.
+
+        # create the CPG network based on the robot kinematic structures
+        nodes = {}
+        for leg_idx, leg in enumerate(legs):
+            for idx, joint in enumerate(leg):
+                # proper node parameters
+                node = {'phi': init_phi, 'offset': offset, 'amplitude': amplitude, 'freq': freq}
+
+                # coupling parameters
+
+                # if first upper joint in the leg, connect it with the other upper joints (in the other legs)
+                if idx == 0:
+                    for l in legs:
+                        if len(l) > 0 and joint != l[0]:  # i.e. not the same node
+                            coupling_params = {'id': l[0], 'weight': weight_legs, 'bias': bias}
+                            node.setdefault('nodes', []).append(coupling_params)
+
+                # add coupling to current joint (except the last one) with the next joint in the leg
+                if idx < len(leg)-1:
+                    # add next node
+                    coupling_params = {'id': leg[idx + 1], 'weight': weight_leg, 'bias': bias}
+                    node.setdefault('nodes', []).append(coupling_params)
+
+                # add coupling to current joint (except the first one) with the previous joint in the leg
+                if idx > 0:
+                    # add next node
+                    coupling_params = {'id': leg[idx - 1], 'weight': weight_leg, 'bias': bias}
+                    node.setdefault('nodes', []).append(coupling_params)
+
+                # add node in the CPG network dictionary
+                nodes[joint] = node
+
+        # create learning model
+        self.model = CPGNetwork(nodes=nodes, timesteps=timesteps)
+
+    def _size(self, x):
+        size = 0
+        if isinstance(x, (State, Action)):
+            if x.isDiscrete():
+                size = x.space[0].n
+            else:
+                size = x.totalSize()
+        elif isinstance(x, np.ndarray):
+            size = x.size
+        elif isinstance(x, torch.Tensor):
+            size = x.numel()
+        elif isinstance(x, int):
+            size = x
+        return size
+
+    def act(self, state=None, deterministic=True, to_numpy=True):
+        if (self.cnt % self.rate) == 0:
+            self.last_action = self.model.step()
+        self.cnt += 1
+        # angles = self.model.step()
+        # self.actions.data = angles
+        self.actions.data = self.last_action
+        return self.actions
+
+    def sample(self, state):
+        pass
+
+    def phase_resetting(self):
+        self.model.reset()
+        pass

pyrobolearn/policies/dmp_policy.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+"""Define the Dynamic Movement Primitive (DMP) Policy.
+
+Define the various DMP policies that can be used.
+"""
+
+import numpy as np
+import torch
+
+from pyrobolearn.models import DMP, DiscreteDMP, RhythmicDMP, BioDiscreteDMP
+from policy import Policy
+from pyrobolearn.states import State
+from pyrobolearn.actions import Action, JointPositionAction, JointVelocityAction, JointAccelerationAction
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class DMPPolicy(Policy):
+    r"""Dynamic Movement Primitive (DMP) policy
+    """
+
+    def __init__(self, states, actions, model, rate=1, *args, **kwargs):
+        if not isinstance(model, DMP):
+            raise TypeError("Expecting model to be an instance of DMP")
+        super(DMPPolicy, self).__init__(states, actions, model, rate=rate, *args, **kwargs)
+        self.y, self.dy, self.ddy = 0, 0, 0
+
+    def _size(self, x):
+        size = 0
+        if isinstance(x, (State, Action)):
+            if x.isDiscrete():
+                size = x.space[0].n
+            else:
+                size = x.totalSize()
+        elif isinstance(x, np.ndarray):
+            size = x.size
+        elif isinstance(x, torch.Tensor):
+            size = x.numel()
+        elif isinstance(x, int):
+            size = x
+        return size
+
+    def act(self, state, deterministic=True, to_numpy=True):
+        # return self.model.predict(state, to_numpy=to_numpy)
+        if (self.cnt % self.rate) == 0:
+            self.y, self.dy, self.ddy = self.model.step(state.data[0][0])
+        self.cnt += 1
+        # y, dy, ddy = self.model.step()
+        # return np.array([y, dy, ddy])
+        if isinstance(self.actions, JointPositionAction):
+            self.actions.data = self.y
+        elif isinstance(self.actions, JointVelocityAction):
+            self.actions.data = self.dy
+        elif isinstance(self.actions, JointAccelerationAction):
+            self.actions.data = self.ddy
+        return self.actions
+
+    def sample(self, state):
+        pass
+
+    def rollout(self):
+        return self.model.rollout()
+
+    def imitate(self, data):
+        if len(data) > 0:
+            print("Imitating with :", data.shape)
+            # y, dy, ddy = data
+            y = data
+            # if len(y.shape) == 1:
+            #     y = y.reshape(1, -1)
+            # if len(dy.shape) == 1:
+            #     dy = dy.reshape(1, -1)
+            # if len(ddy.shape) == 1:
+            #     ddy = ddy.reshape(1, -1)
+            self.model.imitate(y, plot=True)  # dy, ddy, plot=True)  # dy, ddy)
+        else:
+            print("Nothing to imitate.")
+
+
+class DiscreteDMPPolicy(DMPPolicy):
+    r"""Discrete DMP Policy
+    """
+
+    def __init__(self, actions, states=None, num_basis=20, dt=0.01, y0=0, goal=1, forcing_terms=None,
+                 stiffness=None, damping=None, rate=1):
+        if not isinstance(actions, Action):
+            raise TypeError("Expecting actions to be an instance of the 'Action' class.")
+        model = DiscreteDMP(num_dmps=self._size(actions), num_basis=num_basis, dt=dt, y0=y0, goal=goal,
+                            forcing_terms=forcing_terms, stiffness=stiffness, damping=damping)
+        super(DiscreteDMPPolicy, self).__init__(states, actions, model, rate=rate)
+
+
+class RhythmicDMPPolicy(DMPPolicy):
+    r"""Rhythmic DMP Policy
+    """
+
+    def __init__(self, actions, states=None, num_basis=20, dt=0.01, y0=0, goal=1, forcing_terms=None,
+                 stiffness=None, damping=None, rate=1):
+        model = RhythmicDMP(num_dmps=self._size(actions), num_basis=num_basis, dt=dt, y0=y0, goal=goal,
+                            forcing_terms=forcing_terms, stiffness=stiffness, damping=damping)
+        super(RhythmicDMPPolicy, self).__init__(states, actions, model, rate=rate)
+
+
+class BioDiscreteDMPPolicy(DMPPolicy):
+    r"""Bio Discrete DMP Policy
+    """
+
+    def __init__(self, actions, states=None, num_basis=20, dt=0.01, y0=0, goal=1, forcing_terms=None,
+                 stiffness=None, damping=None, rate=1):
+        model = BioDiscreteDMP(num_dmps=self._size(actions), num_basis=num_basis, dt=dt, y0=y0, goal=goal,
+                               forcing_terms=forcing_terms, stiffness=stiffness, damping=damping)
+        super(BioDiscreteDMPPolicy, self).__init__(states, actions, model, rate=rate)

pyrobolearn/policies/neat_policy.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+"""Define the NEAT policy class.
+
+This uses the Neuro-Evolution through Augmenting topologies (NEAT) framework. It allows the evolution of not only the
+parameters/weights but also the topological structure of neural networks. Note that the model associated with
+this policy (i.e. the neural network) is tightly coupled with the algorithm that modifies it.
+"""
+
+
+import numpy as np
+import cPickle as pickle
+
+try:
+    # from neat import nn, population, config, statistics
+    import neat
+except ImportError as e:
+    raise ImportError(e.__str__() + "\n HINT: you can install NEAT directly via 'pip install neat-python'.")
+
+from policy import Policy
+from pyrobolearn.approximators import NEATApproximator
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class NEATPolicy(Policy):
+    r"""NEAT Policy
+
+    NEAT stands for "Neuro-Evolution through Augmenting Topologies" [1] and allows the evolution of not only the
+    parameters/weights but also the topological structure of neural networks. The model associated with this
+    policy (i.e. the neural network) is tightly coupled with the algorithm that modifies it.
+    By the structure of the neural network, we mean the type (i.e. forward or recurrent) and number of connection,
+    as well as the type (i.e. using non-linearity activation function) and number of nodes can change.
+
+    Exploration is thus carried out in the parameter and hyper-parameter spaces.
+
+    Warnings: The associated algorithm is a little bit special and currently only works with the corresponding
+    policy/learning model.
+
+    References:
+        [1] "Evolving Neural Networks through Augmenting Topologies", Stanley et al., 2002
+        [2] NEAT-Python
+            - documentation: https://neat-python.readthedocs.io/en/latest/index.html
+            - github repo: https://github.com/CodeReclaimers/neat-python
+        [3] PyTorch NEAT (built upon NEAT-Python): https://github.com/uber-research/PyTorch-NEAT
+    """
+
+    def __init__(self, states, actions, num_hidden=0, activation_fct='relu', network_type='feedforward',
+                 aggregation='sum', weights_limits=(-20, 20), bias_limits=(-20, 20), rate=1, *args, **kwargs):
+        r"""Initialize the neural network policy for the NEAT algorithm.
+        """
+        model = NEATApproximator(states, actions, num_hidden=num_hidden, activation_fct=activation_fct,
+                                 network_type=network_type, aggregation=aggregation, weights_limits=weights_limits,
+                                 bias_limits=bias_limits)
+        super(NEATPolicy, self).__init__(states, actions, model, rate=rate, *args, **kwargs)
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def config(self):
+        """Return the config object"""
+        return self.model.config
+
+    @config.setter
+    def config(self, config):
+        """Set the config file (str) or object."""
+        self.model.config = config
+
+    @property
+    def genome(self):
+        return self.model.genome
+
+    @genome.setter
+    def genome(self, genome):
+        self.model.genome = genome
+
+    @property
+    def network(self):
+        return self.model.network
+
+    @property
+    def population(self):
+        return self.model.population
+
+    ###########
+    # Methods #
+    ###########
+
+    def update_config(self, config):
+        self.model.update_config(config)
+
+    def set_network(self, genome=None, config=None):
+        self.model.set_network(genome, config)
+
+    def act(self, state, deterministic=True):
+        if (self.cnt % self.rate) == 0:
+            self.last_action = self.model.predict(state)
+        self.cnt += 1
+        return self.last_action
+
+    def sample(self, state):
+        pass
+
+
+class NEATFeedForwardPolicy(NEATPolicy):
+    r"""NEAT feed-forward policy
+
+    This creates a feed-forward network policy.
+    """
+
+    def __init__(self, states, actions, genome):
+        super(NEATFeedForwardPolicy, self).__init__(states, actions, genome, network_type='feedforward')
+
+
+class NEATRecurrentPolicy(NEATPolicy):
+    r"""NEAT recurrent policy
+
+    This creates a recurrent network policy.
+    """
+
+    def __init__(self, states, actions, genome):
+        super(NEATRecurrentPolicy, self).__init__(states, actions, genome, network_type='recurrent')

pyrobolearn/policies/nn_policy.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+"""Define the Neural Network (NN) Policies.
+
+Define the various neural network policies that can be used.
+"""
+
+from pyrobolearn.approximators import NNApproximator, MLPApproximator
+from policy import Policy
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class NNPolicy(Policy):
+    r"""Neural Network Policy
+
+    Defines the neural network policy. If the model is not given,
+
+    Examples:
+        simulator = Bullet()
+        robot = Robot(simulator)
+        policy = NNPolicy(Robot, states=['joint_positions', 'joint_velocities'], actions=['joint_positions'])
+    """
+
+    def __init__(self, states, actions, model=None, *args, **kwargs):
+        if model is None:
+            raise ValueError("Expecting a NN model for the NN policy")
+        else:
+            # checking the input dimension of the model and the dimension of states
+            # checking the output dimension of the model and the dimension of actions
+            pass
+
+        super(NNPolicy, self).__init__(states, actions, model, *args, **kwargs)
+
+    def act(self, state, deterministic=True):
+        pass
+
+    def sample(self, state):
+        pass
+
+
+class MLPPolicy(NNPolicy):
+    r"""Multi-Layer Perceptron (MLP) Policy
+
+    Defines a MLP policy, which is a feedforward fully-connected neural network with linear layers and nonlinear
+    activation functions.
+    """
+
+    def __init__(self, states, actions, hidden_units=(),
+                 activation_fct='linear', last_activation_fct=None, dropout_prob=None,
+                 preprocessors=None, postprocessors=None):
+        """Initialize MLP policy.
+
+        Args:
+            states (State): 1D-states that is feed to the policy (the input dimensions will be inferred from the
+                            states)
+            actions (Action): 1D-actions outputted by the policy and will be applied in the simulator (the output
+                              dimensions will be inferred from the actions)
+            hidden_units (list/tuple of int): number of hidden units in the corresponding layer
+            activation_fct (None, str, or list/tuple of str/None): activation function to be applied after each layer.
+                                                                   If list/tuple, then it has to match the
+            last_activation_fct (None or str): last activation function to be applied. If not specified, it will check
+                                               if it is in the list/tuple of activation functions provided for the
+                                               previous argument.
+            dropout_prob (None, float, or list/tuple of float/None): dropout probability.
+        """
+        model = MLPApproximator(states, actions, hidden_units=hidden_units,
+                                activation_fct=activation_fct, last_activation_fct=last_activation_fct,
+                                dropout_prob=dropout_prob, preprocessors=preprocessors, postprocessors=postprocessors)
+        super(MLPPolicy, self).__init__(states, actions, model)
+
+    def act(self, state, deterministic=True):
+        return self.model.predict(state)

pyrobolearn/policies/policy.py

@@ -0,0 +1,385 @@
+#!/usr/bin/env python
+"""Define the basic Policy class.
+
+A policy couples one or several learning model(s), the state, and action together. In this framework, the policy
+usually represents the robot's "brain".
+
+Dependencies:
+- `pyrobolearn.states`
+- `pyrobolearn.actions`
+- `pyrobolearn.approximators` (and thus `pyrobolearn.models`)
+"""
+
+from abc import ABCMeta, abstractmethod
+import pickle
+import torch
+
+from pyrobolearn.states import State
+from pyrobolearn.actions import Action
+
+from pyrobolearn.models import Model
+from pyrobolearn.approximators import Approximator, NNApproximator
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class Policy(object):
+    r"""Abstract `Policy` class.
+
+    A policy maps a state to an action, and is often denoted as :math:`\pi_{\theta}(a_t|s_t)`, where :math:`\theta`
+    represents the policy parameters. It represents the cognition of the agent(s).
+    In our framework, the policy groups the learning model, state, and action objects.
+
+    Specifically, the policy is dissociated from the learning model, as a learning model can be used for different
+    purposes. For instance, a neural network can be used to represent a policy but also a value function approximator,
+    thus we separate these 2 notions (policy and learning model).
+
+    The policy is also loosely dissociated from the simulator and more specifically from the agent's body, as this last
+    one is seen as being part of the environment. The states and actions are what connects the policy with the
+    environment (and thus the simulator). The states and actions are given to the policy, and allows to build
+    automatically a learning model (if not given) by inferring the dimensions of the inputs and outputs of the model.
+
+    .. note::
+
+        Exploration can be carried out by the policy, by specifying the exploration strategy (that is, exploration
+        in the parameter or action space).
+
+    Example::
+
+        # create simulator
+        simulator = BulletSim(render=True)
+
+        # create world
+        world = BasicWorld(simulator)
+
+        # create robot
+        robot = world.loadRobot('robot_name')
+        # or load the robot (via urdf) and spawns it in the simulator
+        #robot = Robot(simulator)
+        #world.loadRobot(robot) # a robot is part of the world (if not done, it will be done inside Env)
+
+        # create states / actions
+        states = JntPositionState(robot) + JntVelocityState(robot)
+        actions = JntPositionAction(robot)
+
+        # optional: create learning model (if defined, it has to agree with the dimensions of states/actions)
+        model = NN(...)
+
+        # create policy (if learning model not defined, it will create it inside)
+        policy = Policy(states, actions, model)
+
+        # create rewards/costs (i.e. r(s,a,s')): gives robot, or state/actions
+        reward = ForWardProgressReward(robot) - FallenCost(robot) - PowerConsumptionCost(robot)
+
+        # create environment to interact with
+        env = Env(world, states, rewards)
+
+        # create and run task
+        task = Task(env, policy)
+        task.run()
+
+        # Optional: create RL algo (see RL_Algo)
+
+    .. seealso::
+
+        * `state.py`: describes the various states
+        * `action.py`: describes the various actions
+        * `model.py`: describes the abstract learning model class
+        * `exploration.py`: describes how to explore using the policy
+    """
+    __metaclass__ = ABCMeta
+
+    def __init__(self, states, actions, model=None, rate=1, preprocessors=None, postprocessors=None,
+                 distribution=None, *args, **kwargs):
+        r"""
+        Initialize a policy, the learning model.
+
+        Args:
+            states (State): By giving the `states` to the policy, it can automatically infer the type and size/shape of
+                       each state, and thus can be used to automatically build a policy. At each step, the `states`
+                       are filled by the environment, and read by the policy. The `state` connects the policy with
+                       one or several objects (including robots) in the environment.
+                       Note that some policies don't use any state information.
+            actions (Action): At each step, by calling `policy.act(state)`, the `actions` are computed by the policy,
+                        and should be given to the environment. As with the `states`, the type and size/shape of
+                        each action can be inferred and could be used to automatically build a policy.
+                        The `action` connects the policy with a controllable object (such as a robot) in the
+                        environment.
+            model (Model, Approximator, None): inner model or approximator
+            rate (int): rate at which the policy operates
+            distribution:
+            args:
+            kwargs:
+        """
+        self.states = states
+        self.actions = actions
+        self.model = model
+        self.train_mode = False
+        self.rate = rate
+        self.cnt = 0
+        self.last_action = None
+
+    ##############
+    # Properties #
+    ##############
+
+    @property
+    def states(self):
+        return self._states
+
+    @states.setter
+    def states(self, states):
+        if states is not None:
+            if not isinstance(states, State):
+                raise TypeError("Expecting states to be an instance of State.")
+        self._states = states
+
+    @property
+    def actions(self):
+        return self._actions
+
+    @actions.setter
+    def actions(self, actions):
+        if not isinstance(actions, Action):
+            raise TypeError("Expecting actions to be an instance of Action.")
+        self._actions = actions
+
+    @property
+    def model(self):
+        return self._model
+
+    @model.setter
+    def model(self, model):
+        if model is not None and not isinstance(model, Approximator):
+            # Try to wrap it with the corresponding Approximator
+            if isinstance(model, Model):
+                model = Approximator(inputs=self.states, outputs=self.actions, model=model)
+            elif isinstance(model, torch.nn.Module):
+                model = NNApproximator(inputs=self.states, outputs=self.actions, model=model)
+            # else:
+            #     raise TypeError("Expecting the model to be an instance of Model.")
+        self._model = model
+
+    @property
+    def rate(self):
+        """Return the rate at which the policy operates."""
+        return self._rate
+
+    @rate.setter
+    def rate(self, rate):
+        """Set the rate at which the policy operates."""
+        if not isinstance(rate, int):
+            raise TypeError("Expecting the rate to be an integer.")
+        self._rate = rate
+
+    @property
+    def parameters(self):
+        """
+        Return an iterator over the learning model parameters.
+        """
+        if self.model is None:
+            return None
+        return self.model.parameters()
+
+    @property
+    def hyperparameters(self):
+        """
+        Return an iterator over the learning model hyperparameters.
+        """
+        if self.model is None:
+            return None
+        return self.model.hyperparameters()
+
+    @property
+    def input_dims(self):
+        """
+        Return the input dimension of the policy.
+        """
+        if self.model is None:
+            return None
+        return self.model.get_input_dims()
+
+    @property
+    def output_dims(self):
+        """
+        Return the output dimension of the policy.
+        """
+        if self.model is None:
+            return None
+        return self.model.get_output_dims()
+
+    @property
+    def num_parameters(self):
+        """Return the total number of parameters"""
+        return self.model.num_parameters
+
+    ###########
+    # Methods #
+    ###########
+
+    def is_deterministic(self):
+        """
+        Return True if the policy is deterministic; that is, given the same states result in the same actions.
+        .. math:: a_t = f(s_t)
+
+        Returns:
+            bool: True if the policy is deterministic
+        """
+        return self.model.is_deterministic()
+
+    def is_stochastic(self):
+        """
+        Return True if the policy is stochastic; that is, given the same states can result in different actions.
+        .. math:: a_t ~ p(a_t|s_t)
+
+        Returns:
+            bool: True if the policy is stochastic
+        """
+        return self.model.is_stochastic()
+
+    def is_parametric(self):
+        """
+        Return True if the policy is parametric.
+
+        Returns:
+            bool: True if the policy is parametric.
+        """
+        return self.model.is_parametric()
+
+    def is_linear(self):
+        """
+        Return True if the policy is linear (wrt the parameters). This can be for instance useful for some learning
+        algorithms (some only works on linear models).
+
+        Returns:
+            bool: True if it is a linear policy
+        """
+        return self.model.is_linear()
+
+    def is_recurrent(self):
+        """
+        Return True if the policy is recurrent. This can be for instance useful for some learning algorithms which
+        change their behavior when they deal with recurrent learning models.
+
+        Returns:
+            bool: True if it is a recurrent policy.
+        """
+        raise self.model.is_recurrent()
+
+    def get_vectorized_parameters(self, to_numpy=True):
+        return self.model.get_vectorized_parameters(to_numpy=to_numpy)
+
+    def set_vectorized_parameters(self, vector):
+        self.model.set_vectorized_parameters(vector=vector)
+
+    @abstractmethod
+    def act(self, state, deterministic=True, to_numpy=True):
+        """
+        Perform the action given the state.
+
+        Args:
+            state (State): current state
+            deterministic (bool): True by default. It can only be set to False, if the policy is stochastic.
+            to_numpy (bool): if True, return a np.array
+
+        Returns:
+            Action: action
+        """
+        if self.model is not None:
+            if (self.cnt % self.rate) == 0:
+                self.last_action = self.model.predict(state, to_numpy=to_numpy)
+            self.cnt += 1
+            return self.last_action
+    # predict = act
+
+    @abstractmethod
+    def sample(self, state):
+        """
+        Given the state, sample from the policy. This only works if the inner model of the policy is stochastic.
+
+        Args:
+            state (State, array): current state
+
+        Returns:
+            array: sample
+        """
+        pass
+
+    def train(self, mode=True):
+        """
+        Set the policy to train mode.
+
+        Args:
+            mode (bool): if True, set the policy in train mode.
+
+        Returns:
+            None
+        """
+        self.train_mode = mode
+
+    def reset(self, *args, **kwargs):
+        """
+        Reset the policy.
+        """
+        self.model.reset()
+
+    def get_params(self):
+        """
+        Return the learning model parameters.
+        """
+        if self.model is None:
+            return None
+        return self.model.get_params()
+
+    def get_hyperparams(self):
+        """
+        Return the learning model hyperparameters
+        """
+        if self.model is None:
+            return None
+        return self.model.get_hyperparams()
+
+    def save(self, filename):
+        """
+        Save the policy in the given filename.
+
+        Args:
+            filename (str): file to save the policy into
+
+        Returns:
+            None
+        """
+        # self.model.save(filename)
+        pickle.dump(self, open(filename, 'wb'))
+
+    @staticmethod
+    def load(filename):
+        """
+        Load the policy from the given file.
+
+        Args:
+            filename (str): file to load the policy from
+
+        Returns:
+            None
+        """
+        # self.model.load(filename)
+        return pickle.load(open(filename, 'rb'))
+
+    #############
+    # Operators #
+    #############
+
+    def __call__(self, *args, **kwargs):
+        return self.act(*args, **kwargs)
+
+    def __str__(self):
+        return self.model.__str__()

pyrobolearn/processors/__init__.py

@@ -0,0 +1,6 @@
+
+# import processors
+from processor import *
+
+# import linear processors
+from linear_processor import

pyrobolearn/processors/linear_processor.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+"""Define the Linear Processor class.
+"""
+
+import numpy as np
+import torch
+
+from processor import Processor
+
+
+class LinearProcessor(Processor):
+    r"""Linear Processor
+
+    Linear processor is a linear model: :math:`y = ax + b` where :math:`x` is the input, :math:`y` is the output,
+    and :math:`a` and :math:`b` are given and fixed coefficients (slope and bias).
+    """
+
+    def __init__(self, a, b):
+        super(LinearProcessor, self).__init__()
+        self.a = torch.Tensor(a)
+        self.b = torch.Tensor(b)
+
+    def compute(self, x):
+        if isinstance(x, np.ndarray):
+            x = torch.from_numpy(x).float()
+            y = self.a * x + self.b
+            return y.numpy()
+        return self.a * x + self.b

pyrobolearn/processors/processor.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+"""Define the Processor class.
+
+Processors are rules that are applied to the inputs and outputs of a learning model before being processed by the
+model or after. Processors might have parameters but they do not have trainable/optimizable parameters; the parameters
+are fixed and given at the beginning.
+"""
+
+import numpy as np
+import torch
+
+
+class Processor(object):
+    r"""Processor
+
+    Processors are rules that are applied to the inputs and outputs of a model before being processed by the model
+    or after. Processors might have parameters but they do not have trainable/optimizable parameters; the parameters
+    are fixed and given at the beginning.
+    """
+
+    def __init__(self):
+        pass
+
+    def compute(self, x):
+        pass
+
+    def __call__(self, x):
+        return self.compute(x)
+
+
+class CenterProcessor(Processor):
+    r"""Center Processor
+
+    Center the data by the given mean; that is, it returned: :math:`\hat{x} = x - \mu` where :math:`\mu` is the mean.
+    """
+
+    def __init__(self, mean):
+        super(CenterProcessor, self).__init__()
+        self.mean = torch.Tensor(mean)
+
+    def compute(self, x):
+        if isinstance(x, np.ndarray):
+            x = torch.from_numpy(x).float()
+            x -= self.mean
+            return x.numpy()
+        return x - self.mean
+
+
+class StandardizerProcessor(Processor):
+    r"""Standardizer Processor
+
+    Processor that standardize the given data; the returned data is centered around 0 with a standard deviation of 1.
+    That is, it returned :math:`\hat{x} = \frac{x - \mu}{\sigma}`, where :math:`\mu` is the mean, and :math:`\sigma`
+    is the standard deviation.
+    """
+
+    def __init__(self, mean, std):
+        super(StandardizerProcessor, self).__init__()
+        self.mean = torch.Tensor(mean)
+        self.std = torch.Tensor(std)
+
+    def compute(self, x):
+        if isinstance(x, np.ndarray):
+            x = torch.from_numpy(x).float()
+            x = (x - self.mean) / (self.std + 1.e-13)
+            return x.numpy()
+        return (x - self.mean) / (self.std + 1.e-13)
+
+
+class NormalizerProcessor(Processor):
+    r"""Normalizer Processor
+
+    Processor that normalize the given data; the returned data will be between 0 and 1.
+    That is, it returned :math:`\hat{x} = \frac{x - x_{min}}{x_{max} - x_{min}}`, where
+    :math:`x \in [x_{min}, x_{max}]`.
+    """
+
+    def __init__(self, xmin, xmax):
+        super(NormalizerProcessor, self).__init__()
+        self.xmin = torch.Tensor(xmin)
+        self.xmax = torch.Tensor(xmax)
+        if torch.allclose(self.xmin, self.xmax):
+            raise ValueError("The given arguments 'xmin' and 'xmax' are the same.")
+
+    def compute(self, x):
+        if isinstance(x, np.ndarray):
+            x = torch.from_numpy(x).float()
+            x = (x - self.xmin) / (self.xmax - self.xmin)
+            return x.numpy()
+        return (x - self.xmin) / (self.xmax - self.xmin)

pyrobolearn/recorders/__init__.py

@@ -0,0 +1,3 @@
+
+# import recorders
+from recorder import *

pyrobolearn/recorders/recorder.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+"""Define the recorder classes.
+
+The recorders allow to record the data from a source at a certain rate.
+For instance, it can record the robot states and actions. This can be useful for imitation learning tasks where
+the user demonstrates a certain skill through teleoperation or kinesthetic teaching. Using the recorder, you can
+record the data to be replayed later to the learning model.
+
+Dependencies:
+- `pyrobolearn.states`
+- `pyrobolearn.actions`
+"""
+
+# Memory vs Storage vs Recorder vs Sampler
+
+import pickle
+import copy
+import time
+
+from pyrobolearn.states import State
+from pyrobolearn.actions import Action
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class Recorder(object):
+    """Recorder
+
+    This class allows to record the given data, and save it to a file.
+    """
+
+    def __init__(self, filename=None):
+        """
+        Initialize the recorder.
+
+        Args:
+            filename (str, None): file to save/load the data. If None, it will generate a filename based on the class
+                name and the current local time.
+        """
+        if filename is None:
+            filename = self.__class__.__name__ + time.strftime("_%d-%m-%Y_%Hh%Mm%Ss", time.localtime())
+        self.filename = filename
+        # data row
+        self.data = []
+        # data "matrix" (it is not really a matrix because each row may have different dimension)
+        self.all_data = []
+
+    def __repr__(self):
+        return self.__class__.__name__
+
+    def __len__(self):
+        """
+        Return the current number of data points recorded.
+        """
+        return len(self.data)
+
+    def __iter__(self):
+        """
+        Return iterator over the data.
+
+        Returns:
+            iterator
+        """
+        return iter(self.data)
+
+    def __getitem__(self, key):
+        """
+        Access the data from the recorder.
+
+        Args:
+            key (int, slice): data index
+
+        Returns:
+            np.array: data
+        """
+        return self.data[key]
+
+    def __contains__(self, item):
+        """
+        Check if the given item is in the recorder.
+
+        Warnings: Currently, this is an O(N) operation. Need to use OrderedSet.
+
+        Args:
+            item: item to check if it is in the recorder
+
+        Returns:
+            bool: True if the item is in the recorder
+        """
+        return item in self.data
+
+    def add(self, data):
+        """
+        Add the given data to the recorder.
+
+        Args:
+            data: data to add to the recorder
+        """
+        self.data.append(data)
+
+    # alias
+    __lshift__ = add
+
+    def add_row(self):
+        """
+        Add a new data row in the list of data.
+        """
+        self.all_data.append(self.data)
+        self.data = []
+
+    def remove(self, key=0):
+        """
+        Remove and return the specified data from the recorded data.
+
+        Args:
+            key (int): data index
+        """
+        return self.data.pop(key)
+
+    def remove_last_entry(self):
+        """
+        Remove and return the last recorded datum.
+        """
+        return self.remove(key=-1)
+
+    def remove_first_entry(self):
+        """
+        Remove and return the first recorded datum.
+        """
+        return self.remove(key=0)
+
+    def remove_row(self, key=0):
+        """
+        Remove and return a data row in the list of all data.
+
+        Args:
+            key: data row index
+        """
+        return self.all_data.pop(key)
+
+    def remove_last_row(self):
+        """
+        Remove and return the last data row from the data "matrix".
+        """
+        return self.remove_row(key=-1)
+
+    def remove_first_row(self):
+        """
+        Remove and return the first data row from the data "matrix".
+        """
+        return self.remove_row(key=0)
+
+    def save(self, filename=None, append=True):
+        """
+        Save the recorded data into the specified filename.
+
+        Args:
+            filename (str, None): filename to save the data. If None, use the default one provided at the beginning.
+            append (bool): If True, it will append the data to the end of the file
+        """
+        if filename is None:
+            filename = self.filename
+        mode = "wba" if append else "wb"
+        with open(filename, mode) as f:
+            pickle.dump(self.data, f)
+
+    def load(self, filename=None):
+        """
+        Load the recorded data from the specified filename.
+
+        Args:
+            filename (str, None): filename to load the data from. If None, use the default one provided at the
+                beginning.
+        """
+        if filename is None:
+            filename = self.filename
+        with open(filename, "rb") as f:
+            self.data = pickle.load(f)
+
+    def generate(self):
+        """
+        Return a generator over the recorded data.
+
+        Returns:
+            generator
+        """
+        for data in self.data:
+            yield data
+
+    def reset(self):
+        """
+        Reset the recorder; empty it.
+        """
+        self.data = []
+
+
+class DataRecorder(Recorder):
+    r"""Data Recorder
+
+    This class is useful to record data from a source. Basically, we specify what we wish to record.
+    """
+
+    def __init__(self, source, filename=None, rate=1, update=True):
+        """Initialize the recorder.
+
+        Args:
+            source (object): instance that needs to have the property variable `data`. At each acquisition step,
+                we append/save `source.data`
+            filename (str, None): file to save/load the data. If None, it will generate a filename based on the class
+                name and the current local time.
+            rate (int): sampling rate
+            update (bool): update the source by calling it if callable.
+        """
+
+        # Get world and simulator
+        super(DataRecorder, self).__init__(filename)
+
+        # useful variables
+        if not hasattr(source, 'data'):
+            raise AttributeError("The given source doesn't have the 'data' attribute")
+        self.src = source
+        self.rate = rate
+        self.cnt = 0
+        self.update = update
+
+    def record(self):
+        """
+        Record the data.
+        """
+        if (self.cnt % self.rate) == 0:
+            self.cnt = 0
+            self.update_source()
+            self.data.append(copy.deepcopy(self.src.merged_data))
+        self.cnt += 1
+
+    def __call__(self, *args, **kwargs):
+        self.record()
+
+    def update_source(self):
+        if self.update and callable(self.src):
+            self.src()
+
+    def generate(self):
+        """
+       Return a generator over the recorded data.
+
+       Returns:
+           generator
+       """
+        for data in self.data:
+            self.src.data = data
+            yield data
+
+
+class StateRecorder(DataRecorder):
+    r"""State recorder
+
+    Record the state.
+    """
+
+    def __init__(self, states, filename=None, rate=1, update=True):
+        """
+        Record the given states at each `rate` time steps.
+
+        Args:
+            states (State): states to save
+            filename (str, None): file to save/load the states. If None, it will generate a filename based on the class
+                name and the current local time.
+            rate (int): sampling rate
+            update (bool): update the states by calling it if callable.
+        """
+        if not isinstance(states, State):
+            raise TypeError("Expecting 'states' to be an instance of State")
+        super(StateRecorder, self).__init__(source=states, filename=filename, rate=rate, update=update)
+
+
+class ActionRecorder(DataRecorder):
+    r"""Action recorder
+
+    Record the action.
+    """
+
+    def __init__(self, actions, filename=None, rate=1, update=True):
+        """
+        Record the given actions at each `rate` time steps.
+
+        Args:
+            actions (Action): actions to save
+            filename (str, None): file to save/load the states. If None, it will generate a filename based on the class
+                name and the current local time.
+            rate (int): sampling rate
+            update (bool): update the actions by calling it if callable.
+        """
+        if not isinstance(actions, Action):
+            raise TypeError("Expecting 'actions' to be an instance of Action")
+        super(ActionRecorder, self).__init__(source=actions, filename=filename, rate=rate, update=update)