add polynomial approx, policy, value, dynamics

pyrobolearn/approximators/polynomial.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""Define linear function approximator.
+
+Dependencies:
+- `pyrobolearn.models`
+- `pyrobolearn.states`
+- `pyrobolearn.actions`
+"""
+
+from pyrobolearn.approximators.approximator import Approximator
+from pyrobolearn.models.basics.polynomial import Polynomial, PolynomialFunction
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "GNU GPLv3"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class PolynomialApproximator(Approximator):
+    r"""Polynomial Function Approximator
+
+    The polynomial function approximator is a discriminative deterministic model expressed mathematically as
+    :math:`y = f(x) = W \phi(x)`, where :math:`x` is the input vector, :math:`y` is the output vector, :math:`W`
+    is the weight matrix, and :math:`\phi` is the polynomial function which returns the transformed input vector.
+    This transformed input vector is often of higher dimension, based on the idea that if it is not linear with
+    respect to the parameters in the current space, it might be in a higher dimensional space.
+    """
+
+    def __init__(self, inputs, outputs, degree=1, preprocessors=None, postprocessors=None):
+        """
+        Initialize the polynomial approximator.
+
+        Args:
+            inputs (State, Action, np.array, torch.Tensor): inputs of the inner models (instance of Action/State)
+            outputs (State, Action, np.array, torch.Tensor): outputs of the inner models (instance of Action/State)
+            degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply
+                `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs.
+            preprocessors (None, Processor, list of Processor): the inputs are first given to the preprocessors then
+                to the model.
+            postprocessors (None, Processor, list of Processor): the predicted outputs by the model are given to the
+                processors before being returned.
+        """
+        # create inner model
+        polynomial_fct = PolynomialFunction(degree=degree)
+        model = Polynomial(num_inputs=self._size(inputs), num_outputs=self._size(outputs),
+                           polynomial_fct=polynomial_fct)
+
+        # call parent class
+        super(PolynomialApproximator, self).__init__(inputs, outputs, model=model, preprocessors=preprocessors,
+                                                     postprocessors=postprocessors)

pyrobolearn/dynamics/polynomial.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+"""Provides the polynomial dynamic transition function approximators
+
+The polynomial dynamic model predicts using a polynomial model the next state given the current state and action.
+"""
+
+from pyrobolearn.approximators import PolynomialApproximator
+from pyrobolearn.dynamics.dynamic import ParametrizedDynamicModel
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "GNU GPLv3"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class PolynomialDynamicModel(ParametrizedDynamicModel):
+    r"""Polynomial Dynamic Model
+
+    The polynomial dynamic model predicts using a polynomial model the next state given the current state and action.
+    """
+
+    def __init__(self, state, action, next_state=None, distributions=None, degree=1, preprocessors=None,
+                 postprocessors=None):
+        """
+        Initialize the polynomial dynamic transition function / probability :math:`p(s_{t+1} | s_t, a_t)`.
+
+        Args:
+            state (State): state inputs.
+            action (Action): action inputs.
+            next_state (State, None): state outputs. If None, it will take the state inputs as the outputs.
+            distributions (torch.distributions.Distribution): distribution to use to sample the next state. If None,
+                it will be deterministic.
+            degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply
+                `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs.
+            preprocessors (Processor, list of Processor, None): pre-processors to be applied to the given input
+            postprocessors (Processor, list of Processor, None): post-processors to be applied to the output
+        """
+        if next_state is None:
+            next_state = state
+        model = PolynomialApproximator(inputs=[state, action], outputs=next_state, degree=degree,
+                                       preprocessors=preprocessors, postprocessors=postprocessors)
+        super(PolynomialDynamicModel, self).__init__(state, action, model=model, next_state=next_state,
+                                                     distributions=distributions)

pyrobolearn/models/basics/polynomial.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-"""Define the polynomial learning model.
+r"""Define the polynomial learning model.
 
 The polynomial model is a discriminative deterministic model given by: :math:`y = f(x) = W \phi(x)`, where
 :math:`\phi` is a function that returns a transformed input vector (possibly of higher dimension).
@@ -99,7 +99,7 @@ class Polynomial(object):
     """
 
     def __init__(self, num_inputs, num_outputs, polynomial_fct):
-        """
+        r"""
         Initialize the polynomial model: :math:`y = W \phi(x)`
 
         Args:

pyrobolearn/policies/polynomial.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""Provide the polynomial policy.
+
+The polynomial policy uses a polynomial parametric approximator to predict the action vector based on the state vector.
+"""
+
+from pyrobolearn.policies.policy import Policy
+from pyrobolearn.approximators import PolynomialApproximator
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "GNU GPLv3"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class PolynomialPolicy(Policy):
+    r"""Polynomial Policy
+
+    The polynomial policy uses a polynomial parametric approximator: :math:`y = W \phi(x)` where :math:`x` is the state
+    vector, and :math:`y` is the action vector, :math:`W` is the weight matrix, and :math:`\phi` is the polynomial
+    function which returns the transformed state vector.
+    """
+
+    def __init__(self, state, action, degree=1, rate=1, preprocessors=None, postprocessors=None, *args, **kwargs):
+        """
+        Initialize the Polynomial Policy.
+
+        Args:
+            action (Action): At each step, by calling `policy.act(state)`, the `action` is computed by the policy,
+                and can be given to the environment. As with the `state`, the type and size/shape of each inner
+                action can be inferred and could be used to automatically build a policy. The `action` connects the
+                policy with a controllable object (such as a robot) in the environment.
+            state (State): By giving the `state` to the policy, it can automatically infer the type and size/shape
+                of each inner state, and thus can be used to automatically build a policy. At each step, the `state`
+                is filled by the environment, and read by the policy. The `state` connects the policy with one or
+                several objects (including robots) in the environment. Note that some policies don't use any state
+                information.
+            degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply
+                `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs.
+            rate (int, float): rate (float) at which the policy operates if we are operating in real-time. If we are
+                stepping deterministically in the simulator, it represents the number of ticks (int) to sleep before
+                executing the model.
+            preprocessors (Processor, list of Processor, None): pre-processors to be applied to the given input
+            postprocessors (Processor, list of Processor, None): post-processors to be applied to the output
+            *args (list): list of arguments
+            **kwargs (dict): dictionary of arguments
+        """
+        model = PolynomialApproximator(state, action, degree=degree, preprocessors=preprocessors,
+                                       postprocessors=postprocessors)
+        super(PolynomialPolicy, self).__init__(state, action, model, rate=rate, *args, **kwargs)
+
+
+# Tests
+if __name__ == '__main__':
+    import copy
+    from pyrobolearn.states import FixedState
+    from pyrobolearn.actions import FixedAction
+
+    # check polynomial policy
+    policy = PolynomialPolicy(state=FixedState(range(4)), action=FixedAction(range(2)))
+    print(policy)
+
+    target = copy.deepcopy(policy)
+    print(target)

pyrobolearn/values/polynomial.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+"""Provides the polynomial value function approximator.
+"""
+
+import copy
+import torch
+
+from pyrobolearn.approximators import PolynomialApproximator
+from pyrobolearn.values.value import ParametrizedValue, ParametrizedQValue, ParametrizedQValueOutput
+
+
+__author__ = "Brian Delhaisse"
+__copyright__ = "Copyright 2018, PyRoboLearn"
+__credits__ = ["Brian Delhaisse"]
+__license__ = "GNU GPLv3"
+__version__ = "1.0.0"
+__maintainer__ = "Brian Delhaisse"
+__email__ = "briandelhaisse@gmail.com"
+__status__ = "Development"
+
+
+class PolynomialValue(ParametrizedValue):
+    r"""Polynomial State Value Function Approximator
+
+    State value function :math:`V_{\phi}(s)` approximated by a polynomial model, where :math:`\phi` represents
+    the parameters of that model.
+    """
+
+    def __init__(self, state, degree=1, preprocessors=None):
+        """
+        Initialize the polynomial state value function approximator.
+
+        Args:
+            state (State): input state.
+            degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply
+                `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs.
+            preprocessors ((list of) Processor): pre-processors to be applied on the input state before being fed to
+                the inner model / function approximator.
+        """
+        model = PolynomialApproximator(inputs=state, outputs=torch.Tensor([1]), degree=degree,
+                                       preprocessors=preprocessors)
+        super(PolynomialValue, self).__init__(state, model=model)
+
+    def __copy__(self):
+        """Return a shallow copy of the value approximator. This can be overridden in the child class."""
+        return self.__class__(state=self.state, preprocessors=self.model.preprocessors)
+
+    def __deepcopy__(self, memo={}):
+        """Return a deep copy of the value approximator. This can be overridden in the child class.
+
+        Args:
+            memo (dict): memo dictionary of objects already copied during the current copying pass
+        """
+        if self in memo:
+            return memo[self]
+        state = copy.deepcopy(self.state, memo)
+        preprocessors = [copy.deepcopy(preprocessor, memo) for preprocessor in self.model.preprocessors]
+        value = self.__class__(state=state, preprocessors=preprocessors)
+        memo[self] = value
+        return value
+
+
+class PolynomialQValue(ParametrizedQValue):
+    r"""Polynomial Q-value function approximator (which accepts as inputs the states and actions)
+
+    State-action value function :math:`Q_{\phi}(s, a)` approximated by a polynomial model, where :math:`\phi`
+    represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and actions
+    :math:`a`, and outputs the value :math:`Q(s,a)`. This can be used for continuous actions as well as discrete actions.
+    """
+
+    def __init__(self, state, action, degree=1, preprocessors=None):
+        """
+        Initialize the polynomial state-action value function approximator.
+
+        Args:
+            state (State): input state.
+            action (Action): input action.
+            degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply
+                `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs.
+            preprocessors ((list of) Processor): pre-processors to be applied on the input state before being fed to
+                the inner model / function approximator.
+        """
+        model = PolynomialApproximator(inputs=[state, action], outputs=torch.Tensor([1]), degree=degree,
+                                       preprocessors=preprocessors)
+        super(PolynomialQValue, self).__init__(state, action, model=model)
+
+    def __copy__(self):
+        """Return a shallow copy of the value approximator. This can be overridden in the child class."""
+        return self.__class__(state=self.state, action=self.action, preprocessors=self.model.preprocessors)
+
+    def __deepcopy__(self, memo={}):
+        """Return a deep copy of the value approximator. This can be overridden in the child class.
+
+        Args:
+            memo (dict): memo dictionary of objects already copied during the current copying pass
+        """
+        if self in memo:
+            return memo[self]
+
+        state = copy.deepcopy(self.state, memo)
+        action = copy.deepcopy(self.action, memo)
+        preprocessors = [copy.deepcopy(preprocessor, memo) for preprocessor in self.model.preprocessors]
+        value = self.__class__(state=state, action=action, preprocessors=preprocessors)
+        memo[self] = value
+        return value
+
+
+class PolynomialQValueOutput(ParametrizedQValueOutput):
+    r"""Polynomial Q-value function approximator (which accepts as inputs the states and outputs a Q-value for each
+    discrete action)
+
+    State-action value function :math:`Q_{\phi}(s, a)` approximated by a polynomial model, where :math:`\phi`
+    represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and outputs the
+    value :math:`Q(s,a)` for each discrete action. This can NOT be used with continuous actions.
+    """
+
+    def __init__(self, state, action, degree=1, preprocessors=None):
+        """
+        Initialize the polynomial state-action value function approximator.
+
+        Args:
+            state (State): input state.
+            action (Action): output action.
+            degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply
+                `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs.
+            preprocessors ((list of) Processor): pre-processors to be applied on the input state before being fed to
+                the inner model / function approximator.
+        """
+        model = PolynomialApproximator(inputs=state, outputs=action, degree=degree, preprocessors=preprocessors)
+        super(PolynomialQValueOutput, self).__init__(state, action, model=model)
+
+    def __copy__(self):
+        """Return a shallow copy of the value approximator. This can be overridden in the child class."""
+        return self.__class__(state=self.state, action=self.action, preprocessors=self.model.preprocessors)
+
+    def __deepcopy__(self, memo={}):
+        """Return a deep copy of the value approximator. This can be overridden in the child class.
+
+        Args:
+            memo (dict): memo dictionary of objects already copied during the current copying pass
+        """
+        if self in memo:
+            return memo[self]
+
+        state = copy.deepcopy(self.state, memo)
+        action = copy.deepcopy(self.action, memo)
+        preprocessors = [copy.deepcopy(preprocessor, memo) for preprocessor in self.model.preprocessors]
+        value = self.__class__(state=state, action=action, preprocessors=preprocessors)
+        memo[self] = value
+        return value