diff --git a/pyrobolearn/approximators/polynomial.py b/pyrobolearn/approximators/polynomial.py new file mode 100644 index 0000000..7f5952a --- /dev/null +++ b/pyrobolearn/approximators/polynomial.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +"""Define linear function approximator. + +Dependencies: +- `pyrobolearn.models` +- `pyrobolearn.states` +- `pyrobolearn.actions` +""" + +from pyrobolearn.approximators.approximator import Approximator +from pyrobolearn.models.basics.polynomial import Polynomial, PolynomialFunction + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "GNU GPLv3" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class PolynomialApproximator(Approximator): + r"""Polynomial Function Approximator + + The polynomial function approximator is a discriminative deterministic model expressed mathematically as + :math:`y = f(x) = W \phi(x)`, where :math:`x` is the input vector, :math:`y` is the output vector, :math:`W` + is the weight matrix, and :math:`\phi` is the polynomial function which returns the transformed input vector. + This transformed input vector is often of higher dimension, based on the idea that if it is not linear with + respect to the parameters in the current space, it might be in a higher dimensional space. + """ + + def __init__(self, inputs, outputs, degree=1, preprocessors=None, postprocessors=None): + """ + Initialize the polynomial approximator. + + Args: + inputs (State, Action, np.array, torch.Tensor): inputs of the inner models (instance of Action/State) + outputs (State, Action, np.array, torch.Tensor): outputs of the inner models (instance of Action/State) + degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply + `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs. + preprocessors (None, Processor, list of Processor): the inputs are first given to the preprocessors then + to the model. + postprocessors (None, Processor, list of Processor): the predicted outputs by the model are given to the + processors before being returned. + """ + # create inner model + polynomial_fct = PolynomialFunction(degree=degree) + model = Polynomial(num_inputs=self._size(inputs), num_outputs=self._size(outputs), + polynomial_fct=polynomial_fct) + + # call parent class + super(PolynomialApproximator, self).__init__(inputs, outputs, model=model, preprocessors=preprocessors, + postprocessors=postprocessors) diff --git a/pyrobolearn/dynamics/polynomial.py b/pyrobolearn/dynamics/polynomial.py new file mode 100644 index 0000000..ad35b66 --- /dev/null +++ b/pyrobolearn/dynamics/polynomial.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +"""Provides the polynomial dynamic transition function approximators + +The polynomial dynamic model predicts using a polynomial model the next state given the current state and action. +""" + +from pyrobolearn.approximators import PolynomialApproximator +from pyrobolearn.dynamics.dynamic import ParametrizedDynamicModel + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "GNU GPLv3" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class PolynomialDynamicModel(ParametrizedDynamicModel): + r"""Polynomial Dynamic Model + + The polynomial dynamic model predicts using a polynomial model the next state given the current state and action. + """ + + def __init__(self, state, action, next_state=None, distributions=None, degree=1, preprocessors=None, + postprocessors=None): + """ + Initialize the polynomial dynamic transition function / probability :math:`p(s_{t+1} | s_t, a_t)`. + + Args: + state (State): state inputs. + action (Action): action inputs. + next_state (State, None): state outputs. If None, it will take the state inputs as the outputs. + distributions (torch.distributions.Distribution): distribution to use to sample the next state. If None, + it will be deterministic. + degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply + `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs. + preprocessors (Processor, list of Processor, None): pre-processors to be applied to the given input + postprocessors (Processor, list of Processor, None): post-processors to be applied to the output + """ + if next_state is None: + next_state = state + model = PolynomialApproximator(inputs=[state, action], outputs=next_state, degree=degree, + preprocessors=preprocessors, postprocessors=postprocessors) + super(PolynomialDynamicModel, self).__init__(state, action, model=model, next_state=next_state, + distributions=distributions) diff --git a/pyrobolearn/models/basics/polynomial.py b/pyrobolearn/models/basics/polynomial.py index 9f4cf68..8cd608e 100644 --- a/pyrobolearn/models/basics/polynomial.py +++ b/pyrobolearn/models/basics/polynomial.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Define the polynomial learning model. +r"""Define the polynomial learning model. The polynomial model is a discriminative deterministic model given by: :math:`y = f(x) = W \phi(x)`, where :math:`\phi` is a function that returns a transformed input vector (possibly of higher dimension). @@ -99,7 +99,7 @@ class Polynomial(object): """ def __init__(self, num_inputs, num_outputs, polynomial_fct): - """ + r""" Initialize the polynomial model: :math:`y = W \phi(x)` Args: diff --git a/pyrobolearn/policies/polynomial.py b/pyrobolearn/policies/polynomial.py new file mode 100644 index 0000000..2879200 --- /dev/null +++ b/pyrobolearn/policies/polynomial.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +"""Provide the polynomial policy. + +The polynomial policy uses a polynomial parametric approximator to predict the action vector based on the state vector. +""" + +from pyrobolearn.policies.policy import Policy +from pyrobolearn.approximators import PolynomialApproximator + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "GNU GPLv3" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class PolynomialPolicy(Policy): + r"""Polynomial Policy + + The polynomial policy uses a polynomial parametric approximator: :math:`y = W \phi(x)` where :math:`x` is the state + vector, and :math:`y` is the action vector, :math:`W` is the weight matrix, and :math:`\phi` is the polynomial + function which returns the transformed state vector. + """ + + def __init__(self, state, action, degree=1, rate=1, preprocessors=None, postprocessors=None, *args, **kwargs): + """ + Initialize the Polynomial Policy. + + Args: + action (Action): At each step, by calling `policy.act(state)`, the `action` is computed by the policy, + and can be given to the environment. As with the `state`, the type and size/shape of each inner + action can be inferred and could be used to automatically build a policy. The `action` connects the + policy with a controllable object (such as a robot) in the environment. + state (State): By giving the `state` to the policy, it can automatically infer the type and size/shape + of each inner state, and thus can be used to automatically build a policy. At each step, the `state` + is filled by the environment, and read by the policy. The `state` connects the policy with one or + several objects (including robots) in the environment. Note that some policies don't use any state + information. + degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply + `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs. + rate (int, float): rate (float) at which the policy operates if we are operating in real-time. If we are + stepping deterministically in the simulator, it represents the number of ticks (int) to sleep before + executing the model. + preprocessors (Processor, list of Processor, None): pre-processors to be applied to the given input + postprocessors (Processor, list of Processor, None): post-processors to be applied to the output + *args (list): list of arguments + **kwargs (dict): dictionary of arguments + """ + model = PolynomialApproximator(state, action, degree=degree, preprocessors=preprocessors, + postprocessors=postprocessors) + super(PolynomialPolicy, self).__init__(state, action, model, rate=rate, *args, **kwargs) + + +# Tests +if __name__ == '__main__': + import copy + from pyrobolearn.states import FixedState + from pyrobolearn.actions import FixedAction + + # check polynomial policy + policy = PolynomialPolicy(state=FixedState(range(4)), action=FixedAction(range(2))) + print(policy) + + target = copy.deepcopy(policy) + print(target) diff --git a/pyrobolearn/values/polynomial.py b/pyrobolearn/values/polynomial.py new file mode 100644 index 0000000..b65769d --- /dev/null +++ b/pyrobolearn/values/polynomial.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +"""Provides the polynomial value function approximator. +""" + +import copy +import torch + +from pyrobolearn.approximators import PolynomialApproximator +from pyrobolearn.values.value import ParametrizedValue, ParametrizedQValue, ParametrizedQValueOutput + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "GNU GPLv3" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class PolynomialValue(ParametrizedValue): + r"""Polynomial State Value Function Approximator + + State value function :math:`V_{\phi}(s)` approximated by a polynomial model, where :math:`\phi` represents + the parameters of that model. + """ + + def __init__(self, state, degree=1, preprocessors=None): + """ + Initialize the polynomial state value function approximator. + + Args: + state (State): input state. + degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply + `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs. + preprocessors ((list of) Processor): pre-processors to be applied on the input state before being fed to + the inner model / function approximator. + """ + model = PolynomialApproximator(inputs=state, outputs=torch.Tensor([1]), degree=degree, + preprocessors=preprocessors) + super(PolynomialValue, self).__init__(state, model=model) + + def __copy__(self): + """Return a shallow copy of the value approximator. This can be overridden in the child class.""" + return self.__class__(state=self.state, preprocessors=self.model.preprocessors) + + def __deepcopy__(self, memo={}): + """Return a deep copy of the value approximator. This can be overridden in the child class. + + Args: + memo (dict): memo dictionary of objects already copied during the current copying pass + """ + if self in memo: + return memo[self] + state = copy.deepcopy(self.state, memo) + preprocessors = [copy.deepcopy(preprocessor, memo) for preprocessor in self.model.preprocessors] + value = self.__class__(state=state, preprocessors=preprocessors) + memo[self] = value + return value + + +class PolynomialQValue(ParametrizedQValue): + r"""Polynomial Q-value function approximator (which accepts as inputs the states and actions) + + State-action value function :math:`Q_{\phi}(s, a)` approximated by a polynomial model, where :math:`\phi` + represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and actions + :math:`a`, and outputs the value :math:`Q(s,a)`. This can be used for continuous actions as well as discrete actions. + """ + + def __init__(self, state, action, degree=1, preprocessors=None): + """ + Initialize the polynomial state-action value function approximator. + + Args: + state (State): input state. + action (Action): input action. + degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply + `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs. + preprocessors ((list of) Processor): pre-processors to be applied on the input state before being fed to + the inner model / function approximator. + """ + model = PolynomialApproximator(inputs=[state, action], outputs=torch.Tensor([1]), degree=degree, + preprocessors=preprocessors) + super(PolynomialQValue, self).__init__(state, action, model=model) + + def __copy__(self): + """Return a shallow copy of the value approximator. This can be overridden in the child class.""" + return self.__class__(state=self.state, action=self.action, preprocessors=self.model.preprocessors) + + def __deepcopy__(self, memo={}): + """Return a deep copy of the value approximator. This can be overridden in the child class. + + Args: + memo (dict): memo dictionary of objects already copied during the current copying pass + """ + if self in memo: + return memo[self] + + state = copy.deepcopy(self.state, memo) + action = copy.deepcopy(self.action, memo) + preprocessors = [copy.deepcopy(preprocessor, memo) for preprocessor in self.model.preprocessors] + value = self.__class__(state=state, action=action, preprocessors=preprocessors) + memo[self] = value + return value + + +class PolynomialQValueOutput(ParametrizedQValueOutput): + r"""Polynomial Q-value function approximator (which accepts as inputs the states and outputs a Q-value for each + discrete action) + + State-action value function :math:`Q_{\phi}(s, a)` approximated by a polynomial model, where :math:`\phi` + represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and outputs the + value :math:`Q(s,a)` for each discrete action. This can NOT be used with continuous actions. + """ + + def __init__(self, state, action, degree=1, preprocessors=None): + """ + Initialize the polynomial state-action value function approximator. + + Args: + state (State): input state. + action (Action): output action. + degree (int, list of int, np.array[D]): degree(s) of the polynomial. Setting `degree=3`, will apply + `[1,x,x^2,x^3]` to the inputs, while setting `degree=[1,3]` will apply `[x,x^3]` to the inputs. + preprocessors ((list of) Processor): pre-processors to be applied on the input state before being fed to + the inner model / function approximator. + """ + model = PolynomialApproximator(inputs=state, outputs=action, degree=degree, preprocessors=preprocessors) + super(PolynomialQValueOutput, self).__init__(state, action, model=model) + + def __copy__(self): + """Return a shallow copy of the value approximator. This can be overridden in the child class.""" + return self.__class__(state=self.state, action=self.action, preprocessors=self.model.preprocessors) + + def __deepcopy__(self, memo={}): + """Return a deep copy of the value approximator. This can be overridden in the child class. + + Args: + memo (dict): memo dictionary of objects already copied during the current copying pass + """ + if self in memo: + return memo[self] + + state = copy.deepcopy(self.state, memo) + action = copy.deepcopy(self.action, memo) + preprocessors = [copy.deepcopy(preprocessor, memo) for preprocessor in self.model.preprocessors] + value = self.__class__(state=state, action=action, preprocessors=preprocessors) + memo[self] = value + return value