From d0073aaaa501de62755ca615b7e0cd742d6d6098 Mon Sep 17 00:00:00 2001 From: Brian Delhaisse Date: Fri, 12 Apr 2019 02:51:23 +0200 Subject: [PATCH] refactor values: shorten names and update methods --- pyrobolearn/values/basic_value.py | 21 +-- pyrobolearn/values/nn_value.py | 49 +++---- pyrobolearn/values/value.py | 230 ++++++++++++++++++++++++------ 3 files changed, 226 insertions(+), 74 deletions(-) diff --git a/pyrobolearn/values/basic_value.py b/pyrobolearn/values/basic_value.py index 3093209..266c132 100644 --- a/pyrobolearn/values/basic_value.py +++ b/pyrobolearn/values/basic_value.py @@ -7,8 +7,8 @@ import torch # from pyrobolearn.models import Linear from pyrobolearn.approximators import LinearApproximator -from pyrobolearn.values.value import ValueApproximator, ParametrizedValue, ParametrizedStateActionValue, \ - ParametrizedStateOutputActionValue +from pyrobolearn.values.value import ValueApproximator, ParametrizedValue, ParametrizedQValue, \ + ParametrizedQValueOutput __author__ = "Brian Delhaisse" @@ -33,7 +33,7 @@ class ValueTable(ValueApproximator): super(ValueTable, self).__init__(state) -class LinearStateValue(ParametrizedValue): +class LinearValue(ParametrizedValue): r"""Linear State Value Function Approximator State value function :math:`V_{\phi}(s)` approximated by a linear model, where :math:`\phi` represents @@ -50,11 +50,11 @@ class LinearStateValue(ParametrizedValue): the inner model / function approximator. """ model = LinearApproximator(inputs=state, outputs=torch.Tensor([1]), preprocessors=preprocessors) - super(LinearStateValue, self).__init__(state, model=model) + super(LinearValue, self).__init__(state, model=model) -class LinearStateInputActionValue(ParametrizedStateActionValue): - r"""Linear state - input action value function approximator +class LinearQValue(ParametrizedQValue): + r"""Linear Q-value function approximator (which accepts as inputs the states and actions) State-action value function :math:`Q_{\phi}(s, a)` approximated by a linear model, where :math:`\phi` represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and actions :math:`a`, @@ -72,11 +72,12 @@ class LinearStateInputActionValue(ParametrizedStateActionValue): the inner model / function approximator. """ model = LinearApproximator(inputs=[state, action], outputs=torch.Tensor([1]), preprocessors=preprocessors) - super(LinearStateInputActionValue, self).__init__(state, action, model=model) + super(LinearQValue, self).__init__(state, action, model=model) -class LinearStateOutputActionValue(ParametrizedStateOutputActionValue): - r"""Linear state - output action value function approximator +class LinearQValueOutput(ParametrizedQValueOutput): + r"""Linear Q-value function approximator (which accepts as inputs the states and outputs a Q-value for each + discrete action) State-action value function :math:`Q_{\phi}(s, a)` approximated by a linear model, where :math:`\phi` represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and outputs the value @@ -94,4 +95,4 @@ class LinearStateOutputActionValue(ParametrizedStateOutputActionValue): the inner model / function approximator. """ model = LinearApproximator(inputs=state, outputs=action, preprocessors=preprocessors) - super(LinearStateOutputActionValue, self).__init__(state, action, model=model) + super(LinearQValueOutput, self).__init__(state, action, model=model) diff --git a/pyrobolearn/values/nn_value.py b/pyrobolearn/values/nn_value.py index 90e41bf..01b4f86 100644 --- a/pyrobolearn/values/nn_value.py +++ b/pyrobolearn/values/nn_value.py @@ -9,7 +9,7 @@ import torch from pyrobolearn.models import NN from pyrobolearn.approximators import NNApproximator, MLPApproximator -from pyrobolearn.values.value import ParametrizedValue, ParametrizedStateActionValue, ParametrizedStateOutputActionValue +from pyrobolearn.values.value import ParametrizedValue, ParametrizedQValue, ParametrizedQValueOutput __author__ = "Brian Delhaisse" @@ -47,8 +47,8 @@ __status__ = "Development" # self.model = model -class StateValueNetwork(ParametrizedValue): - r"""Stave Value Network +class ValueNetwork(ParametrizedValue): + r"""State Value Network This is defined by :math:`V_{\psi}(s_t)` where :math:`\psi` represents the network parameters. """ @@ -61,11 +61,11 @@ class StateValueNetwork(ParametrizedValue): state (State): input state. model (NN, NNApproximator): Neural Network model / approximator. """ - super(StateValueNetwork, self).__init__(state, model) + super(ValueNetwork, self).__init__(state, model) -class StateInputActionValueNetwork(ParametrizedStateActionValue): - r"""State Input Action Value Network +class QValueNetwork(ParametrizedQValue): + r"""Q-Value Network (which accepts as inputs the states and actions) State-action value function :math:`Q_{\phi}(s, a)` approximated by a neural network, where :math:`\phi` represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and actions :math:`a`, @@ -81,11 +81,11 @@ class StateInputActionValueNetwork(ParametrizedStateActionValue): action (Action): input action. model (NN, NNApproximator): Neural Network model / approximator. """ - super(StateInputActionValueNetwork, self).__init__(state, action, model) + super(QValueNetwork, self).__init__(state, action, model) -class StateOutputActionValueNetwork(ParametrizedStateOutputActionValue): - r"""State Output Action Value Network +class QValueOutputNetwork(ParametrizedQValueOutput): + r"""Q-Value Output Network (which accepts as inputs the states and outputs a Q-value for each discrete action) State-action value function :math:`Q_{\phi}(s, a)` approximated by a neural network, where :math:`\phi` represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and outputs the value @@ -101,10 +101,10 @@ class StateOutputActionValueNetwork(ParametrizedStateOutputActionValue): action (Action): output action. model (NN, NNApproximator): Neural Network model / approximator. """ - super(StateOutputActionValueNetwork, self).__init__(state, action, model) + super(QValueOutputNetwork, self).__init__(state, action, model) -class MLPStateValue(ParametrizedValue): # StateValueNetwork): +class MLPValue(ValueNetwork): r"""Multi-Layer Perceptron (MLP) State Value Function Approximator This is defined by :math:`V_{\psi}(s_t)` where the function :math:`V` is approximated by a multilayer perceptron. @@ -128,14 +128,14 @@ class MLPStateValue(ParametrizedValue): # StateValueNetwork): the inner model / function approximator. """ output = torch.Tensor([1.]) # torch.Tensor([[1.]]) - model = MLPApproximator(state, output, hidden_units=hidden_units, activation_fct=activation_fct, - last_activation_fct=last_activation_fct, dropout_prob=dropout_prob, + model = MLPApproximator(state, output, hidden_units=hidden_units, activation=activation_fct, + last_activation=last_activation_fct, dropout=dropout_prob, preprocessors=preprocessors) - super(MLPStateValue, self).__init__(state, model) + super(MLPValue, self).__init__(state, model) -class MLPStateInputActionValue(ParametrizedStateActionValue): - r"""MLP state - input action value function approximator +class MLPQValue(QValueNetwork): + r"""MLP Q-value function approximator (which accepts as inputs the states and actions) State-action value function :math:`Q_{\phi}(s, a)` approximated by a MLP model, where :math:`\phi` represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and actions :math:`a`, @@ -161,13 +161,14 @@ class MLPStateInputActionValue(ParametrizedStateActionValue): the inner model / function approximator. """ model = MLPApproximator(inputs=[state, action], outputs=torch.Tensor([1]), hidden_units=hidden_units, - activation_fct=activation_fct, last_activation_fct=last_activation_fct, - dropout_prob=dropout_prob, preprocessors=preprocessors) - super(MLPStateInputActionValue, self).__init__(state, action, model=model) + activation=activation_fct, last_activation=last_activation_fct, + dropout=dropout_prob, preprocessors=preprocessors) + super(MLPQValue, self).__init__(state, action, model=model) -class MLPStateOutputActionValue(ParametrizedStateOutputActionValue): - r"""MLP state - output action value function approximator +class MLPQValueOutput(ParametrizedQValueOutput): + r"""MLP Q-value function approximator (which accepts as inputs the states and outputs a Q-value for each discrete + action) State-action value function :math:`Q_{\phi}(s, a)` approximated by a MLP model, where :math:`\phi` represents the parameters of that model. This approximator accepts as inputs the states :math:`s` and outputs the value @@ -193,6 +194,6 @@ class MLPStateOutputActionValue(ParametrizedStateOutputActionValue): the inner model / function approximator. """ model = MLPApproximator(inputs=state, outputs=action, hidden_units=hidden_units, - activation_fct=activation_fct, last_activation_fct=last_activation_fct, - dropout_prob=dropout_prob, preprocessors=preprocessors) - super(MLPStateOutputActionValue, self).__init__(state, action, model=model) + activation=activation_fct, last_activation=last_activation_fct, + dropout=dropout_prob, preprocessors=preprocessors) + super(MLPQValueOutput, self).__init__(state, action, model=model) diff --git a/pyrobolearn/values/value.py b/pyrobolearn/values/value.py index 91dce89..05a27d0 100644 --- a/pyrobolearn/values/value.py +++ b/pyrobolearn/values/value.py @@ -18,6 +18,7 @@ from pyrobolearn.states import State from pyrobolearn.actions import Action from pyrobolearn.approximators import Approximator + __author__ = "Brian Delhaisse" __copyright__ = "Copyright 2018, PyRoboLearn" __credits__ = ["Brian Delhaisse"] @@ -43,6 +44,8 @@ class ValueApproximator(object): starting at state :math:`s`. 2. (state-)action value function denoted by :math:`Q(s,a)`, which represents the expected reward when starting at state :math:`s`, and performing action :math:`a`. + + In both cases, they use the state. This class computes :math:`V(s)`. """ __metaclass__ = ABCMeta @@ -78,36 +81,40 @@ class ValueApproximator(object): # Methods # ########### - def compute(self, *args, **kwargs): + def evaluate(self, *args, **kwargs): """Predict the value.""" pass def __call__(self, *args, **kwargs): """Predict the value.""" - return self.compute(*args, **kwargs) + return self.evaluate(*args, **kwargs) -class StateValueApproximator(ValueApproximator): - r"""State Value Approximator - - Compute :math:`V(s)`. - """ - __metaclass__ = ABCMeta - - def __init__(self, state): - super(StateValueApproximator, self).__init__(state) - - -class ActionValueApproximator(ValueApproximator): - r"""Action Value Approximator +class QValueApproximator(ValueApproximator): + r"""Q-Value Approximator Compute :math:`Q(s,a)`. """ __metaclass__ = ABCMeta - def __init__(self, state, actions): - super(ActionValueApproximator, self).__init__(state) - self.actions = actions + def __init__(self, state, action): + # super(QValueApproximator, self).__init__(state) + ValueApproximator.__init__(self, state) + self.action = action + + @property + def action(self): + """Return the action instance.""" + return self._action + + @action.setter + def action(self, action): + """Set the action input.""" + if isinstance(action, (int, float)): + action = np.array([action]) + elif not isinstance(action, (Action, torch.Tensor, np.ndarray)): + raise TypeError("Expecting the action to be an Action, torch.Tensor, or np.ndarray.") + self._action = action class ParametrizedValue(ValueApproximator): @@ -124,6 +131,7 @@ class ParametrizedValue(ValueApproximator): state (State): input state. model (Approximator): value function approximator. """ + # ValueApproximator.__init__(self, state) super(ParametrizedValue, self).__init__(state) self.model = model @@ -238,7 +246,7 @@ class ParametrizedValue(ValueApproximator): """ self.model.set_vectorized_parameters(vector=vector) - def compute(self, state=None, to_numpy=True): + def evaluate(self, state=None, to_numpy=False): """Compute the output of the value function. Args: @@ -246,16 +254,30 @@ class ParametrizedValue(ValueApproximator): the data from the inputs that were given at the initialization. to_numpy (bool): If True, it will convert the data (torch.Tensors) to numpy arrays. """ + # if no input is given, take the provided inputs at the beginning + if state is None: + state = self.state + + # if the input is an instance of State, get the inner merged data. + if isinstance(state, State): + state = state.merged_data + if len(state) == 1: + state = state[0] + self.value = self.model.predict(state, to_numpy=to_numpy, return_logits=True, set_output_data=False) return self.value - def __call__(self, state=None, to_numpy=True): + def __call__(self, state=None, to_numpy=False): """Predict the value.""" - return self.compute(state=state, to_numpy=to_numpy) + return self.evaluate(state=state, to_numpy=to_numpy) -class ParametrizedStateActionValue(ParametrizedValue): - r"""Parametrized State Action Value Function Approximator +# alias +Value = ParametrizedValue + + +class ParametrizedQValue(QValueApproximator): # ParametrizedValue, QValueApproximator): + r"""Parametrized Q-Value Function Approximator (which accepts as inputs the states and actions) This value function approximator predicts the state-action value :math:`Q_{\phi}(s,a)`. Two different kind of state-action value function approximators can be defined: @@ -276,30 +298,155 @@ class ParametrizedStateActionValue(ParametrizedValue): action (Action): input / output action. model (Approximator): value function approximator. """ - super(ParametrizedStateActionValue, self).__init__(state, model) - self.action = action + # ParametrizedValue.__init__(self, state, model) + # QValueApproximator.__init__(self, state, action) + super(ParametrizedQValue, self).__init__(state, action) + self.model = model ############## # Properties # ############## @property - def action(self): - """Return the action instance.""" - return self._action + def model(self): + """Return the model instance.""" + return self._model - @action.setter - def action(self, action): - """Set the action input.""" - if isinstance(action, (int, float)): - action = np.array([action]) - elif not isinstance(action, (Action, torch.Tensor, np.ndarray)): - raise TypeError("Expecting the action to be an Action, torch.Tensor, or np.ndarray.") - self._action = action + @model.setter + def model(self, model): + """Set the model / approximator instance.""" + if not isinstance(model, Approximator): + raise TypeError("Expecting the model to be an instance of `Approximator`, instead got: " + "{}".format(type(model))) + self._model = model + + @property + def input_size(self): + """Return the policy input size.""" + return self.model.input_size + + @property + def output_size(self): + """Return the policy output size.""" + return self.model.output_size + + @property + def input_shape(self): + """Return the policy input shape.""" + return self.model.input_shape + + @property + def output_shape(self): + """Return the policy output shape.""" + return self.model.output_shape + + @property + def input_dim(self): + """Return the input dimension of the policy; i.e. len(input_shape).""" + return self.model.input_dim + + @property + def output_dim(self): + """Return the output dimension of the policy; i.e. len(output_shape).""" + return self.model.output_dim + + @property + def num_parameters(self): + """Return the total number of parameters""" + return self.model.num_parameters + + ########### + # Methods # + ########### + + def parameters(self): + """ + Return an iterator over the learning model parameters. + """ + return self.model.parameters() + + def named_parameters(self): + """ + Return an iterator over the learning model parameters; yielding both the name and the parameter itself. + """ + return self.model.named_parameters() + + def list_parameters(self): + """ + Return the learning model parameters. + """ + return self.model.list_parameters() + + def hyperparameters(self): + """ + Return an iterator over the learning model hyper-parameters. + """ + return self.model.hyperparameters() + + def named_hyperparameters(self): + """ + Return an iterator over the learning model hyper-parameters; yielding both the name and the hyper-parameter + itself. + """ + return self.model.named_hyperparameters() + + def list_hyperparameters(self): + """ + Return the learning model hyper-parameters + """ + return self.model.list_hyperparameters() + + def get_vectorized_parameters(self, to_numpy=True): + """ + Get the parameters in a vectorized form. + + Args: + to_numpy (bool): if True, it will convert the 1D parameter vector into a numpy array. + """ + return self.model.get_vectorized_parameters(to_numpy=to_numpy) + + def set_vectorized_parameters(self, vector): + """ + Set the vectorized parameters. + + Args: + np.array, torch.Tensor: 1D parameter vector. + """ + self.model.set_vectorized_parameters(vector=vector) + + def evaluate(self, state=None, to_numpy=False): + """Compute the output of the value function. + + Args: + state (None, State, (list of) np.array, (list of) torch.Tensor): state input data. If None, it will get + the data from the inputs that were given at the initialization. + to_numpy (bool): If True, it will convert the data (torch.Tensors) to numpy arrays. + """ + # if no input is given, take the provided inputs at the beginning + if state is None: + state = self.state + + # if the input is an instance of State, get the inner merged data. + if isinstance(state, State): + state = state.merged_data + if len(state) == 1: + state = state[0] + + self.value = self.model.predict(state, to_numpy=to_numpy, return_logits=True, set_output_data=False) + return self.value + + def __call__(self, state=None, to_numpy=False): + """Predict the value.""" + return self.evaluate(state=state, to_numpy=to_numpy) -class ParametrizedStateOutputActionValue(ParametrizedValue): - r"""Parametrized State Output Action Value Function Approximator +# alias +QValue = ParametrizedQValue + + +class ParametrizedQValueOutput(ParametrizedQValue): + r"""Parametrized Q-Value Function Approximator (which accepts as inputs the states and outputs a Q-value for each + discrete action) This value function approximator predicts the state-action value :math:`Q_{\phi}(s,a)` for each discrete action. """ @@ -313,8 +460,7 @@ class ParametrizedStateOutputActionValue(ParametrizedValue): action (Action): output DISCRETE state. model (Approximator): value function approximator. """ - super(ParametrizedStateOutputActionValue, self).__init__(state, model) - self.action = action + super(ParametrizedQValueOutput, self).__init__(state, action, model) ############## # Properties # @@ -336,3 +482,7 @@ class ParametrizedStateOutputActionValue(ParametrizedValue): elif not isinstance(action, (torch.Tensor, np.ndarray)): raise TypeError("Expecting the action to be an int, float, Action, torch.Tensor, or np.ndarray.") self._action = action + + +# alias +QValueOutput = ParametrizedQValueOutput