mirror of
https://github.com/wassname/pyrobolearn.git
synced 2026-09-10 12:21:16 +08:00
refactor values: shorten names and update methods
This commit is contained in:
Brian Delhaisse
@@ -7,8 +7,8 @@ import torch
|
||||
|
||||
# from pyrobolearn.models import Linear
|
||||
from pyrobolearn.approximators import LinearApproximator
|
||||
from pyrobolearn.values.value import ValueApproximator, ParametrizedValue, ParametrizedStateActionValue, \
|
||||
ParametrizedStateOutputActionValue
|
||||
from pyrobolearn.values.value import ValueApproximator, ParametrizedValue, ParametrizedQValue, \
|
||||
ParametrizedQValueOutput
|
||||
|
||||
|
||||
__author__ = "Brian Delhaisse"
|
||||
@@ -33,7 +33,7 @@ class ValueTable(ValueApproximator):
|
||||
super(ValueTable, self).__init__(state)
|
||||
|
||||
|
||||
class LinearStateValue(ParametrizedValue):
|
||||
class LinearValue(ParametrizedValue):
|
||||
r"""Linear State Value Function Approximator
|
||||
|
||||
State value function :math:`V_{\phi}(s)` approximated by a linear model, where :math:`\phi` represents
|
||||
@@ -50,11 +50,11 @@ class LinearStateValue(ParametrizedValue):
|
||||
the inner model / function approximator.
|
||||
"""
|
||||
model = LinearApproximator(inputs=state, outputs=torch.Tensor([1]), preprocessors=preprocessors)
|
||||
super(LinearStateValue, self).__init__(state, model=model)
|
||||
super(LinearValue, self).__init__(state, model=model)
|
||||
|
||||
|
||||
class LinearStateInputActionValue(ParametrizedStateActionValue):
|
||||
r"""Linear state - input action value function approximator
|
||||
class LinearQValue(ParametrizedQValue):
|
||||
r"""Linear Q-value function approximator (which accepts as inputs the states and actions)
|
||||
|
||||
State-action value function :math:`Q_{\phi}(s, a)` approximated by a linear model, where :math:`\phi` represents
|
||||
the parameters of that model. This approximator accepts as inputs the states :math:`s` and actions :math:`a`,
|
||||
@@ -72,11 +72,12 @@ class LinearStateInputActionValue(ParametrizedStateActionValue):
|
||||
the inner model / function approximator.
|
||||
"""
|
||||
model = LinearApproximator(inputs=[state, action], outputs=torch.Tensor([1]), preprocessors=preprocessors)
|
||||
super(LinearStateInputActionValue, self).__init__(state, action, model=model)
|
||||
super(LinearQValue, self).__init__(state, action, model=model)
|
||||
|
||||
|
||||
class LinearStateOutputActionValue(ParametrizedStateOutputActionValue):
|
||||
r"""Linear state - output action value function approximator
|
||||
class LinearQValueOutput(ParametrizedQValueOutput):
|
||||
r"""Linear Q-value function approximator (which accepts as inputs the states and outputs a Q-value for each
|
||||
discrete action)
|
||||
|
||||
State-action value function :math:`Q_{\phi}(s, a)` approximated by a linear model, where :math:`\phi` represents
|
||||
the parameters of that model. This approximator accepts as inputs the states :math:`s` and outputs the value
|
||||
@@ -94,4 +95,4 @@ class LinearStateOutputActionValue(ParametrizedStateOutputActionValue):
|
||||
the inner model / function approximator.
|
||||
"""
|
||||
model = LinearApproximator(inputs=state, outputs=action, preprocessors=preprocessors)
|
||||
super(LinearStateOutputActionValue, self).__init__(state, action, model=model)
|
||||
super(LinearQValueOutput, self).__init__(state, action, model=model)
|
||||
|
||||
@@ -9,7 +9,7 @@ import torch
|
||||
|
||||
from pyrobolearn.models import NN
|
||||
from pyrobolearn.approximators import NNApproximator, MLPApproximator
|
||||
from pyrobolearn.values.value import ParametrizedValue, ParametrizedStateActionValue, ParametrizedStateOutputActionValue
|
||||
from pyrobolearn.values.value import ParametrizedValue, ParametrizedQValue, ParametrizedQValueOutput
|
||||
|
||||
|
||||
__author__ = "Brian Delhaisse"
|
||||
@@ -47,8 +47,8 @@ __status__ = "Development"
|
||||
# self.model = model
|
||||
|
||||
|
||||
class StateValueNetwork(ParametrizedValue):
|
||||
r"""Stave Value Network
|
||||
class ValueNetwork(ParametrizedValue):
|
||||
r"""State Value Network
|
||||
|
||||
This is defined by :math:`V_{\psi}(s_t)` where :math:`\psi` represents the network parameters.
|
||||
"""
|
||||
@@ -61,11 +61,11 @@ class StateValueNetwork(ParametrizedValue):
|
||||
state (State): input state.
|
||||
model (NN, NNApproximator): Neural Network model / approximator.
|
||||
"""
|
||||
super(StateValueNetwork, self).__init__(state, model)
|
||||
super(ValueNetwork, self).__init__(state, model)
|
||||
|
||||
|
||||
class StateInputActionValueNetwork(ParametrizedStateActionValue):
|
||||
r"""State Input Action Value Network
|
||||
class QValueNetwork(ParametrizedQValue):
|
||||
r"""Q-Value Network (which accepts as inputs the states and actions)
|
||||
|
||||
State-action value function :math:`Q_{\phi}(s, a)` approximated by a neural network, where :math:`\phi` represents
|
||||
the parameters of that model. This approximator accepts as inputs the states :math:`s` and actions :math:`a`,
|
||||
@@ -81,11 +81,11 @@ class StateInputActionValueNetwork(ParametrizedStateActionValue):
|
||||
action (Action): input action.
|
||||
model (NN, NNApproximator): Neural Network model / approximator.
|
||||
"""
|
||||
super(StateInputActionValueNetwork, self).__init__(state, action, model)
|
||||
super(QValueNetwork, self).__init__(state, action, model)
|
||||
|
||||
|
||||
class StateOutputActionValueNetwork(ParametrizedStateOutputActionValue):
|
||||
r"""State Output Action Value Network
|
||||
class QValueOutputNetwork(ParametrizedQValueOutput):
|
||||
r"""Q-Value Output Network (which accepts as inputs the states and outputs a Q-value for each discrete action)
|
||||
|
||||
State-action value function :math:`Q_{\phi}(s, a)` approximated by a neural network, where :math:`\phi` represents
|
||||
the parameters of that model. This approximator accepts as inputs the states :math:`s` and outputs the value
|
||||
@@ -101,10 +101,10 @@ class StateOutputActionValueNetwork(ParametrizedStateOutputActionValue):
|
||||
action (Action): output action.
|
||||
model (NN, NNApproximator): Neural Network model / approximator.
|
||||
"""
|
||||
super(StateOutputActionValueNetwork, self).__init__(state, action, model)
|
||||
super(QValueOutputNetwork, self).__init__(state, action, model)
|
||||
|
||||
|
||||
class MLPStateValue(ParametrizedValue): # StateValueNetwork):
|
||||
class MLPValue(ValueNetwork):
|
||||
r"""Multi-Layer Perceptron (MLP) State Value Function Approximator
|
||||
|
||||
This is defined by :math:`V_{\psi}(s_t)` where the function :math:`V` is approximated by a multilayer perceptron.
|
||||
@@ -128,14 +128,14 @@ class MLPStateValue(ParametrizedValue): # StateValueNetwork):
|
||||
the inner model / function approximator.
|
||||
"""
|
||||
output = torch.Tensor([1.]) # torch.Tensor([[1.]])
|
||||
model = MLPApproximator(state, output, hidden_units=hidden_units, activation_fct=activation_fct,
|
||||
last_activation_fct=last_activation_fct, dropout_prob=dropout_prob,
|
||||
model = MLPApproximator(state, output, hidden_units=hidden_units, activation=activation_fct,
|
||||
last_activation=last_activation_fct, dropout=dropout_prob,
|
||||
preprocessors=preprocessors)
|
||||
super(MLPStateValue, self).__init__(state, model)
|
||||
super(MLPValue, self).__init__(state, model)
|
||||
|
||||
|
||||
class MLPStateInputActionValue(ParametrizedStateActionValue):
|
||||
r"""MLP state - input action value function approximator
|
||||
class MLPQValue(QValueNetwork):
|
||||
r"""MLP Q-value function approximator (which accepts as inputs the states and actions)
|
||||
|
||||
State-action value function :math:`Q_{\phi}(s, a)` approximated by a MLP model, where :math:`\phi` represents
|
||||
the parameters of that model. This approximator accepts as inputs the states :math:`s` and actions :math:`a`,
|
||||
@@ -161,13 +161,14 @@ class MLPStateInputActionValue(ParametrizedStateActionValue):
|
||||
the inner model / function approximator.
|
||||
"""
|
||||
model = MLPApproximator(inputs=[state, action], outputs=torch.Tensor([1]), hidden_units=hidden_units,
|
||||
activation_fct=activation_fct, last_activation_fct=last_activation_fct,
|
||||
dropout_prob=dropout_prob, preprocessors=preprocessors)
|
||||
super(MLPStateInputActionValue, self).__init__(state, action, model=model)
|
||||
activation=activation_fct, last_activation=last_activation_fct,
|
||||
dropout=dropout_prob, preprocessors=preprocessors)
|
||||
super(MLPQValue, self).__init__(state, action, model=model)
|
||||
|
||||
|
||||
class MLPStateOutputActionValue(ParametrizedStateOutputActionValue):
|
||||
r"""MLP state - output action value function approximator
|
||||
class MLPQValueOutput(ParametrizedQValueOutput):
|
||||
r"""MLP Q-value function approximator (which accepts as inputs the states and outputs a Q-value for each discrete
|
||||
action)
|
||||
|
||||
State-action value function :math:`Q_{\phi}(s, a)` approximated by a MLP model, where :math:`\phi` represents
|
||||
the parameters of that model. This approximator accepts as inputs the states :math:`s` and outputs the value
|
||||
@@ -193,6 +194,6 @@ class MLPStateOutputActionValue(ParametrizedStateOutputActionValue):
|
||||
the inner model / function approximator.
|
||||
"""
|
||||
model = MLPApproximator(inputs=state, outputs=action, hidden_units=hidden_units,
|
||||
activation_fct=activation_fct, last_activation_fct=last_activation_fct,
|
||||
dropout_prob=dropout_prob, preprocessors=preprocessors)
|
||||
super(MLPStateOutputActionValue, self).__init__(state, action, model=model)
|
||||
activation=activation_fct, last_activation=last_activation_fct,
|
||||
dropout=dropout_prob, preprocessors=preprocessors)
|
||||
super(MLPQValueOutput, self).__init__(state, action, model=model)
|
||||
|
||||
+190
-40
@@ -18,6 +18,7 @@ from pyrobolearn.states import State
|
||||
from pyrobolearn.actions import Action
|
||||
from pyrobolearn.approximators import Approximator
|
||||
|
||||
|
||||
__author__ = "Brian Delhaisse"
|
||||
__copyright__ = "Copyright 2018, PyRoboLearn"
|
||||
__credits__ = ["Brian Delhaisse"]
|
||||
@@ -43,6 +44,8 @@ class ValueApproximator(object):
|
||||
starting at state :math:`s`.
|
||||
2. (state-)action value function denoted by :math:`Q(s,a)`, which represents the expected reward when
|
||||
starting at state :math:`s`, and performing action :math:`a`.
|
||||
|
||||
In both cases, they use the state. This class computes :math:`V(s)`.
|
||||
"""
|
||||
__metaclass__ = ABCMeta
|
||||
|
||||
@@ -78,36 +81,40 @@ class ValueApproximator(object):
|
||||
# Methods #
|
||||
###########
|
||||
|
||||
def compute(self, *args, **kwargs):
|
||||
def evaluate(self, *args, **kwargs):
|
||||
"""Predict the value."""
|
||||
pass
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""Predict the value."""
|
||||
return self.compute(*args, **kwargs)
|
||||
return self.evaluate(*args, **kwargs)
|
||||
|
||||
|
||||
class StateValueApproximator(ValueApproximator):
|
||||
r"""State Value Approximator
|
||||
|
||||
Compute :math:`V(s)`.
|
||||
"""
|
||||
__metaclass__ = ABCMeta
|
||||
|
||||
def __init__(self, state):
|
||||
super(StateValueApproximator, self).__init__(state)
|
||||
|
||||
|
||||
class ActionValueApproximator(ValueApproximator):
|
||||
r"""Action Value Approximator
|
||||
class QValueApproximator(ValueApproximator):
|
||||
r"""Q-Value Approximator
|
||||
|
||||
Compute :math:`Q(s,a)`.
|
||||
"""
|
||||
__metaclass__ = ABCMeta
|
||||
|
||||
def __init__(self, state, actions):
|
||||
super(ActionValueApproximator, self).__init__(state)
|
||||
self.actions = actions
|
||||
def __init__(self, state, action):
|
||||
# super(QValueApproximator, self).__init__(state)
|
||||
ValueApproximator.__init__(self, state)
|
||||
self.action = action
|
||||
|
||||
@property
|
||||
def action(self):
|
||||
"""Return the action instance."""
|
||||
return self._action
|
||||
|
||||
@action.setter
|
||||
def action(self, action):
|
||||
"""Set the action input."""
|
||||
if isinstance(action, (int, float)):
|
||||
action = np.array([action])
|
||||
elif not isinstance(action, (Action, torch.Tensor, np.ndarray)):
|
||||
raise TypeError("Expecting the action to be an Action, torch.Tensor, or np.ndarray.")
|
||||
self._action = action
|
||||
|
||||
|
||||
class ParametrizedValue(ValueApproximator):
|
||||
@@ -124,6 +131,7 @@ class ParametrizedValue(ValueApproximator):
|
||||
state (State): input state.
|
||||
model (Approximator): value function approximator.
|
||||
"""
|
||||
# ValueApproximator.__init__(self, state)
|
||||
super(ParametrizedValue, self).__init__(state)
|
||||
self.model = model
|
||||
|
||||
@@ -238,7 +246,7 @@ class ParametrizedValue(ValueApproximator):
|
||||
"""
|
||||
self.model.set_vectorized_parameters(vector=vector)
|
||||
|
||||
def compute(self, state=None, to_numpy=True):
|
||||
def evaluate(self, state=None, to_numpy=False):
|
||||
"""Compute the output of the value function.
|
||||
|
||||
Args:
|
||||
@@ -246,16 +254,30 @@ class ParametrizedValue(ValueApproximator):
|
||||
the data from the inputs that were given at the initialization.
|
||||
to_numpy (bool): If True, it will convert the data (torch.Tensors) to numpy arrays.
|
||||
"""
|
||||
# if no input is given, take the provided inputs at the beginning
|
||||
if state is None:
|
||||
state = self.state
|
||||
|
||||
# if the input is an instance of State, get the inner merged data.
|
||||
if isinstance(state, State):
|
||||
state = state.merged_data
|
||||
if len(state) == 1:
|
||||
state = state[0]
|
||||
|
||||
self.value = self.model.predict(state, to_numpy=to_numpy, return_logits=True, set_output_data=False)
|
||||
return self.value
|
||||
|
||||
def __call__(self, state=None, to_numpy=True):
|
||||
def __call__(self, state=None, to_numpy=False):
|
||||
"""Predict the value."""
|
||||
return self.compute(state=state, to_numpy=to_numpy)
|
||||
return self.evaluate(state=state, to_numpy=to_numpy)
|
||||
|
||||
|
||||
class ParametrizedStateActionValue(ParametrizedValue):
|
||||
r"""Parametrized State Action Value Function Approximator
|
||||
# alias
|
||||
Value = ParametrizedValue
|
||||
|
||||
|
||||
class ParametrizedQValue(QValueApproximator): # ParametrizedValue, QValueApproximator):
|
||||
r"""Parametrized Q-Value Function Approximator (which accepts as inputs the states and actions)
|
||||
|
||||
This value function approximator predicts the state-action value :math:`Q_{\phi}(s,a)`. Two different kind of
|
||||
state-action value function approximators can be defined:
|
||||
@@ -276,30 +298,155 @@ class ParametrizedStateActionValue(ParametrizedValue):
|
||||
action (Action): input / output action.
|
||||
model (Approximator): value function approximator.
|
||||
"""
|
||||
super(ParametrizedStateActionValue, self).__init__(state, model)
|
||||
self.action = action
|
||||
# ParametrizedValue.__init__(self, state, model)
|
||||
# QValueApproximator.__init__(self, state, action)
|
||||
super(ParametrizedQValue, self).__init__(state, action)
|
||||
self.model = model
|
||||
|
||||
##############
|
||||
# Properties #
|
||||
##############
|
||||
|
||||
@property
|
||||
def action(self):
|
||||
"""Return the action instance."""
|
||||
return self._action
|
||||
def model(self):
|
||||
"""Return the model instance."""
|
||||
return self._model
|
||||
|
||||
@action.setter
|
||||
def action(self, action):
|
||||
"""Set the action input."""
|
||||
if isinstance(action, (int, float)):
|
||||
action = np.array([action])
|
||||
elif not isinstance(action, (Action, torch.Tensor, np.ndarray)):
|
||||
raise TypeError("Expecting the action to be an Action, torch.Tensor, or np.ndarray.")
|
||||
self._action = action
|
||||
@model.setter
|
||||
def model(self, model):
|
||||
"""Set the model / approximator instance."""
|
||||
if not isinstance(model, Approximator):
|
||||
raise TypeError("Expecting the model to be an instance of `Approximator`, instead got: "
|
||||
"{}".format(type(model)))
|
||||
self._model = model
|
||||
|
||||
@property
|
||||
def input_size(self):
|
||||
"""Return the policy input size."""
|
||||
return self.model.input_size
|
||||
|
||||
@property
|
||||
def output_size(self):
|
||||
"""Return the policy output size."""
|
||||
return self.model.output_size
|
||||
|
||||
@property
|
||||
def input_shape(self):
|
||||
"""Return the policy input shape."""
|
||||
return self.model.input_shape
|
||||
|
||||
@property
|
||||
def output_shape(self):
|
||||
"""Return the policy output shape."""
|
||||
return self.model.output_shape
|
||||
|
||||
@property
|
||||
def input_dim(self):
|
||||
"""Return the input dimension of the policy; i.e. len(input_shape)."""
|
||||
return self.model.input_dim
|
||||
|
||||
@property
|
||||
def output_dim(self):
|
||||
"""Return the output dimension of the policy; i.e. len(output_shape)."""
|
||||
return self.model.output_dim
|
||||
|
||||
@property
|
||||
def num_parameters(self):
|
||||
"""Return the total number of parameters"""
|
||||
return self.model.num_parameters
|
||||
|
||||
###########
|
||||
# Methods #
|
||||
###########
|
||||
|
||||
def parameters(self):
|
||||
"""
|
||||
Return an iterator over the learning model parameters.
|
||||
"""
|
||||
return self.model.parameters()
|
||||
|
||||
def named_parameters(self):
|
||||
"""
|
||||
Return an iterator over the learning model parameters; yielding both the name and the parameter itself.
|
||||
"""
|
||||
return self.model.named_parameters()
|
||||
|
||||
def list_parameters(self):
|
||||
"""
|
||||
Return the learning model parameters.
|
||||
"""
|
||||
return self.model.list_parameters()
|
||||
|
||||
def hyperparameters(self):
|
||||
"""
|
||||
Return an iterator over the learning model hyper-parameters.
|
||||
"""
|
||||
return self.model.hyperparameters()
|
||||
|
||||
def named_hyperparameters(self):
|
||||
"""
|
||||
Return an iterator over the learning model hyper-parameters; yielding both the name and the hyper-parameter
|
||||
itself.
|
||||
"""
|
||||
return self.model.named_hyperparameters()
|
||||
|
||||
def list_hyperparameters(self):
|
||||
"""
|
||||
Return the learning model hyper-parameters
|
||||
"""
|
||||
return self.model.list_hyperparameters()
|
||||
|
||||
def get_vectorized_parameters(self, to_numpy=True):
|
||||
"""
|
||||
Get the parameters in a vectorized form.
|
||||
|
||||
Args:
|
||||
to_numpy (bool): if True, it will convert the 1D parameter vector into a numpy array.
|
||||
"""
|
||||
return self.model.get_vectorized_parameters(to_numpy=to_numpy)
|
||||
|
||||
def set_vectorized_parameters(self, vector):
|
||||
"""
|
||||
Set the vectorized parameters.
|
||||
|
||||
Args:
|
||||
np.array, torch.Tensor: 1D parameter vector.
|
||||
"""
|
||||
self.model.set_vectorized_parameters(vector=vector)
|
||||
|
||||
def evaluate(self, state=None, to_numpy=False):
|
||||
"""Compute the output of the value function.
|
||||
|
||||
Args:
|
||||
state (None, State, (list of) np.array, (list of) torch.Tensor): state input data. If None, it will get
|
||||
the data from the inputs that were given at the initialization.
|
||||
to_numpy (bool): If True, it will convert the data (torch.Tensors) to numpy arrays.
|
||||
"""
|
||||
# if no input is given, take the provided inputs at the beginning
|
||||
if state is None:
|
||||
state = self.state
|
||||
|
||||
# if the input is an instance of State, get the inner merged data.
|
||||
if isinstance(state, State):
|
||||
state = state.merged_data
|
||||
if len(state) == 1:
|
||||
state = state[0]
|
||||
|
||||
self.value = self.model.predict(state, to_numpy=to_numpy, return_logits=True, set_output_data=False)
|
||||
return self.value
|
||||
|
||||
def __call__(self, state=None, to_numpy=False):
|
||||
"""Predict the value."""
|
||||
return self.evaluate(state=state, to_numpy=to_numpy)
|
||||
|
||||
|
||||
class ParametrizedStateOutputActionValue(ParametrizedValue):
|
||||
r"""Parametrized State Output Action Value Function Approximator
|
||||
# alias
|
||||
QValue = ParametrizedQValue
|
||||
|
||||
|
||||
class ParametrizedQValueOutput(ParametrizedQValue):
|
||||
r"""Parametrized Q-Value Function Approximator (which accepts as inputs the states and outputs a Q-value for each
|
||||
discrete action)
|
||||
|
||||
This value function approximator predicts the state-action value :math:`Q_{\phi}(s,a)` for each discrete action.
|
||||
"""
|
||||
@@ -313,8 +460,7 @@ class ParametrizedStateOutputActionValue(ParametrizedValue):
|
||||
action (Action): output DISCRETE state.
|
||||
model (Approximator): value function approximator.
|
||||
"""
|
||||
super(ParametrizedStateOutputActionValue, self).__init__(state, model)
|
||||
self.action = action
|
||||
super(ParametrizedQValueOutput, self).__init__(state, action, model)
|
||||
|
||||
##############
|
||||
# Properties #
|
||||
@@ -336,3 +482,7 @@ class ParametrizedStateOutputActionValue(ParametrizedValue):
|
||||
elif not isinstance(action, (torch.Tensor, np.ndarray)):
|
||||
raise TypeError("Expecting the action to be an int, float, Action, torch.Tensor, or np.ndarray.")
|
||||
self._action = action
|
||||
|
||||
|
||||
# alias
|
||||
QValueOutput = ParametrizedQValueOutput
|
||||
|
||||
Reference in New Issue
Block a user