wassname/pyrobolearn
1
0
Fork 0
mirror of https://github.com/wassname/pyrobolearn.git synced 2026-09-09 11:31:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix bugs in algos/storages + update metrics

Browse Source
This commit is contained in:
Brian Delhaisse
2019-07-24 03:22:04 +02:00
parent 56d01638d9
commit 5aaeb5e8cf
19 changed files with 731 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyrobolearn/algos/evaluator.py
+24 -2
View File
@@ -5,6 +5,7 @@ The evaluator assesses the quality of the actions/trajectories performed by the
It is the step performed after the exploration phase, and before the update step.
"""
import torch
from pyrobolearn.returns import Estimator
__author__ = "Brian Delhaisse"
@@ -80,14 +81,35 @@ class Evaluator(object):
"""
if self.estimator is not None:
if verbose:
print("\n#### Starting the Evaluation phase ####")
print("\n#### 2. Starting the Evaluation phase ####")
print("Using estimator: {}".format(self.estimator))
# compute the returns
returns = self.estimator.evaluate(self.storage)
if verbose:
# print("Returns: {}".format(returns))
print("#### End of the Evaluation phase ####")
print("\nFinal storage status: ")
states = self.storage['states'][0]
states = states.view(-1, *states.size()[2:])
print("states: {}".format(torch.cat((torch.arange(len(states), dtype=torch.float).view(-1, 1),
states), dim=1)))
actions = self.storage['actions'][0]
actions = actions.view(-1, *actions.size()[2:])
print("actions: {}".format(torch.cat((torch.arange(len(actions), dtype=torch.float).view(-1, 1),
actions), dim=1)))
rewards = self.storage['rewards'][:, :, 0]
print("rewards: {}".format(torch.cat((torch.arange(len(rewards), dtype=torch.float).view(-1, 1),
rewards), dim=1)))
masks = self.storage['masks'][:, :, 0]
print("masks: {}".format(torch.cat((torch.arange(len(masks), dtype=torch.float).view(-1, 1),
masks), dim=1)))
returns = self.storage[self.estimator][:, :, 0]
print("returns: {}".format(torch.cat((torch.arange(len(returns), dtype=torch.float).view(-1, 1),
returns), dim=1)))
print("\n#### End of the Evaluation phase ####")
#############
# Operators #
pyrobolearn/algos/explorer.py
+10 -2
View File
@@ -528,12 +528,15 @@ class Explorer(object):
Returns:
DictStorage: updated memory storage
"""
if verbose:
print("\n#### 1. Starting the Exploration phase ####")
for rollout in range(num_rollouts):
# reset environment
observation = self.env.reset()
if verbose:
print("Start rollout: {}/{}".format(rollout + 1, num_rollouts))
print("\nStart rollout: {}/{}".format(rollout + 1, num_rollouts))
# print("Explorer - initial state: {}".format(observation))
# reset storage
@@ -543,6 +546,7 @@ class Explorer(object):
self.explorer.reset()
# run RL task for T steps
step = 0
for step in range(num_steps):
# if we need to render
if render:
@@ -578,7 +582,8 @@ class Explorer(object):
self.storage.end(rollout)
if verbose:
print("End rollout: {}/{}".format(rollout + 1, num_rollouts))
print("End rollout: {}/{} with performed step: {}/{}".format(rollout + 1, num_rollouts,
step + 1, num_steps))
# print("states: {}".format(self.storage['states']))
# print("actions: {}".format(self.storage['actions']))
# print("rewards: {}".format(self.storage['rewards']))
@@ -588,6 +593,9 @@ class Explorer(object):
# # clear explorer
# self.explorer.clear()
if verbose:
print("\n#### End of the Exploration phase ####")
# return storage unit
return self.storage
pyrobolearn/algos/reinforce.py
+13 -11
View File
@@ -15,7 +15,7 @@ from pyrobolearn.actorcritics import ActorCritic
from pyrobolearn.exploration import ActionExploration
from pyrobolearn.storages import RolloutStorage
from pyrobolearn.samplers import StorageSampler
from pyrobolearn.samplers import BatchRandomSampler
from pyrobolearn.returns import ActionRewardEstimator, PolicyEvaluator
from pyrobolearn.losses import PGLoss, ValueL2Loss
from pyrobolearn.optimizers import Adam
@@ -132,15 +132,17 @@ class REINFORCE(GradientRLAlgo):
References:
[1] "Simple Statistical Gradient-Following Algorithms for Connectionist Reinforcement Learning", Williams, 1992
[2] "Policy Gradient Methods", Peters, 2010 (Scholarpedia)
[3] "A Survey on Policy Search for Robotics", Deisenroth et al., 2013
[4] PyTorch Reinforce: https://github.com/pytorch/examples/blob/master/reinforcement_learning/reinforce.py
[5] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/vpg.html
[6] "Policy Gradient Algorithms":
https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html
- [1] "Simple Statistical Gradient-Following Algorithms for Connectionist Reinforcement Learning", Williams,
1992
- [2] "Policy Gradient Methods", Peters, 2010 (Scholarpedia)
- [3] "A Survey on Policy Search for Robotics", Deisenroth et al., 2013
- [4] PyTorch Reinforce: https://github.com/pytorch/examples/blob/master/reinforcement_learning/reinforce.py
- [5] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/vpg.html
- [6] "Policy Gradient Algorithms":
https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html
Other implementations:
- https://github.com/rll/rllab/blob/master/rllab/algos/vpg.py
- https://github.com/pytorch/examples/blob/master/reinforcement_learning/reinforce.py
- https://github.com/JamesChuanggg/pytorch-REINFORCE
@@ -149,7 +151,7 @@ class REINFORCE(GradientRLAlgo):
- https://github.com/rlcode/reinforcement-learning/blob/master/2-cartpole/3-reinforce/cartpole_reinforce.py
"""
def __init__(self, task, approximators, gamma=0.99, lr=0.001, num_workers=1):
def __init__(self, task, approximators, gamma=0.99, lr=0.001, num_batches=10, batch_size=10, num_workers=1):
"""
Initialize the REINFORCE on-policy RL algorithm.
@@ -190,7 +192,7 @@ class REINFORCE(GradientRLAlgo):
states, actions = policy.states, policy.actions
storage = RolloutStorage(num_steps=1000, state_shapes=states.merged_shape, action_shapes=actions.merged_shape,
num_trajectories=1)
sampler = StorageSampler(storage)
sampler = BatchRandomSampler(storage, num_batches=10, batch_size_bounds=(8, 64))
# create return: R_t = \sum_{t'=t}^{T} \gamma^{t'-t} r_{t'}
returns = ActionRewardEstimator(storage, gamma=gamma)
@@ -218,7 +220,7 @@ class REINFORCE(GradientRLAlgo):
updater = Updater(approximators, sampler, loss, optimizer, evaluators=[policy_evaluator])
# initialize RL algorithm
super(REINFORCE, self).__init__(explorer, evaluator, updater)
super(REINFORCE, self).__init__(explorer, evaluator, updater, )
# alias
pyrobolearn/algos/rl_algo.py
+9
View File
@@ -323,9 +323,16 @@ class RLAlgo(object): # Algo):
# set the policy in training mode
self.policy.train()
# compute metrics # TODO
# for each episode
for episode in range(num_episodes):
if verbose:
print("\n#####################")
print("#### Episode {}/{} ####".format(episode+1, num_episodes))
print("#####################")
# # for each rollout
# for rollout in range(num_rollouts):
# # TODO: consider to learn the dynamic model if provided
@@ -350,6 +357,8 @@ class RLAlgo(object): # Algo):
# set the policy in test mode
self.policy.eval()
# compute metrics # TODO
if verbose:
print("\n#### End of the RL algo ####")
pyrobolearn/algos/updater.py
+6 -5
View File
@@ -283,7 +283,7 @@ class Updater(object):
losses = {}
if verbose:
print("\n#### Starting the Update phase ####")
print("\n#### 3. Starting the Update phase ####")
# for each epoch
for epoch in range(num_epochs):
@@ -292,7 +292,8 @@ class Updater(object):
for batch_idx, batch in enumerate(self.sampler):
if verbose:
print("Epoch: {}/{} - Batch: {}/{}".format(epoch + 1, num_epochs, batch_idx + 1, num_batches))
print("Epoch: {}/{} - Batch: {}/{} with size {}".format(epoch + 1, num_epochs, batch_idx + 1,
num_batches, batch.size))
# evaluate the evaluators with the current parameters on the given batch and save the results in the
# batch's `current` attribute
@@ -315,9 +316,8 @@ class Updater(object):
# append the loss value in the history of losses
if loss not in losses:
losses[loss] = [[]] * num_epochs
else:
losses[loss][epoch].append(loss_value)
losses[loss] = [[] for epoch in range(num_epochs)]
losses[loss][epoch].append(loss_value.detach())
# update parameters
if verbose:
@@ -336,6 +336,7 @@ class Updater(object):
self._cnt += 1
if verbose:
print("Losses: {}".format(losses))
print("#### End of the Update phase ####")
return losses # shape=(epochs, batches)
pyrobolearn/metrics/README.md
+6 -3
View File
@@ -1,6 +1,9 @@
## Metrics
Different learning tasks use different metrics. For instance, in transfer learning other metrics are used to evaluate the task than in reinforcement learning.
Different learning tasks use different metrics. For instance, in transfer learning other metrics are used to evaluate
the task than in reinforcement learning.
This folder will contain in the future the various metrics. They should be able to collect various information, evaluate how well the learning task is performed, and plot the results.
By providing the different metrics, the user can select which metric he/she wants to use to evaluate his/her learning task.
This folder will contain in the future the various metrics. They should be able to collect various information,
evaluate how well the learning task is performed, and plot the results.
By providing the different metrics, the user can select which metric he/she wants to use to evaluate his/her learning
task.
pyrobolearn/metrics/il_metrics.py
+1 -1
View File
@@ -8,7 +8,7 @@ from pyrobolearn.tasks import ILTask
from pyrobolearn.metrics import Metric
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__copyright__ = "Copyright 2019, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "GNU GPLv3"
__version__ = "1.0.0"
pyrobolearn/metrics/metric.py
+112 -36
View File
@@ -1,16 +1,14 @@
#!/usr/bin/env python
"""Defines the various metrics used in different learning paradigms.
Dependencies:
- `pyrobolearn.tasks`
"""
import collections
import numpy as np
import matplotlib.pyplot as plt
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__copyright__ = "Copyright 2019, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "GNU GPLv3"
__version__ = "1.0.0"
@@ -25,7 +23,7 @@ class Metric(object):
The metric class contains the various metrics used to evaluate a certain learning paradigm (e.g. imitation
learning, reinforcement learning, transfer learning, active learning, and so on).
It notably contains the functionalities to evaluate a certain task using the metric, and different to plot them.
It notably contains the functionalities to evaluate a certain task using the metric, and to plot them.
"""
def __init__(self, metrics=None):
@@ -33,7 +31,7 @@ class Metric(object):
Initialize the metric object.
Args:
metrics (None, Metric, list of Metric): inner metric objects.
metrics (None, Metric, list of Metric): inner metric objects. Each metric will be plot in a subplot.
"""
self._metrics = metrics
@@ -63,62 +61,140 @@ class Metric(object):
# Methods #
###########
def append(self, *args, **kwargs):
pass
def append(self, metric):
"""Append the given metric to the list of metrics."""
if not isinstance(metric, Metric):
raise TypeError("Expecting the given metric to be an instance of `Metric`, but got instead: "
"{}".format(type(metric)))
self.metrics.append(metric)
def update(self, *args, **kwargs):
"""Update the metrics."""
pass
def _plot(self, ax=None, filename=None):
def step_update(self, step_idx=None):
"""Update at each time step."""
if self.metrics:
for metric in self.metrics:
metric._step_update(step_idx=step_idx)
else:
self._step_update(step_idx=step_idx)
def _step_update(self, step_idx=None):
"""Update at each time step; this has to be implemented in the child class."""
pass
def episode_update(self, episode_idx=None):
"""Update at each episode."""
if self.metrics:
for metric in self.metrics:
metric._episode_step(episode_idx=episode_idx)
else:
self._episode_update(episode_idx=episode_idx)
def _episode_update(self, episode_idx=None):
"""Update at each episode; this has to be implemented in the child class."""
pass
def _plot(self, ax):
"""
Plot the metric. This has to be implemented in the child classes.
Args:
ax (plt.Axes): axis to plot the figure.
filename (str, None): if a string is given, it will save the plot in the given filename.
Returns:
plt.Axes: ax
Plot the metric in the given axis. This has to be implemented in the child classes.
"""
pass
def plot(self, ax=None, block=False, filename=None, subplots=()):
def plot(self, nrows=-1, ncols=-1, block=False, filename=None):
"""
Plot the metric.
Plot the metric(s).
Args:
nrows (int): number of rows in the subplot.
ncols (int): number of columns in the subplot.
ax (plt.Axes): axis to plot the figure.
block (bool): if True, it will block when showing the graphs.
filename (str, None): if a string is given, it will save the plot in the given filename.
Returns:
matplotlib.figure.Figure: figure
np.array of matplotlib.axes._subplots.AxesSubplot: axes
"""
# if multiple metrics
if self.metrics:
metrics = self.metrics if self.metrics else [self]
# if we want to use subplots
if len(subplots) > 0:
pass
# get nrows and ncols
if nrows < 1 or ncols < 1:
if nrows < 1 and ncols < 1:
if len(metrics) <= 4:
if len(metrics) <= 2:
nrows = 1
ncols = len(metrics)
else:
nrows = 2
ncols = int(len(metrics) / 2)
else:
ncols = 4
# if we just want multiple figures
for metric in self.metrics:
metric._plot(ax=ax, filename=filename)
if nrows < 1: # ncols is given
nrows = int(len(metrics) / ncols)
if len(metrics) % ncols != 0:
nrows += 1
plt.show(block=block)
else:
self._plot(ax=ax, filename=filename)
plt.show(block=block)
elif ncols < 1: # nrows is given
ncols = int(len(metrics) / nrows)
if len(metrics) % nrows != 0:
ncols += 1
# get number of subplots
nplots = nrows * ncols
# create figure and axes
fig, axes = plt.subplots(nrows=nrows, ncols=ncols)
if not isinstance(axes, np.ndarray):
axes = np.array(axes)
axes = axes.reshape(-1)
# plot each metric
for i, metric in enumerate(self.metrics):
metric._plot(ax=axes[i])
# save figure if specified
if filename is not None:
fig.savefig(filename)
# show plot
plt.show(block=block)
# return figure and axes
return fig, axes
#############
# Operators #
#############
def __repr__(self):
"""Return a representation string of the object."""
if self.metrics:
return ' + '.join(self.metrics)
return self.__class__.__name__
# def __repr__(self):
# """Return a representation string of the object."""
# if self.metrics:
# return ' + '.join(self.metrics)
# return self.__class__.__name__
def __str__(self):
"""Return a string describing the object."""
if self.metrics:
return ' + '.join(self.metrics)
return self.__class__.__name__
def __add__(self, other):
"""Add two sets of metrics together."""
if not isinstance(other, Metric):
raise TypeError("Expecting the given other metric to be an instance of `Metric`, but got instead: "
"{}".format(type(other)))
return Metric(metrics=self.metrics + other.metrics)
def __radd__(self, other):
"""Add two sets of metrics together."""
return self.__add__(other)
def __iadd__(self, other):
"""Append the other metrics to this one."""
if not isinstance(other, Metric):
raise TypeError("Expecting the given other metric to be an instance of `Metric`, but got instead: "
"{}".format(type(other)))
self.metrics += other.metrics
pyrobolearn/metrics/rl_metrics.py
+52 -14
View File
@@ -2,11 +2,13 @@
"""Defines the metrics used in reinforcement learning (RL).
"""
import numpy as np
import matplotlib.pyplot as plt
from pyrobolearn.tasks import RLTask
from pyrobolearn.algos import RLAlgo
from pyrobolearn.metrics import Metric
from pyrobolearn.losses import BatchLoss
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
@@ -44,7 +46,7 @@ class AverageReturnMetric(RLMetric):
where :math:`R(\tau) = \sum_{t=0}^T \gamma^t r_t` is the discounted return.
"""
def __init__(self, task, gamma=1.):
def __init__(self, task, gamma=1., num_episodes=10, num_steps=100):
"""
Initialize the average return metric.
@@ -55,6 +57,8 @@ class AverageReturnMetric(RLMetric):
self.gamma = gamma
self.task = task
self.returns = []
self._num_steps = num_steps
self._num_episodes = num_episodes
##############
# Properties #
@@ -92,35 +96,69 @@ class AverageReturnMetric(RLMetric):
# Methods #
###########
def update(self):
pass
def _episode_update(self, episode_idx=None):
"""Update the metric."""
rewards = []
for ep in range(self._num_episodes):
reward = self.task.run(num_steps=self._num_steps)
rewards.append(reward)
def _plot(self, ax=None, filename=None):
rewards = np.asarray(rewards).mean()
self.returns.append(rewards)
def _plot(self, ax):
"""
Plot the average return metric.
Args:
ax (plt.Axes): axis to plot the figure.
filename (str, None): if a string is given, it will save the plot in the given filename.
Plot the average return metric in the given axis.
"""
if ax is None:
fig, ax = plt.subplots()
ax.set_title('Average Return per iteration') # per epoch, per iteration=epoch*batch
ax.set_xlabel('iterations')
ax.set_ylabel('Average return')
ax.plot(self.returns)
return ax
class LossMetric(RLMetric):
r"""Loss Metric
"""
def __init__(self, loss):
"""
Initialize the loss metric.
Args:
loss (BatchLoss): batch loss.
"""
super(LossMetric, self).__init__()
self.loss = loss
self.losses = []
##############
# Properties #
##############
@property
def loss(self):
"""Return the loss instance."""
return self._loss
@loss.setter
def loss(self, loss):
"""Set the loss instance."""
if not isinstance(loss, BatchLoss):
raise TypeError("Expecting the given 'loss' to be an instance of `BatchLoss`, but got instead: "
"{}".format(type(loss)))
self._loss = loss
###########
# Methods #
###########
def update(self):
pass
def _plot(self, ax):
"""
Plot the loss in the given axis.
"""
ax.set_title(self.loss.__class__.__name__ + ' per iteration')
ax.set_xlabel('iterations')
ax.set_ylabel('Loss')
pyrobolearn/metrics/tl_metrics.py
+99 -3
View File
@@ -1,5 +1,9 @@
#!/usr/bin/env python
"""Defines the metrics used in transfer learning (TL).
References:
- [1] "A Survey on Transfer Learning", Pan et al., 2010
- [2] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
"""
import matplotlib.pyplot as plt
@@ -8,7 +12,7 @@ from pyrobolearn.tasks import TLTask
from pyrobolearn.metrics import Metric
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__copyright__ = "Copyright 2019, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "GNU GPLv3"
__version__ = "1.0.0"
@@ -23,9 +27,101 @@ class TLMetric(Metric):
Metrics used in transfer learning.
References:
[1] "A Survey on Transfer Learning", Pan et al., 2010
[2] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
- [1] "A Survey on Transfer Learning", Pan et al., 2010
- [2] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
"""
def __init__(self):
"""
Initialize the transfer learning metric.
"""
super(TLMetric, self).__init__()
class JumpstartMetric(TLMetric):
r"""Jumpstart metric
The jumpstart metric measures how much the initial performance of an agent in a target task may be improved by
transferring knowledge from a source task. [1]
References:
- [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
"""
def __init__(self):
"""
Initialize the jumpstart metric.
"""
super(JumpstartMetric, self).__init__()
class AsymptoticPerformance(TLMetric):
r"""Asymptotic performance metric
The asymptotic performance metric measures how much the final learned performance of an agent in the target task
has improved via transfer. [1]
References:
- [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
"""
def __init__(self):
"""
Initialize the asymptotic performance metric.
"""
super(AsymptoticPerformance, self).__init__()
class TotalRewardMetric(TLMetric):
r"""Total reward metric
The total reward metric measures the total reward accumulated by a learning agent. This one may be improved if
knowledge transfer was used. [1]
References:
- [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
"""
def __init__(self):
"""
Initialize the total reward metric.
"""
super(TotalRewardMetric, self).__init__()
class TransferRatioMetric(TLMetric):
r"""Transfer ratio metric
The transfer ratio metric measures "the ratio of the total reward accumulated by the transfer learner and the total
reward accumulated by the non-transfer learner". [1]
References:
- [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
"""
def __init__(self):
"""
Initialize the transfer ratio metric.
"""
super(TransferRatioMetric, self).__init__()
class TimeToThresholdMetric(TLMetric):
r"""Time to threshold metric
The time to threshold metric measures how much the learning time needed by the agent to perform a pre-specified
performance level (i.e. the threshold) is reduced via knowledge transfer. [1]
References:
- [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
"""
def __init__(self, threshold):
"""
Initialize the time to threshold metric.
Args:
threshold (float): threshold performance level.
"""
super(TimeToThresholdMetric, self).__init__()
self._threshold = threshold
pyrobolearn/optimizers/torch_optimizer.py
+3 -3
View File
@@ -130,7 +130,7 @@ class Adagrad(Optimizer):
# optimize
self.optimizer.zero_grad()
loss.backward()
loss.backward(retain_graph=True)
if self.max_grad_norm is not None:
nn.utils.clip_grad_norm_(params, self.max_grad_norm)
self.optimizer.step()
@@ -165,7 +165,7 @@ class RMSprop(Optimizer):
# optimize
self.optimizer.zero_grad()
loss.backward()
loss.backward(retain_graph=True)
if self.max_grad_norm is not None:
nn.utils.clip_grad_norm_(params, self.max_grad_norm)
self.optimizer.step()
@@ -198,7 +198,7 @@ class SGD(Optimizer):
# optimize
self.optimizer.zero_grad()
loss.backward()
loss.backward(retain_graph=True)
if self.max_grad_norm is not None:
nn.utils.clip_grad_norm_(params, self.max_grad_norm)
self.optimizer.step()
pyrobolearn/returns/estimators.py
+4 -4
View File
@@ -20,7 +20,7 @@ from pyrobolearn.values import Value, QValue
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "GNU GPLv3"
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
@@ -166,9 +166,9 @@ class Estimator(object):
# Operators #
#############
def __repr__(self):
"""Return a representation string of the object."""
return self.__class__.__name__
# def __repr__(self):
# """Return a representation string of the object."""
# return self.__class__.__name__
def __str__(self):
"""Return a string describing the object."""
pyrobolearn/robots/legged_robot.py
+57
View File
@@ -189,6 +189,13 @@ class LeggedRobot(Robot):
Implications", Popovic et al., 2005
"""
if floor_id is not None:
cop_key = 'cop_' + str(floor_id)
# checked if already cached
if cop_key in self._state:
return self._state[cop_key]
# get contact points between the robot's links and the floor
points = self.sim.get_contact_points(body1=self.id, body2=floor_id)
@@ -204,6 +211,9 @@ class LeggedRobot(Robot):
cop = forces * positions / np.sum(forces)
cop = np.sum(cop, axis=0)
# cache it
self._state[cop_key] = cop
return cop
# check if there are force/pressure sensors at the links/joints
@@ -277,6 +287,13 @@ class LeggedRobot(Robot):
# if the floor id is given, use the simulator to compute the ZMP (using the contact points)
if floor_id is not None:
zmp_key = 'zmp_' + str(floor_id)
# checked if already cached
if zmp_key in self._state:
return self._state[zmp_key]
# get contact points between the robot's links and the floor
points = self.sim.get_contact_points(body1=self.id, body2=floor_id)
@@ -311,6 +328,9 @@ class LeggedRobot(Robot):
zmp[0] += -forces[0]/forces[2] * self.com[2] - moments[1]/forces[2]
zmp[1] += -forces[1]/forces[2] * self.com[2] + moments[0]/forces[2]
# cache it
self._state[zmp_key] = zmp
# return ZMP
return zmp
@@ -382,6 +402,13 @@ class LeggedRobot(Robot):
self.get_center_of_mass_position()
if floor_id is not None:
cmp_key = 'cmp_' + str(floor_id)
# checked if already cached
if cmp_key in self._state:
return self._state[cmp_key]
# get contact points between the robot's links and the floor
points = self.sim.get_contact_points(body1=self.id, body2=floor_id)
@@ -409,6 +436,9 @@ class LeggedRobot(Robot):
cmp[0] -= forces[0] / forces[2] * self.com[2]
cmp[1] -= forces[1] / forces[2] * self.com[2]
# cache it
self._state[cmp_key] = cmp
# return CMP
return cmp
@@ -734,6 +764,33 @@ class LeggedRobot(Robot):
self.sim.remove_body(self.fri_visual)
self.fri_visual = None
def update_visuals(self): # TODO: finish this
"""
Update all visuals.
"""
# update robot visuals
super(LeggedRobot, self).update_visual()
# update support polygon
# update friction cones/pyramids
# update cop
if self.cop_visual is not None:
self.draw_cop()
# update zmp
if self.zmp_visual is not None:
self.draw_zmp()
# update cmp
if self.cmp_visual is not None:
self.draw_cmp()
# update fri
if self.fri_visual is not None:
self.draw_fri()
class BipedRobot(LeggedRobot):
r"""Biped Robot
pyrobolearn/robots/robot.py
+84 -7
View File
@@ -58,7 +58,8 @@ class Robot(ControllableBody):
- [9] "Motion Planning and Control of Dynamic Humanoid Locomotion" (PhD thesis), Xin, 2018
"""
def __init__(self, simulator, urdf, position=None, orientation=None, fixed_base=False, scale=1., *args, **kwargs):
def __init__(self, simulator, urdf, position=None, orientation=None, fixed_base=False, scale=1., visual_ticks=12,
*args, **kwargs):
"""
Initialize the robot.
@@ -69,6 +70,7 @@ class Robot(ControllableBody):
orientation (np.array[4]): initial orientation represented as a quaternion (x,y,z,w).
fixed_base (bool, None): if True, the base of the robot will be fixed.
scale (float): scaling factor.
visual_ticks (int): the number of ticks to sleep before updating the visuals.
"""
# check parameters
if position is None:
@@ -144,6 +146,8 @@ class Robot(ControllableBody):
self._set_end_effectors()
# visual debug: sliders and drawing
self.visual_ticks = visual_ticks
self._visual_cnt = 0
self.joint_sliders = {}
self.com_visual = None
self.projected_com_visual = None
@@ -298,6 +302,18 @@ class Robot(ControllableBody):
# sense
self.sense()
# update visuals
if self._visual_cnt % self.visual_ticks == 0:
self._visual_cnt = 0
# update joint sliders
self.update_joint_slider()
# update all visuals
self.update_visuals()
self._visual_cnt += 1
def sense(self):
"""Run all the sensors."""
for sensors in self.sensors.itervalues():
@@ -545,9 +561,12 @@ class Robot(ControllableBody):
Return the center of mass position.
Returns:
np.array[3]: center of mass position
np.array[3]: center of mass position [m]
"""
if 'com' in self._state:
return self._state['com']
self.com = self.sim.get_center_of_mass_position(self.id)
self._state['com'] = self.com
return self.com
# alias
@@ -558,13 +577,55 @@ class Robot(ControllableBody):
Return the center of mass velocity.
Returns:
np.array[3]: center of mass velocity
np.array[3]: center of mass velocity [m/s]
"""
return self.sim.get_center_of_mass_velocity(self.id)
if 'com_vel' in self._state:
return self._state['com_vel'][0]
com_vel = self.sim.get_center_of_mass_velocity(self.id)
self._state['com_vel'] = [com_vel, time.time()]
return com_vel
# alias
get_com_velocity = get_center_of_mass_velocity
def get_center_of_mass_acceleration(self):
"""
Return the center of mass acceleration.
Returns:
np.array[3]: center of mass acceleration [m/s]
"""
# if already cached, return it
if 'com_acc' in self._state:
return self._state['com_acc'][0]
# compute com velocity
self.get_center_of_mass_velocity()
# if didn't find previous com velocity
if 'com_vel' not in self._prev_state:
acc = np.zeros(3)
self._state['com_acc'] = [acc, time.time()]
return acc
# get current center of mass velocity and time
com_vel, t = self._state['com_vel']
# retrieve previous joint velocities and time
com_vel_prev, t_prev = self._prev_state['com_vel']
# compute time finite difference
if self.sim.use_real_time(): # if the simulator is in real-time mode
dt = (t - t_prev)
else: # if we are stepping in the simulator
dt = self.sim.timestep
# compute com acceleration using finite difference, and cache it
acc = (com_vel - com_vel_prev) / dt
self._state['com_acc'] = [acc, t]
return acc
# def get_linear_momentum(self):
# """
# Compute the linear momentum around the center of mass.
@@ -1061,7 +1122,7 @@ class Robot(ControllableBody):
# retrieve previous joint velocities and time
dq_prev, t_prev = self._prev_state['dq']
# compute time difference
# compute time finite difference
if self.sim.use_real_time(): # if the simulator is in real-time mode
dt = (t - t_prev)
else: # if we are stepping in the simulator
@@ -4531,11 +4592,27 @@ class Robot(ControllableBody):
self.sim.change_visual_shape(self.id, shapeId, rgba_color=rgba)
# print("Link {} - color: {}".format(link, rgba))
def update_visual(self):
def update_visuals(self): # TODO: finish this
"""
Update all visuals.
"""
pass
# update CoM
if self.com_visual is not None:
self.draw_com_position()
# update projected CoM
if self.projected_com_visual is not None:
self.compute_and_draw_projected_com_position()
# update each link's CoM
# update each link frame
# update each link bounding box
# update each joint axis
# update manipulability ellipsoids
def compute_and_draw_com_position(self, radius=0.05, color=(1, 0, 0, 0.8)):
"""
pyrobolearn/samplers/sampler.py
+162 -23
View File
@@ -51,29 +51,70 @@ class StorageSampler(Sampler):
Sampler used with the storage.
"""
def __init__(self, storage, sampler=None, num_batches=10):
def __init__(self, storage, sampler=None, num_batches=10, batch_size=None, batch_size_bounds=None,
replacement=True):
"""
Initialize the storage sampler.
Args:
storage (RolloutStorage): rollout storage.
storage (Storage): storage sampler.
sampler (Sampler, None): If None, it will use a sampler that randomly sample batches of the storage. It
will by default sample :attr:`num_batches`.
num_batches (int): number of batches
num_batches (int): number of batches.
batch_size (int, None): size of the batch. If None, it will be computed based on the size of the storage,
where batch_size = size(storage) // num_batches. Note that the batch size must be smaller than the size
of the storage itself. The num_batches * batch_size can however be bigger than the storage size if
:attr:`replacement = True`.
batch_size_bounds (tuple of int, None): if :attr:`batch_size` is None, we can instead specify the lower
and upper bounds for the `batch_size`. For instance, we can set it to `(16, 128)` along with
`batch_size=None`; this will result to compute `batch_size = size(storage) // num_batches` but if this
one is too small (<16), it will be set to 16, and if this one is too big (>128), it will be set to 128.
replacement (bool): if we should sample each element only one time, or we can sample the same ones multiple
times.
"""
# set the storage
self.storage = storage
# set variables
self._num_batches = num_batches
self._replacement = bool(replacement)
self._batch_size_bounds = batch_size_bounds
self._batch_size_given = batch_size is not None
# set the sampler
if sampler is None:
batch_size = self.size // num_batches
if batch_size > self.size:
raise ValueError("Expecting the batch size (={}) to be smaller than the size of the storage (={})"
".".format(batch_size, self.size))
sampler = torch_sampler.BatchSampler(sampler=torch_sampler.SubsetRandomSampler(range(self.size)),
batch_size=batch_size, drop_last=True)
# check batch size and compute it if necessary
if batch_size is None:
batch_size = self.size // num_batches
# check batch size bounds
if isinstance(batch_size_bounds, (tuple, list)) and len(batch_size_bounds) == 2:
if batch_size < batch_size_bounds[0]:
batch_size = batch_size_bounds[0]
elif batch_size > batch_size_bounds[1]:
batch_size = batch_size_bounds[1]
# check the batch size * number of batches wrt the storage size
if batch_size * num_batches > self.size and not self.replacement:
raise ValueError("Expecting the batch size (={}) * num_batches (={}) to be smaller than the size of "
"the storage (={}), if we can not use replacement.".format(batch_size, num_batches,
self.size))
# subsampler
if replacement:
subsampler = torch_sampler.RandomSampler(data_source=range(self.size), replacement=replacement,
num_samples=self.size)
else:
subsampler = torch_sampler.SubsetRandomSampler(indices=range(self.size))
# create sampler
sampler = torch_sampler.BatchSampler(sampler=subsampler, batch_size=batch_size, drop_last=True)
self.sampler = sampler
print("Sampler: size: {} - num batches: {} - batch size: {}".format(self.size, num_batches, self.batch_size))
##############
# Properties #
##############
@@ -96,6 +137,11 @@ class StorageSampler(Sampler):
"""Return the total size of the storage number of time steps * number of processes."""
return self.storage.size
@property
def filled_size(self):
"""Return the filled size of the storage."""
return self.storage.filled_size
@property
def sampler(self):
"""Return the sampler."""
@@ -115,22 +161,52 @@ class StorageSampler(Sampler):
return self.sampler.batch_size
@batch_size.setter
def batch_size(self, size):
def batch_size(self, batch_size):
"""Set the batch size."""
if size > self.size:
raise ValueError("Expecting the batch size (={}) to be smaller than the size of the storage (={})"
".".format(size, self.size))
self.sampler.batch_size = size
# check the batch size * number of batches wrt the storage size
if batch_size * self._num_batches > self.size and not self.replacement:
raise ValueError("Expecting the batch size (={}) * num_batches (={}) to be smaller than the size of "
"the storage (={}), if we can not use replacement.".format(batch_size, self._num_batches,
self.size))
# check batch size bounds
if isinstance(self._batch_size_bounds, (tuple, list)) and len(self._batch_size_bounds) == 2:
if batch_size < self._batch_size_bounds[0]:
batch_size = self._batch_size_bounds[0]
elif batch_size > self._batch_size_bounds[1]:
batch_size = self._batch_size_bounds[1]
# set the batch size for the sampler
self.sampler.batch_size = batch_size
@property
def num_batches(self):
"""Return the number of batches (based on the size of the storage and the batch size)."""
return self.size // self.batch_size
# return self.filled_size // self.batch_size
return self._num_batches
@num_batches.setter
def num_batches(self, num_batches):
"""Set the number of batches."""
self.batch_size = self.size // num_batches
self._num_batches = num_batches
if not self._batch_size_given:
self.batch_size = self.filled_size // num_batches
else:
if self.batch_size * self._num_batches > self.size and not self.replacement:
raise ValueError("Expecting the batch size (={}) * num_batches (={}) to be smaller than the size of "
"the storage (={}), if we can not use replacement.".format(self.batch_size,
self._num_batches,
self.size))
@property
def replacement(self):
"""Return the replacement boolean."""
return self._replacement
@property
def batch_size_bounds(self):
"""Return the batch size bounds."""
return self._batch_size_bounds
###########
# Methods #
@@ -142,15 +218,67 @@ class StorageSampler(Sampler):
Args:
num_batches (int): number of batches
"""
self.batch_size = self.size // num_batches
# get filled size of the storage
size = self.filled_size
# get batch size
self.batch_size = size // num_batches
# check if there is a sub-sampler
if hasattr(self.sampler, 'sampler'):
if hasattr(self.sampler.sampler, 'data_source'):
self.sampler.sampler.data_source = range(size)
elif hasattr(self.sampler.sampler, 'indices'):
self.sampler.sampler.indices = range(size)
else:
if hasattr(self.sampler, 'data_source'):
self.sampler.data_source = range(size)
elif hasattr(self.sampler, 'indices'):
self.sampler.indices = range(size)
for indices in self.sampler:
yield self.storage.get_batch(indices)
def __iter__(self):
"""Iterate over the storage."""
for indices in self.sampler:
yield self.storage.get_batch(indices)
# get the filled size
size = self.filled_size
print("Storage size: {}".format(self.size))
print("Storage filled size: {}".format(size))
# modify the sampler (by changing the size)
# check if there is a sub-sampler
if hasattr(self.sampler, 'sampler'):
# change the size of the sampler
if hasattr(self.sampler.sampler, 'data_source'):
self.sampler.sampler.data_source = range(size)
elif hasattr(self.sampler.sampler, 'indices'):
self.sampler.sampler.indices = range(size)
# compute the batch size if specified
if not self._batch_size_given:
self.batch_size = size // self.num_batches
else:
# change the size of the sampler
if hasattr(self.sampler, 'data_source'):
self.sampler.data_source = range(size)
elif hasattr(self.sampler, 'indices'):
self.sampler.indices = range(size)
# provide the batches
batch_idx = 0
while True: # this is to account for replacement = True
for indices in self.sampler:
batch_idx += 1
yield self.storage.get_batch(indices)
if batch_idx >= self.num_batches:
break
if batch_idx >= self.num_batches:
break
class BatchRandomSampler(StorageSampler):
@@ -158,12 +286,23 @@ class BatchRandomSampler(StorageSampler):
"""
def __init__(self, storage, num_batches=10):
def __init__(self, storage, num_batches=10, batch_size=None, batch_size_bounds=None, replacement=True):
"""
Initialize the storage sampler.
Args:
storage (RolloutStorage): rollout storage.
num_batches (int): number of batches
storage (Storage): storage to sample the batches from.
num_batches (int): number of batches.
batch_size (int, None): size of the batch. If None, it will be computed based on the size of the storage,
where batch_size = size(storage) // num_batches. Note that the batch size must be smaller than the size
of the storage itself. The num_batches * batch_size can however be bigger than the storage size if
:attr:`replacement = True`.
batch_size_bounds (tuple of int, None): if :attr:`batch_size` is None, we can instead specify the lower
and upper bounds for the `batch_size`. For instance, we can set it to `(16, 128)` along with
`batch_size=None`; this will result to compute `batch_size = size(storage) // num_batches` but if this
one is too small (<16), it will be set to 16, and if this one is too big (>128), it will be set to 128.
replacement (bool): if we should sample each element only one time, or we can sample the same ones multiple
times.
"""
super(BatchRandomSampler, self).__init__(storage, num_batches=num_batches)
super(BatchRandomSampler, self).__init__(storage=storage, num_batches=num_batches, batch_size=batch_size,
batch_size_bounds=batch_size_bounds, replacement=replacement)
pyrobolearn/storages/__init__.py
+1 -2
View File
@@ -1,7 +1,6 @@
# import memory
# import storages
from .storage import *
# import experience replay memories/storages
from .er import *
pyrobolearn/storages/er.py
+8 -1
View File
@@ -208,6 +208,13 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage):
# return self.position
return self.capacity
@property
def filled_size(self):
"""Return the filled sized of the experience replay storage."""
if self.full:
return self.capacity
return self.position
###########
# Methods #
###########
@@ -364,7 +371,7 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage):
"""Return a batch of the experience replay storage in the form of a `DictStorage`.
Args:
indices (list of int): indices. Each index must be between 0 and `self.size`.
indices (list of int): indices. Each index must be between 0 and `self.filled_size`.
Returns:
DictStorage / Batch: batch containing a part of the storage. Variables such as `states`, `actions`,
pyrobolearn/storages/storage.py
+77 -23
View File
@@ -32,19 +32,24 @@ __status__ = "Development"
class Storage(object):
"""Main abstract storage class."""
def save(self, filename):
"""Save the storage on the disk."""
pickle.dump(self, open(filename, 'wb'))
@property
def size(self):
"""Return the size of the storage. Need to be implemented in the child class."""
return 0
@property
def filled_size(self):
"""Return the filled size of the storage."""
return self.size
@staticmethod
def load(filename):
"""Load the storage from the disk."""
return pickle.load(open(filename, 'r'))
@property
def size(self):
"""Return the size of the storage. Need to be implemented in the child class."""
return 0
def save(self, filename):
"""Save the storage on the disk."""
pickle.dump(self, open(filename, 'wb'))
def get_batch(self, indices):
"""Return a batch of the storage as a `Storage` type.
@@ -624,7 +629,7 @@ class Batch(DictStorage):
are filled by the exploration phase in RL algorithms (see `pyrobolearn/algos/explorer`).
"""
def __init__(self, kwargs=None, device=None, dtype=None):
def __init__(self, kwargs=None, device=None, dtype=None, size=None):
"""
Initialize the Batch storage.
@@ -635,12 +640,19 @@ class Batch(DictStorage):
to which the tensor is allocated.
dtype (torch.dtype, None): convert the `torch.Tensor` to the specified data type. If None, it will keep
the original dtype
size (int, None): size of the batch.
"""
super(Batch, self).__init__(kwargs=kwargs, device=device, dtype=dtype, update=False)
# contains the current values evaluated during the update phase of RL algorithms.
self.current = DictStorage(kwargs={}, device=device, dtype=dtype, update=False)
# indices where the masks is different from 0 in the current batch
self.indices = None
self._size = size if size is not None else len(kwargs['masks']) # TODO: need to generalize this
@property
def size(self):
"""Return the size of the batch."""
return self._size
def get_current(self, key, default=None):
"""Try first to get the key from :attr:`current`, if not present, try to get it from the batch storage.
@@ -728,6 +740,14 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies:
"""Return the size (=number of steps * number of processes) of the rollout storage."""
return self._num_steps * self._num_trajectories
@property
def filled_size(self):
"""Return the filled size by looking at the entries where the mask = 1."""
# print("Mask: {}".format(self.masks[:, :, 0]))
# print("Mask shape: {}".format(self.masks.shape))
# print("Steps: {}".format(self._step))
return np.minimum(len(self.masks[self.masks == 1]), self.size)
@property
def capacity(self):
"""Return the capacity of the rollout storage (=number of steps * number of processes)."""
@@ -760,7 +780,7 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies:
`self.num_trajectories`.
"""
# if end of storage, go at the beginning
self._step[rollout_idx] = (self._step[rollout_idx] + 1) % self.num_steps
self._step[rollout_idx] = (self._step[rollout_idx] + 1) % (self.num_steps + 1) # self.num_steps
def create_new_entry(self, key, shapes, num_steps=None, dtype=torch.dtype):
"""Create a new entry (=tensor) in the rollout storage dictionary. The tensor will have the dimension
@@ -881,7 +901,8 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies:
self._step[rollout_idx] = 0
# reset masks
self.masks[:, rollout_idx].copy_(torch.ones_like(self.masks[:, rollout_idx]))
# self.masks[:, rollout_idx].copy_(torch.ones_like(self.masks[:, rollout_idx])) # fill with ones
self.masks[:, rollout_idx].copy_(torch.zeros_like(self.masks[:, rollout_idx])) # fill with zeros
# insert initial states
if init_states is None:
@@ -972,6 +993,13 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies:
"""
t = self._step[rollout_idx]
print("\nStorage: rollout = {}, step = {}".format(rollout_idx, t))
print("Storage: insert state: {}".format(states))
print("Storage: insert action: {}".format(actions))
print("Storage: insert next state: {}".format(next_states))
print("Storage: insert reward: {}".format(reward))
print("Storage: insert mask: {}".format(mask))
# check given observations/states and actions
if not isinstance(next_states, list):
next_states = [next_states]
@@ -1005,7 +1033,7 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies:
# update the step if specified
if update_step:
self.step()
self.step(rollout_idx=rollout_idx)
def add_trajectory(self, trajectory, rollout_idx=0):
r"""
@@ -1030,28 +1058,54 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies:
"""Return a batch of the Rollout storage in the form of a `DictStorage`.
Args:
indices (list of int): indices. Each index must be between 0 and `num_steps * num_trajectories`.
indices (list of int): indices. Each index must be between 0 and `self.filled_size`-1.
Returns:
DictStorage / Batch: batch containing a part of the storage. Variables such as `states`, `actions`,
`rewards`, `masks`, and others can be accessed from the object.
"""
# In the next comments, T = number of time steps, P = number of processes, and I = number of indices
# In the next comments, T = number of time steps, P = number of processes, and I = number of indices, F =
# number of filled entries (where the mask == 1)
batch = {}
size = len(indices)
# # The following sampling method was not optimal as it would sample entries where the mask == 0 (i.e. when
# # the episode is over)
# def sample(item, indices):
# if isinstance(item, torch.Tensor):
# if len(item) == self.num_steps + 1: # = T+1
# item = item[:-1] # take only the T steps
# return item.view(-1, *item.size()[2:])[indices] # reshape to (T*P, *shape) and from T*P takes I
#
# elif isinstance(item, np.ndarray):
# if len(item) == self.num_steps + 1: # = T+1
# item = item[:-1] # take only the T steps
# return item.reshape(-1, *item.shape[2:])[indices] # reshape to (T*P, *shape) and from T*P takes I
print("Indices: {} - length: {}".format(indices, len(indices)))
def sample(item, indices):
if isinstance(item, torch.Tensor):
if len(item) == self.num_steps + 1: # = T+1
item = item[:-1] # take only the T steps
return item.view(-1, *item.size()[2:])[indices] # reshape to (T*P, *shape) and from T*P takes I
elif isinstance(item, np.ndarray):
if len(item) == self.num_steps + 1: # = T+1
item = item[:-1] # take only the T steps
return item.reshape(-1, *item.shape[2:])[indices] # reshape to (T*P, *shape) and from T*P takes I
"""Given indices where each index is between 0 and `self.filled_size` (=number of masks that are equal
to 1), it returns the corresponding entries in the item.
"""
if isinstance(item, torch.Tensor): # TODO: improve performance of this
masks = (self.masks[:, :, 0] == 1).nonzero() # [F, 2] --> indices for (steps, trajs)
masks[:, 0] -= 1 # remove 1 because indices for masks are in [1, t+1] --> [0, t]
indices = masks[indices] # [I,2] --> allowed indices for (steps, trajs)
print("Torch mask length: {}".format(len(masks)))
print("Indices: {}".format(indices))
return item[indices[:, 0], indices[:, 1]] # [I, *shape]
elif isinstance(item, np.ndarray): # TODO: improve performance of this
masks = np.vstack(((self.masks[:, :, 0] == 1).nonzero())) # [F,2] --> indices for (steps, trajs)
masks[:, 0] -= 1 # remove 1 because indices for masks are in [1, t+1] --> [0, t]
indices = masks[indices] # [I,2] --> allowed indices for (steps, trajs)
print("Numpy Mask length: {}".format(len(masks)))
print("Indices: {}".format(indices))
return item[indices[:, 0], indices[:, 1]] # [I, *shape]
# go through each attribute and sample from the tensors
for key, value in self.iteritems():
print("batch - add key: {}".format(key))
if isinstance(value, list): # value = list of tensors
batch[key] = [sample(val, indices) for val in value]
else: # value = tensor
@@ -1074,7 +1128,7 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies:
distributions[i] = dist.__class__.from_list(distribution)
# create Batch object
batch = Batch(batch, device=self.device, dtype=self.dtype)
batch = Batch(batch, device=self.device, dtype=self.dtype, size=size)
batch.indices = torch.tensor(range(len(indices)))[batch['masks'][:, 0] != 0].tolist()
# return batch (which is given to the updater (and loss))
pyrobolearn/tasks/transfer.py
+3 -3
View File
@@ -7,7 +7,7 @@ from pyrobolearn.tasks.task import Task
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "GNU GPLv3"
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
@@ -21,8 +21,8 @@ class TLTask(object):
different but similar problem [1,2].
References:
[1] "A Survey on Transfer Learning", Pan et al., 2010
[2]" Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
- [1] "A Survey on Transfer Learning", Pan et al., 2010
- [2]" Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009
"""
def __init__(self, domain_task, target_task):