diff --git a/pyrobolearn/algos/evaluator.py b/pyrobolearn/algos/evaluator.py index b1fb3ab..288f232 100644 --- a/pyrobolearn/algos/evaluator.py +++ b/pyrobolearn/algos/evaluator.py @@ -5,6 +5,7 @@ The evaluator assesses the quality of the actions/trajectories performed by the It is the step performed after the exploration phase, and before the update step. """ +import torch from pyrobolearn.returns import Estimator __author__ = "Brian Delhaisse" @@ -80,14 +81,35 @@ class Evaluator(object): """ if self.estimator is not None: if verbose: - print("\n#### Starting the Evaluation phase ####") + print("\n#### 2. Starting the Evaluation phase ####") + print("Using estimator: {}".format(self.estimator)) # compute the returns returns = self.estimator.evaluate(self.storage) if verbose: # print("Returns: {}".format(returns)) - print("#### End of the Evaluation phase ####") + + print("\nFinal storage status: ") + states = self.storage['states'][0] + states = states.view(-1, *states.size()[2:]) + print("states: {}".format(torch.cat((torch.arange(len(states), dtype=torch.float).view(-1, 1), + states), dim=1))) + actions = self.storage['actions'][0] + actions = actions.view(-1, *actions.size()[2:]) + print("actions: {}".format(torch.cat((torch.arange(len(actions), dtype=torch.float).view(-1, 1), + actions), dim=1))) + rewards = self.storage['rewards'][:, :, 0] + print("rewards: {}".format(torch.cat((torch.arange(len(rewards), dtype=torch.float).view(-1, 1), + rewards), dim=1))) + masks = self.storage['masks'][:, :, 0] + print("masks: {}".format(torch.cat((torch.arange(len(masks), dtype=torch.float).view(-1, 1), + masks), dim=1))) + returns = self.storage[self.estimator][:, :, 0] + print("returns: {}".format(torch.cat((torch.arange(len(returns), dtype=torch.float).view(-1, 1), + returns), dim=1))) + + print("\n#### End of the Evaluation phase ####") ############# # Operators # diff --git a/pyrobolearn/algos/explorer.py b/pyrobolearn/algos/explorer.py index e16d2aa..ba63cbf 100644 --- a/pyrobolearn/algos/explorer.py +++ b/pyrobolearn/algos/explorer.py @@ -528,12 +528,15 @@ class Explorer(object): Returns: DictStorage: updated memory storage """ + if verbose: + print("\n#### 1. Starting the Exploration phase ####") + for rollout in range(num_rollouts): # reset environment observation = self.env.reset() if verbose: - print("Start rollout: {}/{}".format(rollout + 1, num_rollouts)) + print("\nStart rollout: {}/{}".format(rollout + 1, num_rollouts)) # print("Explorer - initial state: {}".format(observation)) # reset storage @@ -543,6 +546,7 @@ class Explorer(object): self.explorer.reset() # run RL task for T steps + step = 0 for step in range(num_steps): # if we need to render if render: @@ -578,7 +582,8 @@ class Explorer(object): self.storage.end(rollout) if verbose: - print("End rollout: {}/{}".format(rollout + 1, num_rollouts)) + print("End rollout: {}/{} with performed step: {}/{}".format(rollout + 1, num_rollouts, + step + 1, num_steps)) # print("states: {}".format(self.storage['states'])) # print("actions: {}".format(self.storage['actions'])) # print("rewards: {}".format(self.storage['rewards'])) @@ -588,6 +593,9 @@ class Explorer(object): # # clear explorer # self.explorer.clear() + if verbose: + print("\n#### End of the Exploration phase ####") + # return storage unit return self.storage diff --git a/pyrobolearn/algos/reinforce.py b/pyrobolearn/algos/reinforce.py index 99e34e3..aeba020 100755 --- a/pyrobolearn/algos/reinforce.py +++ b/pyrobolearn/algos/reinforce.py @@ -15,7 +15,7 @@ from pyrobolearn.actorcritics import ActorCritic from pyrobolearn.exploration import ActionExploration from pyrobolearn.storages import RolloutStorage -from pyrobolearn.samplers import StorageSampler +from pyrobolearn.samplers import BatchRandomSampler from pyrobolearn.returns import ActionRewardEstimator, PolicyEvaluator from pyrobolearn.losses import PGLoss, ValueL2Loss from pyrobolearn.optimizers import Adam @@ -132,15 +132,17 @@ class REINFORCE(GradientRLAlgo): References: - [1] "Simple Statistical Gradient-Following Algorithms for Connectionist Reinforcement Learning", Williams, 1992 - [2] "Policy Gradient Methods", Peters, 2010 (Scholarpedia) - [3] "A Survey on Policy Search for Robotics", Deisenroth et al., 2013 - [4] PyTorch Reinforce: https://github.com/pytorch/examples/blob/master/reinforcement_learning/reinforce.py - [5] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/vpg.html - [6] "Policy Gradient Algorithms": - https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html + - [1] "Simple Statistical Gradient-Following Algorithms for Connectionist Reinforcement Learning", Williams, + 1992 + - [2] "Policy Gradient Methods", Peters, 2010 (Scholarpedia) + - [3] "A Survey on Policy Search for Robotics", Deisenroth et al., 2013 + - [4] PyTorch Reinforce: https://github.com/pytorch/examples/blob/master/reinforcement_learning/reinforce.py + - [5] OpenAI - Spinning Up: https://spinningup.openai.com/en/latest/algorithms/vpg.html + - [6] "Policy Gradient Algorithms": + https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html Other implementations: + - https://github.com/rll/rllab/blob/master/rllab/algos/vpg.py - https://github.com/pytorch/examples/blob/master/reinforcement_learning/reinforce.py - https://github.com/JamesChuanggg/pytorch-REINFORCE @@ -149,7 +151,7 @@ class REINFORCE(GradientRLAlgo): - https://github.com/rlcode/reinforcement-learning/blob/master/2-cartpole/3-reinforce/cartpole_reinforce.py """ - def __init__(self, task, approximators, gamma=0.99, lr=0.001, num_workers=1): + def __init__(self, task, approximators, gamma=0.99, lr=0.001, num_batches=10, batch_size=10, num_workers=1): """ Initialize the REINFORCE on-policy RL algorithm. @@ -190,7 +192,7 @@ class REINFORCE(GradientRLAlgo): states, actions = policy.states, policy.actions storage = RolloutStorage(num_steps=1000, state_shapes=states.merged_shape, action_shapes=actions.merged_shape, num_trajectories=1) - sampler = StorageSampler(storage) + sampler = BatchRandomSampler(storage, num_batches=10, batch_size_bounds=(8, 64)) # create return: R_t = \sum_{t'=t}^{T} \gamma^{t'-t} r_{t'} returns = ActionRewardEstimator(storage, gamma=gamma) @@ -218,7 +220,7 @@ class REINFORCE(GradientRLAlgo): updater = Updater(approximators, sampler, loss, optimizer, evaluators=[policy_evaluator]) # initialize RL algorithm - super(REINFORCE, self).__init__(explorer, evaluator, updater) + super(REINFORCE, self).__init__(explorer, evaluator, updater, ) # alias diff --git a/pyrobolearn/algos/rl_algo.py b/pyrobolearn/algos/rl_algo.py index 95532d3..46f6e69 100755 --- a/pyrobolearn/algos/rl_algo.py +++ b/pyrobolearn/algos/rl_algo.py @@ -323,9 +323,16 @@ class RLAlgo(object): # Algo): # set the policy in training mode self.policy.train() + # compute metrics # TODO + # for each episode for episode in range(num_episodes): + if verbose: + print("\n#####################") + print("#### Episode {}/{} ####".format(episode+1, num_episodes)) + print("#####################") + # # for each rollout # for rollout in range(num_rollouts): # # TODO: consider to learn the dynamic model if provided @@ -350,6 +357,8 @@ class RLAlgo(object): # Algo): # set the policy in test mode self.policy.eval() + # compute metrics # TODO + if verbose: print("\n#### End of the RL algo ####") diff --git a/pyrobolearn/algos/updater.py b/pyrobolearn/algos/updater.py index 8890212..8ab97ec 100644 --- a/pyrobolearn/algos/updater.py +++ b/pyrobolearn/algos/updater.py @@ -283,7 +283,7 @@ class Updater(object): losses = {} if verbose: - print("\n#### Starting the Update phase ####") + print("\n#### 3. Starting the Update phase ####") # for each epoch for epoch in range(num_epochs): @@ -292,7 +292,8 @@ class Updater(object): for batch_idx, batch in enumerate(self.sampler): if verbose: - print("Epoch: {}/{} - Batch: {}/{}".format(epoch + 1, num_epochs, batch_idx + 1, num_batches)) + print("Epoch: {}/{} - Batch: {}/{} with size {}".format(epoch + 1, num_epochs, batch_idx + 1, + num_batches, batch.size)) # evaluate the evaluators with the current parameters on the given batch and save the results in the # batch's `current` attribute @@ -315,9 +316,8 @@ class Updater(object): # append the loss value in the history of losses if loss not in losses: - losses[loss] = [[]] * num_epochs - else: - losses[loss][epoch].append(loss_value) + losses[loss] = [[] for epoch in range(num_epochs)] + losses[loss][epoch].append(loss_value.detach()) # update parameters if verbose: @@ -336,6 +336,7 @@ class Updater(object): self._cnt += 1 if verbose: + print("Losses: {}".format(losses)) print("#### End of the Update phase ####") return losses # shape=(epochs, batches) diff --git a/pyrobolearn/metrics/README.md b/pyrobolearn/metrics/README.md index eab6148..a89ece7 100644 --- a/pyrobolearn/metrics/README.md +++ b/pyrobolearn/metrics/README.md @@ -1,6 +1,9 @@ ## Metrics -Different learning tasks use different metrics. For instance, in transfer learning other metrics are used to evaluate the task than in reinforcement learning. +Different learning tasks use different metrics. For instance, in transfer learning other metrics are used to evaluate +the task than in reinforcement learning. -This folder will contain in the future the various metrics. They should be able to collect various information, evaluate how well the learning task is performed, and plot the results. -By providing the different metrics, the user can select which metric he/she wants to use to evaluate his/her learning task. +This folder will contain in the future the various metrics. They should be able to collect various information, +evaluate how well the learning task is performed, and plot the results. +By providing the different metrics, the user can select which metric he/she wants to use to evaluate his/her learning +task. diff --git a/pyrobolearn/metrics/il_metrics.py b/pyrobolearn/metrics/il_metrics.py index 480dc07..890fa6a 100644 --- a/pyrobolearn/metrics/il_metrics.py +++ b/pyrobolearn/metrics/il_metrics.py @@ -8,7 +8,7 @@ from pyrobolearn.tasks import ILTask from pyrobolearn.metrics import Metric __author__ = "Brian Delhaisse" -__copyright__ = "Copyright 2018, PyRoboLearn" +__copyright__ = "Copyright 2019, PyRoboLearn" __credits__ = ["Brian Delhaisse"] __license__ = "GNU GPLv3" __version__ = "1.0.0" diff --git a/pyrobolearn/metrics/metric.py b/pyrobolearn/metrics/metric.py index e187d6d..5b2059b 100644 --- a/pyrobolearn/metrics/metric.py +++ b/pyrobolearn/metrics/metric.py @@ -1,16 +1,14 @@ #!/usr/bin/env python """Defines the various metrics used in different learning paradigms. - -Dependencies: -- `pyrobolearn.tasks` """ import collections +import numpy as np import matplotlib.pyplot as plt __author__ = "Brian Delhaisse" -__copyright__ = "Copyright 2018, PyRoboLearn" +__copyright__ = "Copyright 2019, PyRoboLearn" __credits__ = ["Brian Delhaisse"] __license__ = "GNU GPLv3" __version__ = "1.0.0" @@ -25,7 +23,7 @@ class Metric(object): The metric class contains the various metrics used to evaluate a certain learning paradigm (e.g. imitation learning, reinforcement learning, transfer learning, active learning, and so on). - It notably contains the functionalities to evaluate a certain task using the metric, and different to plot them. + It notably contains the functionalities to evaluate a certain task using the metric, and to plot them. """ def __init__(self, metrics=None): @@ -33,7 +31,7 @@ class Metric(object): Initialize the metric object. Args: - metrics (None, Metric, list of Metric): inner metric objects. + metrics (None, Metric, list of Metric): inner metric objects. Each metric will be plot in a subplot. """ self._metrics = metrics @@ -63,62 +61,140 @@ class Metric(object): # Methods # ########### - def append(self, *args, **kwargs): - pass + def append(self, metric): + """Append the given metric to the list of metrics.""" + if not isinstance(metric, Metric): + raise TypeError("Expecting the given metric to be an instance of `Metric`, but got instead: " + "{}".format(type(metric))) + self.metrics.append(metric) def update(self, *args, **kwargs): + """Update the metrics.""" pass - def _plot(self, ax=None, filename=None): + def step_update(self, step_idx=None): + """Update at each time step.""" + if self.metrics: + for metric in self.metrics: + metric._step_update(step_idx=step_idx) + else: + self._step_update(step_idx=step_idx) + + def _step_update(self, step_idx=None): + """Update at each time step; this has to be implemented in the child class.""" + pass + + def episode_update(self, episode_idx=None): + """Update at each episode.""" + if self.metrics: + for metric in self.metrics: + metric._episode_step(episode_idx=episode_idx) + else: + self._episode_update(episode_idx=episode_idx) + + def _episode_update(self, episode_idx=None): + """Update at each episode; this has to be implemented in the child class.""" + pass + + def _plot(self, ax): """ - Plot the metric. This has to be implemented in the child classes. - - Args: - ax (plt.Axes): axis to plot the figure. - filename (str, None): if a string is given, it will save the plot in the given filename. - - Returns: - plt.Axes: ax + Plot the metric in the given axis. This has to be implemented in the child classes. """ pass - def plot(self, ax=None, block=False, filename=None, subplots=()): + def plot(self, nrows=-1, ncols=-1, block=False, filename=None): """ - Plot the metric. + Plot the metric(s). Args: + nrows (int): number of rows in the subplot. + ncols (int): number of columns in the subplot. ax (plt.Axes): axis to plot the figure. block (bool): if True, it will block when showing the graphs. filename (str, None): if a string is given, it will save the plot in the given filename. + + Returns: + matplotlib.figure.Figure: figure + np.array of matplotlib.axes._subplots.AxesSubplot: axes """ - # if multiple metrics - if self.metrics: + metrics = self.metrics if self.metrics else [self] - # if we want to use subplots - if len(subplots) > 0: - pass + # get nrows and ncols + if nrows < 1 or ncols < 1: + if nrows < 1 and ncols < 1: + if len(metrics) <= 4: + if len(metrics) <= 2: + nrows = 1 + ncols = len(metrics) + else: + nrows = 2 + ncols = int(len(metrics) / 2) + else: + ncols = 4 - # if we just want multiple figures - for metric in self.metrics: - metric._plot(ax=ax, filename=filename) + if nrows < 1: # ncols is given + nrows = int(len(metrics) / ncols) + if len(metrics) % ncols != 0: + nrows += 1 - plt.show(block=block) - else: - self._plot(ax=ax, filename=filename) - plt.show(block=block) + elif ncols < 1: # nrows is given + ncols = int(len(metrics) / nrows) + if len(metrics) % nrows != 0: + ncols += 1 + + # get number of subplots + nplots = nrows * ncols + + # create figure and axes + fig, axes = plt.subplots(nrows=nrows, ncols=ncols) + if not isinstance(axes, np.ndarray): + axes = np.array(axes) + axes = axes.reshape(-1) + + # plot each metric + for i, metric in enumerate(self.metrics): + metric._plot(ax=axes[i]) + + # save figure if specified + if filename is not None: + fig.savefig(filename) + + # show plot + plt.show(block=block) + + # return figure and axes + return fig, axes ############# # Operators # ############# - def __repr__(self): - """Return a representation string of the object.""" - if self.metrics: - return ' + '.join(self.metrics) - return self.__class__.__name__ + # def __repr__(self): + # """Return a representation string of the object.""" + # if self.metrics: + # return ' + '.join(self.metrics) + # return self.__class__.__name__ def __str__(self): """Return a string describing the object.""" if self.metrics: return ' + '.join(self.metrics) return self.__class__.__name__ + + def __add__(self, other): + """Add two sets of metrics together.""" + if not isinstance(other, Metric): + raise TypeError("Expecting the given other metric to be an instance of `Metric`, but got instead: " + "{}".format(type(other))) + return Metric(metrics=self.metrics + other.metrics) + + def __radd__(self, other): + """Add two sets of metrics together.""" + return self.__add__(other) + + def __iadd__(self, other): + """Append the other metrics to this one.""" + if not isinstance(other, Metric): + raise TypeError("Expecting the given other metric to be an instance of `Metric`, but got instead: " + "{}".format(type(other))) + self.metrics += other.metrics diff --git a/pyrobolearn/metrics/rl_metrics.py b/pyrobolearn/metrics/rl_metrics.py index 040d2e9..8d43739 100644 --- a/pyrobolearn/metrics/rl_metrics.py +++ b/pyrobolearn/metrics/rl_metrics.py @@ -2,11 +2,13 @@ """Defines the metrics used in reinforcement learning (RL). """ +import numpy as np import matplotlib.pyplot as plt from pyrobolearn.tasks import RLTask from pyrobolearn.algos import RLAlgo from pyrobolearn.metrics import Metric +from pyrobolearn.losses import BatchLoss __author__ = "Brian Delhaisse" __copyright__ = "Copyright 2018, PyRoboLearn" @@ -44,7 +46,7 @@ class AverageReturnMetric(RLMetric): where :math:`R(\tau) = \sum_{t=0}^T \gamma^t r_t` is the discounted return. """ - def __init__(self, task, gamma=1.): + def __init__(self, task, gamma=1., num_episodes=10, num_steps=100): """ Initialize the average return metric. @@ -55,6 +57,8 @@ class AverageReturnMetric(RLMetric): self.gamma = gamma self.task = task self.returns = [] + self._num_steps = num_steps + self._num_episodes = num_episodes ############## # Properties # @@ -92,35 +96,69 @@ class AverageReturnMetric(RLMetric): # Methods # ########### - def update(self): - pass + def _episode_update(self, episode_idx=None): + """Update the metric.""" + rewards = [] + for ep in range(self._num_episodes): + reward = self.task.run(num_steps=self._num_steps) + rewards.append(reward) - def _plot(self, ax=None, filename=None): + rewards = np.asarray(rewards).mean() + self.returns.append(rewards) + + def _plot(self, ax): """ - Plot the average return metric. - - Args: - ax (plt.Axes): axis to plot the figure. - filename (str, None): if a string is given, it will save the plot in the given filename. + Plot the average return metric in the given axis. """ - if ax is None: - fig, ax = plt.subplots() - ax.set_title('Average Return per iteration') # per epoch, per iteration=epoch*batch ax.set_xlabel('iterations') ax.set_ylabel('Average return') ax.plot(self.returns) - return ax - class LossMetric(RLMetric): r"""Loss Metric """ def __init__(self, loss): + """ + Initialize the loss metric. + + Args: + loss (BatchLoss): batch loss. + """ super(LossMetric, self).__init__() self.loss = loss + self.losses = [] + + ############## + # Properties # + ############## + + @property + def loss(self): + """Return the loss instance.""" + return self._loss + + @loss.setter + def loss(self, loss): + """Set the loss instance.""" + if not isinstance(loss, BatchLoss): + raise TypeError("Expecting the given 'loss' to be an instance of `BatchLoss`, but got instead: " + "{}".format(type(loss))) + self._loss = loss + + ########### + # Methods # + ########### def update(self): pass + + def _plot(self, ax): + """ + Plot the loss in the given axis. + """ + ax.set_title(self.loss.__class__.__name__ + ' per iteration') + ax.set_xlabel('iterations') + ax.set_ylabel('Loss') diff --git a/pyrobolearn/metrics/tl_metrics.py b/pyrobolearn/metrics/tl_metrics.py index d8d51c6..6458bb7 100644 --- a/pyrobolearn/metrics/tl_metrics.py +++ b/pyrobolearn/metrics/tl_metrics.py @@ -1,5 +1,9 @@ #!/usr/bin/env python """Defines the metrics used in transfer learning (TL). + +References: + - [1] "A Survey on Transfer Learning", Pan et al., 2010 + - [2] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 """ import matplotlib.pyplot as plt @@ -8,7 +12,7 @@ from pyrobolearn.tasks import TLTask from pyrobolearn.metrics import Metric __author__ = "Brian Delhaisse" -__copyright__ = "Copyright 2018, PyRoboLearn" +__copyright__ = "Copyright 2019, PyRoboLearn" __credits__ = ["Brian Delhaisse"] __license__ = "GNU GPLv3" __version__ = "1.0.0" @@ -23,9 +27,101 @@ class TLMetric(Metric): Metrics used in transfer learning. References: - [1] "A Survey on Transfer Learning", Pan et al., 2010 - [2] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 + - [1] "A Survey on Transfer Learning", Pan et al., 2010 + - [2] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 """ def __init__(self): + """ + Initialize the transfer learning metric. + """ super(TLMetric, self).__init__() + + +class JumpstartMetric(TLMetric): + r"""Jumpstart metric + + The jumpstart metric measures how much the initial performance of an agent in a target task may be improved by + transferring knowledge from a source task. [1] + + References: + - [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 + """ + + def __init__(self): + """ + Initialize the jumpstart metric. + """ + super(JumpstartMetric, self).__init__() + + +class AsymptoticPerformance(TLMetric): + r"""Asymptotic performance metric + + The asymptotic performance metric measures how much the final learned performance of an agent in the target task + has improved via transfer. [1] + + References: + - [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 + """ + + def __init__(self): + """ + Initialize the asymptotic performance metric. + """ + super(AsymptoticPerformance, self).__init__() + + +class TotalRewardMetric(TLMetric): + r"""Total reward metric + + The total reward metric measures the total reward accumulated by a learning agent. This one may be improved if + knowledge transfer was used. [1] + + References: + - [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 + """ + + def __init__(self): + """ + Initialize the total reward metric. + """ + super(TotalRewardMetric, self).__init__() + + +class TransferRatioMetric(TLMetric): + r"""Transfer ratio metric + + The transfer ratio metric measures "the ratio of the total reward accumulated by the transfer learner and the total + reward accumulated by the non-transfer learner". [1] + + References: + - [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 + """ + + def __init__(self): + """ + Initialize the transfer ratio metric. + """ + super(TransferRatioMetric, self).__init__() + + +class TimeToThresholdMetric(TLMetric): + r"""Time to threshold metric + + The time to threshold metric measures how much the learning time needed by the agent to perform a pre-specified + performance level (i.e. the threshold) is reduced via knowledge transfer. [1] + + References: + - [1] "Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 + """ + + def __init__(self, threshold): + """ + Initialize the time to threshold metric. + + Args: + threshold (float): threshold performance level. + """ + super(TimeToThresholdMetric, self).__init__() + self._threshold = threshold diff --git a/pyrobolearn/optimizers/torch_optimizer.py b/pyrobolearn/optimizers/torch_optimizer.py index d258213..73ab4a1 100644 --- a/pyrobolearn/optimizers/torch_optimizer.py +++ b/pyrobolearn/optimizers/torch_optimizer.py @@ -130,7 +130,7 @@ class Adagrad(Optimizer): # optimize self.optimizer.zero_grad() - loss.backward() + loss.backward(retain_graph=True) if self.max_grad_norm is not None: nn.utils.clip_grad_norm_(params, self.max_grad_norm) self.optimizer.step() @@ -165,7 +165,7 @@ class RMSprop(Optimizer): # optimize self.optimizer.zero_grad() - loss.backward() + loss.backward(retain_graph=True) if self.max_grad_norm is not None: nn.utils.clip_grad_norm_(params, self.max_grad_norm) self.optimizer.step() @@ -198,7 +198,7 @@ class SGD(Optimizer): # optimize self.optimizer.zero_grad() - loss.backward() + loss.backward(retain_graph=True) if self.max_grad_norm is not None: nn.utils.clip_grad_norm_(params, self.max_grad_norm) self.optimizer.step() diff --git a/pyrobolearn/returns/estimators.py b/pyrobolearn/returns/estimators.py index 4da2667..c6cf7e0 100644 --- a/pyrobolearn/returns/estimators.py +++ b/pyrobolearn/returns/estimators.py @@ -20,7 +20,7 @@ from pyrobolearn.values import Value, QValue __author__ = "Brian Delhaisse" __copyright__ = "Copyright 2018, PyRoboLearn" __credits__ = ["Brian Delhaisse"] -__license__ = "GNU GPLv3" +__license__ = "MIT" __version__ = "1.0.0" __maintainer__ = "Brian Delhaisse" __email__ = "briandelhaisse@gmail.com" @@ -166,9 +166,9 @@ class Estimator(object): # Operators # ############# - def __repr__(self): - """Return a representation string of the object.""" - return self.__class__.__name__ + # def __repr__(self): + # """Return a representation string of the object.""" + # return self.__class__.__name__ def __str__(self): """Return a string describing the object.""" diff --git a/pyrobolearn/robots/legged_robot.py b/pyrobolearn/robots/legged_robot.py index ff88b75..24ae889 100644 --- a/pyrobolearn/robots/legged_robot.py +++ b/pyrobolearn/robots/legged_robot.py @@ -189,6 +189,13 @@ class LeggedRobot(Robot): Implications", Popovic et al., 2005 """ if floor_id is not None: + + cop_key = 'cop_' + str(floor_id) + + # checked if already cached + if cop_key in self._state: + return self._state[cop_key] + # get contact points between the robot's links and the floor points = self.sim.get_contact_points(body1=self.id, body2=floor_id) @@ -204,6 +211,9 @@ class LeggedRobot(Robot): cop = forces * positions / np.sum(forces) cop = np.sum(cop, axis=0) + # cache it + self._state[cop_key] = cop + return cop # check if there are force/pressure sensors at the links/joints @@ -277,6 +287,13 @@ class LeggedRobot(Robot): # if the floor id is given, use the simulator to compute the ZMP (using the contact points) if floor_id is not None: + + zmp_key = 'zmp_' + str(floor_id) + + # checked if already cached + if zmp_key in self._state: + return self._state[zmp_key] + # get contact points between the robot's links and the floor points = self.sim.get_contact_points(body1=self.id, body2=floor_id) @@ -311,6 +328,9 @@ class LeggedRobot(Robot): zmp[0] += -forces[0]/forces[2] * self.com[2] - moments[1]/forces[2] zmp[1] += -forces[1]/forces[2] * self.com[2] + moments[0]/forces[2] + # cache it + self._state[zmp_key] = zmp + # return ZMP return zmp @@ -382,6 +402,13 @@ class LeggedRobot(Robot): self.get_center_of_mass_position() if floor_id is not None: + + cmp_key = 'cmp_' + str(floor_id) + + # checked if already cached + if cmp_key in self._state: + return self._state[cmp_key] + # get contact points between the robot's links and the floor points = self.sim.get_contact_points(body1=self.id, body2=floor_id) @@ -409,6 +436,9 @@ class LeggedRobot(Robot): cmp[0] -= forces[0] / forces[2] * self.com[2] cmp[1] -= forces[1] / forces[2] * self.com[2] + # cache it + self._state[cmp_key] = cmp + # return CMP return cmp @@ -734,6 +764,33 @@ class LeggedRobot(Robot): self.sim.remove_body(self.fri_visual) self.fri_visual = None + def update_visuals(self): # TODO: finish this + """ + Update all visuals. + """ + # update robot visuals + super(LeggedRobot, self).update_visual() + + # update support polygon + + # update friction cones/pyramids + + # update cop + if self.cop_visual is not None: + self.draw_cop() + + # update zmp + if self.zmp_visual is not None: + self.draw_zmp() + + # update cmp + if self.cmp_visual is not None: + self.draw_cmp() + + # update fri + if self.fri_visual is not None: + self.draw_fri() + class BipedRobot(LeggedRobot): r"""Biped Robot diff --git a/pyrobolearn/robots/robot.py b/pyrobolearn/robots/robot.py index cd851ba..9969650 100644 --- a/pyrobolearn/robots/robot.py +++ b/pyrobolearn/robots/robot.py @@ -58,7 +58,8 @@ class Robot(ControllableBody): - [9] "Motion Planning and Control of Dynamic Humanoid Locomotion" (PhD thesis), Xin, 2018 """ - def __init__(self, simulator, urdf, position=None, orientation=None, fixed_base=False, scale=1., *args, **kwargs): + def __init__(self, simulator, urdf, position=None, orientation=None, fixed_base=False, scale=1., visual_ticks=12, + *args, **kwargs): """ Initialize the robot. @@ -69,6 +70,7 @@ class Robot(ControllableBody): orientation (np.array[4]): initial orientation represented as a quaternion (x,y,z,w). fixed_base (bool, None): if True, the base of the robot will be fixed. scale (float): scaling factor. + visual_ticks (int): the number of ticks to sleep before updating the visuals. """ # check parameters if position is None: @@ -144,6 +146,8 @@ class Robot(ControllableBody): self._set_end_effectors() # visual debug: sliders and drawing + self.visual_ticks = visual_ticks + self._visual_cnt = 0 self.joint_sliders = {} self.com_visual = None self.projected_com_visual = None @@ -298,6 +302,18 @@ class Robot(ControllableBody): # sense self.sense() + # update visuals + if self._visual_cnt % self.visual_ticks == 0: + self._visual_cnt = 0 + + # update joint sliders + self.update_joint_slider() + + # update all visuals + self.update_visuals() + + self._visual_cnt += 1 + def sense(self): """Run all the sensors.""" for sensors in self.sensors.itervalues(): @@ -545,9 +561,12 @@ class Robot(ControllableBody): Return the center of mass position. Returns: - np.array[3]: center of mass position + np.array[3]: center of mass position [m] """ + if 'com' in self._state: + return self._state['com'] self.com = self.sim.get_center_of_mass_position(self.id) + self._state['com'] = self.com return self.com # alias @@ -558,13 +577,55 @@ class Robot(ControllableBody): Return the center of mass velocity. Returns: - np.array[3]: center of mass velocity + np.array[3]: center of mass velocity [m/s] """ - return self.sim.get_center_of_mass_velocity(self.id) + if 'com_vel' in self._state: + return self._state['com_vel'][0] + com_vel = self.sim.get_center_of_mass_velocity(self.id) + self._state['com_vel'] = [com_vel, time.time()] + return com_vel # alias get_com_velocity = get_center_of_mass_velocity + def get_center_of_mass_acceleration(self): + """ + Return the center of mass acceleration. + + Returns: + np.array[3]: center of mass acceleration [m/s] + """ + # if already cached, return it + if 'com_acc' in self._state: + return self._state['com_acc'][0] + + # compute com velocity + self.get_center_of_mass_velocity() + + # if didn't find previous com velocity + if 'com_vel' not in self._prev_state: + acc = np.zeros(3) + self._state['com_acc'] = [acc, time.time()] + return acc + + # get current center of mass velocity and time + com_vel, t = self._state['com_vel'] + + # retrieve previous joint velocities and time + com_vel_prev, t_prev = self._prev_state['com_vel'] + + # compute time finite difference + if self.sim.use_real_time(): # if the simulator is in real-time mode + dt = (t - t_prev) + else: # if we are stepping in the simulator + dt = self.sim.timestep + + # compute com acceleration using finite difference, and cache it + acc = (com_vel - com_vel_prev) / dt + self._state['com_acc'] = [acc, t] + + return acc + # def get_linear_momentum(self): # """ # Compute the linear momentum around the center of mass. @@ -1061,7 +1122,7 @@ class Robot(ControllableBody): # retrieve previous joint velocities and time dq_prev, t_prev = self._prev_state['dq'] - # compute time difference + # compute time finite difference if self.sim.use_real_time(): # if the simulator is in real-time mode dt = (t - t_prev) else: # if we are stepping in the simulator @@ -4531,11 +4592,27 @@ class Robot(ControllableBody): self.sim.change_visual_shape(self.id, shapeId, rgba_color=rgba) # print("Link {} - color: {}".format(link, rgba)) - def update_visual(self): + def update_visuals(self): # TODO: finish this """ Update all visuals. """ - pass + # update CoM + if self.com_visual is not None: + self.draw_com_position() + + # update projected CoM + if self.projected_com_visual is not None: + self.compute_and_draw_projected_com_position() + + # update each link's CoM + + # update each link frame + + # update each link bounding box + + # update each joint axis + + # update manipulability ellipsoids def compute_and_draw_com_position(self, radius=0.05, color=(1, 0, 0, 0.8)): """ diff --git a/pyrobolearn/samplers/sampler.py b/pyrobolearn/samplers/sampler.py index dfe9e79..193668b 100644 --- a/pyrobolearn/samplers/sampler.py +++ b/pyrobolearn/samplers/sampler.py @@ -51,29 +51,70 @@ class StorageSampler(Sampler): Sampler used with the storage. """ - def __init__(self, storage, sampler=None, num_batches=10): + def __init__(self, storage, sampler=None, num_batches=10, batch_size=None, batch_size_bounds=None, + replacement=True): """ Initialize the storage sampler. Args: - storage (RolloutStorage): rollout storage. + storage (Storage): storage sampler. sampler (Sampler, None): If None, it will use a sampler that randomly sample batches of the storage. It will by default sample :attr:`num_batches`. - num_batches (int): number of batches + num_batches (int): number of batches. + batch_size (int, None): size of the batch. If None, it will be computed based on the size of the storage, + where batch_size = size(storage) // num_batches. Note that the batch size must be smaller than the size + of the storage itself. The num_batches * batch_size can however be bigger than the storage size if + :attr:`replacement = True`. + batch_size_bounds (tuple of int, None): if :attr:`batch_size` is None, we can instead specify the lower + and upper bounds for the `batch_size`. For instance, we can set it to `(16, 128)` along with + `batch_size=None`; this will result to compute `batch_size = size(storage) // num_batches` but if this + one is too small (<16), it will be set to 16, and if this one is too big (>128), it will be set to 128. + replacement (bool): if we should sample each element only one time, or we can sample the same ones multiple + times. """ # set the storage self.storage = storage + # set variables + self._num_batches = num_batches + self._replacement = bool(replacement) + self._batch_size_bounds = batch_size_bounds + self._batch_size_given = batch_size is not None + # set the sampler if sampler is None: - batch_size = self.size // num_batches - if batch_size > self.size: - raise ValueError("Expecting the batch size (={}) to be smaller than the size of the storage (={})" - ".".format(batch_size, self.size)) - sampler = torch_sampler.BatchSampler(sampler=torch_sampler.SubsetRandomSampler(range(self.size)), - batch_size=batch_size, drop_last=True) + + # check batch size and compute it if necessary + if batch_size is None: + batch_size = self.size // num_batches + + # check batch size bounds + if isinstance(batch_size_bounds, (tuple, list)) and len(batch_size_bounds) == 2: + if batch_size < batch_size_bounds[0]: + batch_size = batch_size_bounds[0] + elif batch_size > batch_size_bounds[1]: + batch_size = batch_size_bounds[1] + + # check the batch size * number of batches wrt the storage size + if batch_size * num_batches > self.size and not self.replacement: + raise ValueError("Expecting the batch size (={}) * num_batches (={}) to be smaller than the size of " + "the storage (={}), if we can not use replacement.".format(batch_size, num_batches, + self.size)) + + # subsampler + if replacement: + subsampler = torch_sampler.RandomSampler(data_source=range(self.size), replacement=replacement, + num_samples=self.size) + else: + subsampler = torch_sampler.SubsetRandomSampler(indices=range(self.size)) + + # create sampler + sampler = torch_sampler.BatchSampler(sampler=subsampler, batch_size=batch_size, drop_last=True) + self.sampler = sampler + print("Sampler: size: {} - num batches: {} - batch size: {}".format(self.size, num_batches, self.batch_size)) + ############## # Properties # ############## @@ -96,6 +137,11 @@ class StorageSampler(Sampler): """Return the total size of the storage number of time steps * number of processes.""" return self.storage.size + @property + def filled_size(self): + """Return the filled size of the storage.""" + return self.storage.filled_size + @property def sampler(self): """Return the sampler.""" @@ -115,22 +161,52 @@ class StorageSampler(Sampler): return self.sampler.batch_size @batch_size.setter - def batch_size(self, size): + def batch_size(self, batch_size): """Set the batch size.""" - if size > self.size: - raise ValueError("Expecting the batch size (={}) to be smaller than the size of the storage (={})" - ".".format(size, self.size)) - self.sampler.batch_size = size + # check the batch size * number of batches wrt the storage size + if batch_size * self._num_batches > self.size and not self.replacement: + raise ValueError("Expecting the batch size (={}) * num_batches (={}) to be smaller than the size of " + "the storage (={}), if we can not use replacement.".format(batch_size, self._num_batches, + self.size)) + + # check batch size bounds + if isinstance(self._batch_size_bounds, (tuple, list)) and len(self._batch_size_bounds) == 2: + if batch_size < self._batch_size_bounds[0]: + batch_size = self._batch_size_bounds[0] + elif batch_size > self._batch_size_bounds[1]: + batch_size = self._batch_size_bounds[1] + + # set the batch size for the sampler + self.sampler.batch_size = batch_size @property def num_batches(self): """Return the number of batches (based on the size of the storage and the batch size).""" - return self.size // self.batch_size + # return self.filled_size // self.batch_size + return self._num_batches @num_batches.setter def num_batches(self, num_batches): """Set the number of batches.""" - self.batch_size = self.size // num_batches + self._num_batches = num_batches + if not self._batch_size_given: + self.batch_size = self.filled_size // num_batches + else: + if self.batch_size * self._num_batches > self.size and not self.replacement: + raise ValueError("Expecting the batch size (={}) * num_batches (={}) to be smaller than the size of " + "the storage (={}), if we can not use replacement.".format(self.batch_size, + self._num_batches, + self.size)) + + @property + def replacement(self): + """Return the replacement boolean.""" + return self._replacement + + @property + def batch_size_bounds(self): + """Return the batch size bounds.""" + return self._batch_size_bounds ########### # Methods # @@ -142,15 +218,67 @@ class StorageSampler(Sampler): Args: num_batches (int): number of batches """ - self.batch_size = self.size // num_batches + # get filled size of the storage + size = self.filled_size + + # get batch size + self.batch_size = size // num_batches + + # check if there is a sub-sampler + if hasattr(self.sampler, 'sampler'): + if hasattr(self.sampler.sampler, 'data_source'): + self.sampler.sampler.data_source = range(size) + elif hasattr(self.sampler.sampler, 'indices'): + self.sampler.sampler.indices = range(size) + else: + if hasattr(self.sampler, 'data_source'): + self.sampler.data_source = range(size) + elif hasattr(self.sampler, 'indices'): + self.sampler.indices = range(size) for indices in self.sampler: yield self.storage.get_batch(indices) def __iter__(self): """Iterate over the storage.""" - for indices in self.sampler: - yield self.storage.get_batch(indices) + # get the filled size + size = self.filled_size + + print("Storage size: {}".format(self.size)) + print("Storage filled size: {}".format(size)) + + # modify the sampler (by changing the size) + # check if there is a sub-sampler + if hasattr(self.sampler, 'sampler'): + + # change the size of the sampler + if hasattr(self.sampler.sampler, 'data_source'): + self.sampler.sampler.data_source = range(size) + elif hasattr(self.sampler.sampler, 'indices'): + self.sampler.sampler.indices = range(size) + + # compute the batch size if specified + if not self._batch_size_given: + self.batch_size = size // self.num_batches + + else: + # change the size of the sampler + if hasattr(self.sampler, 'data_source'): + self.sampler.data_source = range(size) + elif hasattr(self.sampler, 'indices'): + self.sampler.indices = range(size) + + # provide the batches + batch_idx = 0 + while True: # this is to account for replacement = True + for indices in self.sampler: + batch_idx += 1 + yield self.storage.get_batch(indices) + if batch_idx >= self.num_batches: + break + + if batch_idx >= self.num_batches: + break class BatchRandomSampler(StorageSampler): @@ -158,12 +286,23 @@ class BatchRandomSampler(StorageSampler): """ - def __init__(self, storage, num_batches=10): + def __init__(self, storage, num_batches=10, batch_size=None, batch_size_bounds=None, replacement=True): """ Initialize the storage sampler. Args: - storage (RolloutStorage): rollout storage. - num_batches (int): number of batches + storage (Storage): storage to sample the batches from. + num_batches (int): number of batches. + batch_size (int, None): size of the batch. If None, it will be computed based on the size of the storage, + where batch_size = size(storage) // num_batches. Note that the batch size must be smaller than the size + of the storage itself. The num_batches * batch_size can however be bigger than the storage size if + :attr:`replacement = True`. + batch_size_bounds (tuple of int, None): if :attr:`batch_size` is None, we can instead specify the lower + and upper bounds for the `batch_size`. For instance, we can set it to `(16, 128)` along with + `batch_size=None`; this will result to compute `batch_size = size(storage) // num_batches` but if this + one is too small (<16), it will be set to 16, and if this one is too big (>128), it will be set to 128. + replacement (bool): if we should sample each element only one time, or we can sample the same ones multiple + times. """ - super(BatchRandomSampler, self).__init__(storage, num_batches=num_batches) + super(BatchRandomSampler, self).__init__(storage=storage, num_batches=num_batches, batch_size=batch_size, + batch_size_bounds=batch_size_bounds, replacement=replacement) diff --git a/pyrobolearn/storages/__init__.py b/pyrobolearn/storages/__init__.py index e4a3f7f..dfeb0f0 100644 --- a/pyrobolearn/storages/__init__.py +++ b/pyrobolearn/storages/__init__.py @@ -1,7 +1,6 @@ -# import memory +# import storages from .storage import * # import experience replay memories/storages from .er import * - diff --git a/pyrobolearn/storages/er.py b/pyrobolearn/storages/er.py index 07f0a02..0a112cb 100644 --- a/pyrobolearn/storages/er.py +++ b/pyrobolearn/storages/er.py @@ -208,6 +208,13 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): # return self.position return self.capacity + @property + def filled_size(self): + """Return the filled sized of the experience replay storage.""" + if self.full: + return self.capacity + return self.position + ########### # Methods # ########### @@ -364,7 +371,7 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): """Return a batch of the experience replay storage in the form of a `DictStorage`. Args: - indices (list of int): indices. Each index must be between 0 and `self.size`. + indices (list of int): indices. Each index must be between 0 and `self.filled_size`. Returns: DictStorage / Batch: batch containing a part of the storage. Variables such as `states`, `actions`, diff --git a/pyrobolearn/storages/storage.py b/pyrobolearn/storages/storage.py index 4fd8198..64b001d 100644 --- a/pyrobolearn/storages/storage.py +++ b/pyrobolearn/storages/storage.py @@ -32,19 +32,24 @@ __status__ = "Development" class Storage(object): """Main abstract storage class.""" - def save(self, filename): - """Save the storage on the disk.""" - pickle.dump(self, open(filename, 'wb')) + @property + def size(self): + """Return the size of the storage. Need to be implemented in the child class.""" + return 0 + + @property + def filled_size(self): + """Return the filled size of the storage.""" + return self.size @staticmethod def load(filename): """Load the storage from the disk.""" return pickle.load(open(filename, 'r')) - @property - def size(self): - """Return the size of the storage. Need to be implemented in the child class.""" - return 0 + def save(self, filename): + """Save the storage on the disk.""" + pickle.dump(self, open(filename, 'wb')) def get_batch(self, indices): """Return a batch of the storage as a `Storage` type. @@ -624,7 +629,7 @@ class Batch(DictStorage): are filled by the exploration phase in RL algorithms (see `pyrobolearn/algos/explorer`). """ - def __init__(self, kwargs=None, device=None, dtype=None): + def __init__(self, kwargs=None, device=None, dtype=None, size=None): """ Initialize the Batch storage. @@ -635,12 +640,19 @@ class Batch(DictStorage): to which the tensor is allocated. dtype (torch.dtype, None): convert the `torch.Tensor` to the specified data type. If None, it will keep the original dtype + size (int, None): size of the batch. """ super(Batch, self).__init__(kwargs=kwargs, device=device, dtype=dtype, update=False) # contains the current values evaluated during the update phase of RL algorithms. self.current = DictStorage(kwargs={}, device=device, dtype=dtype, update=False) # indices where the masks is different from 0 in the current batch self.indices = None + self._size = size if size is not None else len(kwargs['masks']) # TODO: need to generalize this + + @property + def size(self): + """Return the size of the batch.""" + return self._size def get_current(self, key, default=None): """Try first to get the key from :attr:`current`, if not present, try to get it from the batch storage. @@ -728,6 +740,14 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies: """Return the size (=number of steps * number of processes) of the rollout storage.""" return self._num_steps * self._num_trajectories + @property + def filled_size(self): + """Return the filled size by looking at the entries where the mask = 1.""" + # print("Mask: {}".format(self.masks[:, :, 0])) + # print("Mask shape: {}".format(self.masks.shape)) + # print("Steps: {}".format(self._step)) + return np.minimum(len(self.masks[self.masks == 1]), self.size) + @property def capacity(self): """Return the capacity of the rollout storage (=number of steps * number of processes).""" @@ -760,7 +780,7 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies: `self.num_trajectories`. """ # if end of storage, go at the beginning - self._step[rollout_idx] = (self._step[rollout_idx] + 1) % self.num_steps + self._step[rollout_idx] = (self._step[rollout_idx] + 1) % (self.num_steps + 1) # self.num_steps def create_new_entry(self, key, shapes, num_steps=None, dtype=torch.dtype): """Create a new entry (=tensor) in the rollout storage dictionary. The tensor will have the dimension @@ -881,7 +901,8 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies: self._step[rollout_idx] = 0 # reset masks - self.masks[:, rollout_idx].copy_(torch.ones_like(self.masks[:, rollout_idx])) + # self.masks[:, rollout_idx].copy_(torch.ones_like(self.masks[:, rollout_idx])) # fill with ones + self.masks[:, rollout_idx].copy_(torch.zeros_like(self.masks[:, rollout_idx])) # fill with zeros # insert initial states if init_states is None: @@ -972,6 +993,13 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies: """ t = self._step[rollout_idx] + print("\nStorage: rollout = {}, step = {}".format(rollout_idx, t)) + print("Storage: insert state: {}".format(states)) + print("Storage: insert action: {}".format(actions)) + print("Storage: insert next state: {}".format(next_states)) + print("Storage: insert reward: {}".format(reward)) + print("Storage: insert mask: {}".format(mask)) + # check given observations/states and actions if not isinstance(next_states, list): next_states = [next_states] @@ -1005,7 +1033,7 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies: # update the step if specified if update_step: - self.step() + self.step(rollout_idx=rollout_idx) def add_trajectory(self, trajectory, rollout_idx=0): r""" @@ -1030,28 +1058,54 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies: """Return a batch of the Rollout storage in the form of a `DictStorage`. Args: - indices (list of int): indices. Each index must be between 0 and `num_steps * num_trajectories`. + indices (list of int): indices. Each index must be between 0 and `self.filled_size`-1. Returns: DictStorage / Batch: batch containing a part of the storage. Variables such as `states`, `actions`, `rewards`, `masks`, and others can be accessed from the object. """ - # In the next comments, T = number of time steps, P = number of processes, and I = number of indices + # In the next comments, T = number of time steps, P = number of processes, and I = number of indices, F = + # number of filled entries (where the mask == 1) batch = {} + size = len(indices) + + # # The following sampling method was not optimal as it would sample entries where the mask == 0 (i.e. when + # # the episode is over) + # def sample(item, indices): + # if isinstance(item, torch.Tensor): + # if len(item) == self.num_steps + 1: # = T+1 + # item = item[:-1] # take only the T steps + # return item.view(-1, *item.size()[2:])[indices] # reshape to (T*P, *shape) and from T*P takes I + # + # elif isinstance(item, np.ndarray): + # if len(item) == self.num_steps + 1: # = T+1 + # item = item[:-1] # take only the T steps + # return item.reshape(-1, *item.shape[2:])[indices] # reshape to (T*P, *shape) and from T*P takes I + + print("Indices: {} - length: {}".format(indices, len(indices))) def sample(item, indices): - if isinstance(item, torch.Tensor): - if len(item) == self.num_steps + 1: # = T+1 - item = item[:-1] # take only the T steps - return item.view(-1, *item.size()[2:])[indices] # reshape to (T*P, *shape) and from T*P takes I - - elif isinstance(item, np.ndarray): - if len(item) == self.num_steps + 1: # = T+1 - item = item[:-1] # take only the T steps - return item.reshape(-1, *item.shape[2:])[indices] # reshape to (T*P, *shape) and from T*P takes I + """Given indices where each index is between 0 and `self.filled_size` (=number of masks that are equal + to 1), it returns the corresponding entries in the item. + """ + if isinstance(item, torch.Tensor): # TODO: improve performance of this + masks = (self.masks[:, :, 0] == 1).nonzero() # [F, 2] --> indices for (steps, trajs) + masks[:, 0] -= 1 # remove 1 because indices for masks are in [1, t+1] --> [0, t] + indices = masks[indices] # [I,2] --> allowed indices for (steps, trajs) + print("Torch mask length: {}".format(len(masks))) + print("Indices: {}".format(indices)) + return item[indices[:, 0], indices[:, 1]] # [I, *shape] + elif isinstance(item, np.ndarray): # TODO: improve performance of this + masks = np.vstack(((self.masks[:, :, 0] == 1).nonzero())) # [F,2] --> indices for (steps, trajs) + masks[:, 0] -= 1 # remove 1 because indices for masks are in [1, t+1] --> [0, t] + indices = masks[indices] # [I,2] --> allowed indices for (steps, trajs) + print("Numpy Mask length: {}".format(len(masks))) + print("Indices: {}".format(indices)) + return item[indices[:, 0], indices[:, 1]] # [I, *shape] # go through each attribute and sample from the tensors for key, value in self.iteritems(): + print("batch - add key: {}".format(key)) if isinstance(value, list): # value = list of tensors batch[key] = [sample(val, indices) for val in value] else: # value = tensor @@ -1074,7 +1128,7 @@ class RolloutStorage(DictStorage): # TODO: think about when multiple policies: distributions[i] = dist.__class__.from_list(distribution) # create Batch object - batch = Batch(batch, device=self.device, dtype=self.dtype) + batch = Batch(batch, device=self.device, dtype=self.dtype, size=size) batch.indices = torch.tensor(range(len(indices)))[batch['masks'][:, 0] != 0].tolist() # return batch (which is given to the updater (and loss)) diff --git a/pyrobolearn/tasks/transfer.py b/pyrobolearn/tasks/transfer.py index 546c996..81f5bd8 100644 --- a/pyrobolearn/tasks/transfer.py +++ b/pyrobolearn/tasks/transfer.py @@ -7,7 +7,7 @@ from pyrobolearn.tasks.task import Task __author__ = "Brian Delhaisse" __copyright__ = "Copyright 2018, PyRoboLearn" __credits__ = ["Brian Delhaisse"] -__license__ = "GNU GPLv3" +__license__ = "MIT" __version__ = "1.0.0" __maintainer__ = "Brian Delhaisse" __email__ = "briandelhaisse@gmail.com" @@ -21,8 +21,8 @@ class TLTask(object): different but similar problem [1,2]. References: - [1] "A Survey on Transfer Learning", Pan et al., 2010 - [2]" Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 + - [1] "A Survey on Transfer Learning", Pan et al., 2010 + - [2]" Transfer Learning for Reinforcement Learning Domains: A Survey", Taylor et al., 2009 """ def __init__(self, domain_task, target_task):