From fae7c57a86a034f4271616a12ac956bf9a17b4e4 Mon Sep 17 00:00:00 2001 From: Brian Delhaisse Date: Mon, 15 Apr 2019 14:35:08 +0200 Subject: [PATCH] refactor storages: change observations to states + minor details --- pyrobolearn/dynamics/nn_dynamic.py | 4 +- pyrobolearn/exploration/exploration.py | 2 +- pyrobolearn/storages/er.py | 54 +++++++++++++------------- pyrobolearn/storages/storage.py | 54 +++++++++++++------------- 4 files changed, 58 insertions(+), 56 deletions(-) diff --git a/pyrobolearn/dynamics/nn_dynamic.py b/pyrobolearn/dynamics/nn_dynamic.py index a1b69c1..ccd7b41 100644 --- a/pyrobolearn/dynamics/nn_dynamic.py +++ b/pyrobolearn/dynamics/nn_dynamic.py @@ -81,8 +81,8 @@ class MLPDynamicModel(NNDynamicModel): if next_states is None: next_states = states model = MLPApproximator(inputs=[states, actions], outputs=next_states, hidden_units=hidden_units, - activation_fct=activation_fct, last_activation_fct=last_activation_fct, - dropout_prob=dropout_prob) + activation=activation_fct, last_activation=last_activation_fct, + dropout=dropout_prob) super(MLPDynamicModel, self).__init__(states, actions, model=model, next_states=next_states, distributions=distributions, preprocessors=preprocessors, postprocessors=postprocessors) diff --git a/pyrobolearn/exploration/exploration.py b/pyrobolearn/exploration/exploration.py index 51333d2..1ad7d0a 100644 --- a/pyrobolearn/exploration/exploration.py +++ b/pyrobolearn/exploration/exploration.py @@ -24,7 +24,7 @@ __email__ = "briandelhaisse@gmail.com" __status__ = "Development" -class Exploration(object): +class Exploration(object): # TODO: inherit from Policy? r"""Exploration class Exploration strategy: it wraps the policy, and defines how the policy should explore in the environment. diff --git a/pyrobolearn/storages/er.py b/pyrobolearn/storages/er.py index b570a17..26431a0 100644 --- a/pyrobolearn/storages/er.py +++ b/pyrobolearn/storages/er.py @@ -133,7 +133,7 @@ __status__ = "Development" # indices (list of int): indices. Each index must be between 0 and `self.size`. # # Returns: -# DictStorage / Batch: batch containing a part of the storage. Variables such as `observations`, `actions`, +# DictStorage / Batch: batch containing a part of the storage. Variables such as `states`, `actions`, # `rewards`, `next_states`, `masks`, and others can be accessed from the object. # """ # # create batch @@ -145,7 +145,7 @@ __status__ = "Development" # # for idx in indices: # transition = self.memory[idx] -# batch.setdefault('observations', []).append(transition[0]) +# batch.setdefault('states', []).append(transition[0]) # batch.setdefault('actions', []).append(transition[1]) # batch.setdefault('next_states', []).append(transition[2]) # batch.setdefault('rewards', []).append(transition[3]) @@ -176,17 +176,17 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): [2] "Playing Atari with Deep Reinforcement Learning", Mnih et al., 2013 """ - def __init__(self, observation_shapes, action_shapes, capacity=10000, *args, **kwargs): + def __init__(self, state_shapes, action_shapes, capacity=10000, *args, **kwargs): """ Initialize the experience replay storage. Args: - observation_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an observation. + state_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an observation/state. action_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an action. capacity (int): maximum size of the experience replay storage. """ # recurrent_hidden_state_size (int): size of the internal state - print("\nStorage: observation shape: {}".format(observation_shapes)) + print("\nStorage: state shape: {}".format(state_shapes)) print("Storage: action shape: {}".format(action_shapes)) super(ExperienceReplay, self).__init__() self.position = 0 @@ -194,7 +194,7 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): raise ValueError("Expecting the capacity to be bigger than 0, instead got: {}".format(capacity)) self.capacity = capacity self.full = False - self.init(observation_shapes, action_shapes) + self.init(state_shapes, action_shapes) ############## # Properties # @@ -254,22 +254,22 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): raise TypeError("Expecting the given shapes {} to be a list of tuple of int, a tuple of int, or an int, " "instead got: {}".format({}, type(shapes))) - def init(self, observation_shapes, action_shapes): + def init(self, state_shapes, action_shapes): """ Initialize the experience replay storage by allocating the appropriate tensors for the observations (states), actions, next states, rewards, and masks. Args: - observation_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an observation. + state_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an observation/state. action_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an action. """ # clear itself: remove all items from the DictStorage, and reset all variables self.reset() # allocate space for observations / states - if not isinstance(observation_shapes, list): - observation_shapes = [observation_shapes] - self.create_new_entry('observations', shapes=observation_shapes) + if not isinstance(state_shapes, list): + state_shapes = [state_shapes] + self.create_new_entry('states', shapes=state_shapes) # allocate space for actions if not isinstance(action_shapes, list): @@ -277,9 +277,9 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): self.create_new_entry('actions', shapes=action_shapes) # allocate space for next observations / states # TODO: improve this, don't copy, just keep reference - if not isinstance(observation_shapes, list): - observation_shapes = [observation_shapes] - self.create_new_entry('next_states', shapes=observation_shapes) + if not isinstance(state_shapes, list): + state_shapes = [state_shapes] + self.create_new_entry('next_states', shapes=state_shapes) # allocate space for rewards self.create_new_entry('rewards', shapes=1) @@ -297,37 +297,38 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): def clear(self): """Clear the experience replay storage.""" self.full = False - super(ExperienceReplayStorage, self).clear() + # super(ExperienceReplayStorage, self).clear() + super(ExperienceReplay, self).clear() - def insert(self, observations, actions, reward, mask, next_states, **kwargs): + def insert(self, states, actions, reward, mask, next_states, **kwargs): """ Insert the given parameters into the storage. Args: - observations (torch.Tensor, list of torch.Tensor): (list of) state(s) / observation(s). + states (torch.Tensor, list of torch.Tensor): (list of) state(s) / observation(s). actions (torch.Tensor, list of torch.Tensor): (list of) action(s). reward (float, int, torch.Tensor): reward value. mask (float, int, torch.Tensor): masks. They are set to zeros after an episode has terminated. next_states (torch.Tensor, list of torch.Tensor): (list of) next state(s) / observation(s). **kwargs (dict): kwargs """ - print("ER - insert observation: {}".format(observations)) + print("ER - insert state: {}".format(states)) print("ER - insert action: {}".format(actions)) print("ER - insert next state: {}".format(next_states)) print("ER - insert reward: {}".format(reward)) print("ER - insert mask: {}".format(mask)) - # check given observations and actions - if not isinstance(observations, list): - observations = [observations] + # check given states/observations and actions + if not isinstance(states, list): + states = [states] if not isinstance(actions, list): actions = [actions] if not isinstance(next_states, list): next_states = [next_states] - # insert each observation / action / next states - for observation, storage in zip(observations, self.observations): - storage[self.position].copy_(self._convert_to_tensor(observation)) + # insert each states / actions / next states + for state, storage in zip(states, self.states): + storage[self.position].copy_(self._convert_to_tensor(state)) for action, storage in zip(actions, self.actions): storage[self.position].copy_(self._convert_to_tensor(action)) for next_state, storage in zip(next_states, self.next_states): @@ -350,7 +351,7 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): indices (list of int): indices. Each index must be between 0 and `self.size`. Returns: - DictStorage / Batch: batch containing a part of the storage. Variables such as `observations`, `actions`, + DictStorage / Batch: batch containing a part of the storage. Variables such as `states`, `actions`, `next_states`, `rewards`, and `masks`. """ batch = {} @@ -374,7 +375,8 @@ class ExperienceReplay(DictStorage): # ExperienceReplayStorage(DictStorage): # if not isinstance(key, str): # raise TypeError("The experience replay storage only accepts key as strings! Instead got: {} with type " # "{}".format(key, type(key))) - super(ExperienceReplayStorage, self).__setitem__(key, value) + # super(ExperienceReplayStorage, self).__setitem__(key, value) + super(ExperienceReplay, self).__setitem__(key, value) # def __setattr__(self, key, value): # """Set the attribute using the given key and value. That is, instead of `D[key] = value`, you can do diff --git a/pyrobolearn/storages/storage.py b/pyrobolearn/storages/storage.py index f6e5602..374a4f9 100644 --- a/pyrobolearn/storages/storage.py +++ b/pyrobolearn/storages/storage.py @@ -626,7 +626,7 @@ class RolloutStorage(DictStorage): key variable check that it is correctly present inside the storage. Nonetheless, having a dynamic rollout storage has its advantages, as you can for example store multiple value scalars from multiple value function approximators. - Also, in contrast to [1], we do not compute the returns / estimators here. This is done by the `Estimator` class + Also, in contrast to [1], we do not compute the returns / returns here. This is done by the `Estimator` class which takes as input a `RolloutStorage`. In PRL, this storage is notably used by `RLAlgo` (`Explorator`, `Evaluator`, `Updater`), `Loss`, `Estimators`, etc. @@ -635,26 +635,26 @@ class RolloutStorage(DictStorage): [1] https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail/blob/master/a2c_ppo_acktr/storage.py """ - def __init__(self, num_steps, observation_shapes, action_shapes, num_processes=1): + def __init__(self, num_steps, state_shapes, action_shapes, num_processes=1): # , recurrent_hidden_state_size=0): """ Initialize the rollout storage. Args: num_steps (int): number of steps in one episode - observation_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an observation. + state_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an observation/state. action_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an action. num_processes (int): number of processes """ # recurrent_hidden_state_size (int): size of the internal state - print("\nStorage: observation shape: {}".format(observation_shapes)) + print("\nStorage: state shape: {}".format(state_shapes)) print("Storage: action shape: {}".format(action_shapes)) super(RolloutStorage, self).__init__() self._step = 0 self._num_steps = int(num_steps) self._num_processes = int(num_processes) self._shifts = {} # dictionary that maps the key to the time shift; this is add to the current time step - self.init(self.num_steps, observation_shapes, action_shapes, self.num_processes) + self.init(self.num_steps, state_shapes, action_shapes, self.num_processes) ############## # Properties # @@ -748,7 +748,7 @@ class RolloutStorage(DictStorage): # add shift self._shifts[key] = num_steps - self.num_steps - def init(self, num_steps, observation_shapes, action_shapes, num_processes=1): + def init(self, num_steps, state_shapes, action_shapes, num_processes=1): """ Initialize the rollout storage by allocating the appropriate tensors for the observations (states), actions, rewards, masks, and returns. @@ -757,7 +757,7 @@ class RolloutStorage(DictStorage): Args: num_steps (int): number of time steps in the finite-horizon RL setting. - observation_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an observation. + state_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an observation/state. action_shapes (list of tuple of int, tuple of int): each tuple represents the shape of an action. num_processes (int): number of process. """ @@ -768,13 +768,13 @@ class RolloutStorage(DictStorage): self._num_processes = int(num_processes) # allocate space for observations / states - logger.debug('creating space for observations of shapes: {}'.format(observation_shapes)) - if not isinstance(observation_shapes, list): - observation_shapes = [observation_shapes] - self.create_new_entry('observations', shapes=observation_shapes, num_steps=self.num_steps+1) + logger.debug('creating space for states with shape: {}'.format(state_shapes)) + if not isinstance(state_shapes, list): + state_shapes = [state_shapes] + self.create_new_entry('states', shapes=state_shapes, num_steps=self.num_steps + 1) # allocate space for actions - logger.debug('creating space for actions of shapes: {}'.format(action_shapes)) + logger.debug('creating space for actions with shape: {}'.format(action_shapes)) if not isinstance(action_shapes, list): action_shapes = [action_shapes] self.create_new_entry('actions', shapes=action_shapes, num_steps=self.num_steps) @@ -796,7 +796,7 @@ class RolloutStorage(DictStorage): # space for log probabilities on policy, distributions, scalar values from value functions, # recurrent hidden states, and others have to be allocated outside the class - def reset(self, init_observations=None): + def reset(self, init_states=None): """Reset the storage by copying the last value and setting it to the first value.""" # for key, value in self.iteritems(): # if isinstance(value, list): @@ -805,13 +805,13 @@ class RolloutStorage(DictStorage): # item[0].copy_(item[-1]) # elif isinstance(value, torch.Tensor) and len(value) == self.num_steps + 1: # self[key][0].copy_(self[key][-1]) - if init_observations is None: - for observation in self.observations: - observation[0].copy_(observation[-1]) + if init_states is None: + for state in self.states: + state[0].copy_(state[-1]) else: - if not isinstance(init_observations, list): - init_observations = [init_observations] - for observation, value in zip(self.observations, init_observations): + if not isinstance(init_states, list): + init_states = [init_states] + for observation, value in zip(self.states, init_states): observation[0].copy_(self._convert_to_tensor(value)) self.masks[0].copy_(self.masks[-1]) # self.recurrent_hidden_states[0].copy_(self.recurrent_hidden_states[-1]) @@ -868,14 +868,14 @@ class RolloutStorage(DictStorage): else: set_tensor(self[key], step, values, copy=copy) - def insert(self, observations, actions, reward, mask, distributions=None, update_step=True, **kwargs): + def insert(self, states, actions, reward, mask, distributions=None, update_step=True, **kwargs): # distributions, values=None): # recurrent_hidden_state, action_log_prob): """ Insert the given parameters into the storage. Args: - observations (torch.Tensor, list of torch.Tensor): (list of) state(s) / observation(s). + states (torch.Tensor, list of torch.Tensor): (list of) state(s) / observation(s). actions (torch.Tensor, list of torch.Tensor): (list of) action(s) reward (float, int, torch.Tensor): reward value mask (float, int, torch.Tensor): masks. They are set to zeros after an episode has terminated. @@ -885,19 +885,19 @@ class RolloutStorage(DictStorage): **kwargs (dict): dictionary containing other parameters to update in the storage. The other parameters had to be added using the `create_new_entry()` method. """ - print("Storage - insert observation: {}".format(observations)) + print("Storage - insert state: {}".format(states)) print("Storage - insert action: {}".format(actions)) - # check given observations and actions - if not isinstance(observations, list): - observations = [observations] + # check given observations/states and actions + if not isinstance(states, list): + states = [states] if not isinstance(actions, list): actions = [actions] if not isinstance(distributions, list): distributions = [distributions] # insert each observation / action - for observation, storage in zip(observations, self.observations): + for observation, storage in zip(states, self.states): storage[self._step + 1].copy_(self._convert_to_tensor(observation)) for action, storage in zip(actions, self.actions): storage[self._step].copy_(self._convert_to_tensor(action)) @@ -930,7 +930,7 @@ class RolloutStorage(DictStorage): indices (list of int): indices. Each index must be between 0 and `num_steps * num_processes`. Returns: - DictStorage / Batch: batch containing a part of the storage. Variables such as `observations`, `actions`, + DictStorage / Batch: batch containing a part of the storage. Variables such as `states`, `actions`, `rewards`, `returns`, and others can be accessed from the object. """ # In the next comments, T = number of time steps, P = number of processes, and I = number of indices