wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-27 11:26:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

[rllib] Support discrete observation spaces such as FrozenLake-v0 (#1140)

Browse Source 
* add

* remove transform_shape

* fix test

* fix
This commit is contained in:
Eric Liang
2017-10-23 23:16:52 -07:00
committed by Richard Liaw
parent 0c9817fa76
commit cd9dc398ff
10 changed files with 98 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/rllib/es/es.py
+4 -10
View File
@@ -74,10 +74,7 @@ class Worker(object):
self.noise = SharedNoiseTable(noise)
self.env = env_creator()
self.preprocessor = ModelCatalog.get_preprocessor(
self.env.spec.id, self.env.observation_space.shape)
self.preprocessor_shape = self.preprocessor.transform_shape(
self.env.observation_space.shape)
self.preprocessor = ModelCatalog.get_preprocessor(self.env)
self.sess = utils.make_session(single_threaded=True)
self.policy = policies.GenericPolicy(
@@ -118,7 +115,7 @@ class Worker(object):
noise_inds, returns, sign_returns, lengths = [], [], [], []
# We set eps=0 because we're incrementing only.
task_ob_stat = utils.RunningStat(self.preprocessor_shape, eps=0)
task_ob_stat = utils.RunningStat(self.preprocessor.shape, eps=0)
# Perform some rollouts with noise.
task_tstart = time.time()
@@ -169,10 +166,7 @@ class ESAgent(Agent):
}
env = self.env_creator()
preprocessor = ModelCatalog.get_preprocessor(
env.spec.id, env.observation_space.shape)
preprocessor_shape = preprocessor.transform_shape(
env.observation_space.shape)
preprocessor = ModelCatalog.get_preprocessor(env)
self.sess = utils.make_session(single_threaded=False)
self.policy = policies.GenericPolicy(
@@ -180,7 +174,7 @@ class ESAgent(Agent):
**policy_params)
tf_util.initialize()
self.optimizer = optimizers.Adam(self.policy, self.config["stepsize"])
self.ob_stat = utils.RunningStat(preprocessor_shape, eps=1e-2)
self.ob_stat = utils.RunningStat(preprocessor.shape, eps=1e-2)
# Create the shared noise table.
print("Creating shared noise table.")
python/ray/rllib/es/policies.py
+6 -6
View File
@@ -144,25 +144,25 @@ class GenericPolicy(Policy):
def _initialize(self, ob_space, ac_space, preprocessor, ac_noise_std):
self.ac_space = ac_space
self.ac_noise_std = ac_noise_std
self.preprocessor_shape = preprocessor.transform_shape(ob_space.shape)
self.preprocessor = preprocessor
with tf.variable_scope(type(self).__name__) as scope:
# Observation normalization.
ob_mean = tf.get_variable(
'ob_mean', self.preprocessor_shape, tf.float32,
'ob_mean', self.preprocessor.shape, tf.float32,
tf.constant_initializer(np.nan), trainable=False)
ob_std = tf.get_variable(
'ob_std', self.preprocessor_shape, tf.float32,
'ob_std', self.preprocessor.shape, tf.float32,
tf.constant_initializer(np.nan), trainable=False)
in_mean = tf.placeholder(tf.float32, self.preprocessor_shape)
in_std = tf.placeholder(tf.float32, self.preprocessor_shape)
in_mean = tf.placeholder(tf.float32, self.preprocessor.shape)
in_std = tf.placeholder(tf.float32, self.preprocessor.shape)
self._set_ob_mean_std = U.function([in_mean, in_std], [], updates=[
tf.assign(ob_mean, in_mean),
tf.assign(ob_std, in_std),
])
inputs = tf.placeholder(
tf.float32, [None] + list(self.preprocessor_shape))
tf.float32, [None] + list(self.preprocessor.shape))
# TODO(ekl): we should do clipping in a standard RLlib preprocessor
clipped_inputs = tf.clip_by_value(