[rllib] improve custom env docs (#1447)

* env docs

* add env

* update env

* Fri Jan 19 18:55:34 PST 2018

doc/source/rllib.rst

@@ -56,7 +56,7 @@ be trained, checkpointed, or an action computed.
 
 You can train a simple DQN agent with the following command
 
-::
+.. code-block:: bash
 
     python ray/python/ray/rllib/train.py --run DQN --env CartPole-v0
 
@@ -66,14 +66,14 @@ hyperparameters, a file ``result.json`` which contains a training summary
 for each episode and a TensorBoard file that can be used to visualize
 training process with TensorBoard by running
 
-::
+.. code-block:: bash
 
      tensorboard --logdir=~/ray_results
 
 
 The ``train.py`` script has a number of options you can show by running
 
-::
+.. code-block:: bash
 
     python ray/python/ray/rllib/train.py --help
 
@@ -93,7 +93,10 @@ Each algorithm has specific hyperparameters that can be set with ``--config`` -
 `DQN <https://github.com/ray-project/ray/blob/master/python/ray/rllib/dqn/dqn.py>`__.
 
 In an example below, we train A3C by specifying 8 workers through the config flag.
-::
+function that creates the env to refer to it by name. The contents of the env_config agent config field will be passed to that function to allow the environment to be configured. The return type should be an OpenAI gym.Env. For example:
+
+
+.. code-block:: bash
 
     python ray/python/ray/rllib/train.py --env=PongDeterministic-v4 \
         --run=A3C --config '{"num_workers": 8}'
@@ -108,7 +111,7 @@ when running ``train.py``.
 
 An example of evaluating a previously trained DQN agent is as follows:
 
-::
+.. code-block:: bash
 
     python ray/python/ray/rllib/eval.py \
           ~/ray_results/default/DQN_CartPole-v0_0upjmdgr0/checkpoint-1 \
@@ -134,7 +137,7 @@ The Python API provides the needed flexibility for applying RLlib to new problem
 
 Here is an example of the basic usage:
 
-::
+.. code-block:: python
 
     import ray
     import ray.rllib.ppo as ppo
@@ -171,15 +174,18 @@ Custom Environments
 To train against a custom environment, i.e. one not in the gym catalog, you
 can register a function that creates the env to refer to it by name. The contents of the
 ``env_config`` agent config field will be passed to that function to allow the
-environment to be configured. For example:
+environment to be configured. The return type should be an `OpenAI gym.Env <https://github.com/openai/gym/blob/master/gym/core.py>`__. For example:
 
-::
+.. code-block:: python
 
     import ray
     from ray.tune.registry import register_env
     from ray.rllib import ppo
 
-    env_creator = lambda env_config: MyCustomEnv(env_config)
+    def env_creator(env_config):
+        import gym
+        gym.make("CartPole-v0")  # or return your own custom env
+
     env_creator_name = "custom_env"
     register_env(env_creator_name, env_creator)
 
@@ -188,6 +194,8 @@ environment to be configured. For example:
         "env_config": {},  # config to pass to env creator
     })
 
+For a code example of a custom env, see the `SimpleCorridor example <https://github.com/ray-project/ray/blob/master/examples/custom_env/custom_env.py>`__. For a more complex example, also see the `Carla RLlib env <https://github.com/ray-project/ray/blob/master/examples/carla/env.py>`__.
+
 Custom Preprocessors and Models
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -198,7 +206,7 @@ output a vector of the size specified in the constructor. The interfaces for
 these custom classes can be found in the
 `RLlib Developer Guide <rllib-dev.html>`__.
 
-::
+.. code-block:: python
 
     import ray
     from ray.rllib.models import ModelCatalog, Model
@@ -245,14 +253,14 @@ All Agents implemented in RLlib support the
 
 Here is an example of using the command-line interface with RLlib:
 
-::
+.. code-block:: bash
 
     python ray/python/ray/rllib/train.py -f tuned_examples/cartpole-grid-search-example.yaml
 
 Here is an example using the Python API. The same config passed to ``Agents`` may be placed
 in the ``config`` section of the experiments.
 
-::
+.. code-block:: python
 
     from ray.tune.tune import run_experiments
     from ray.tune.variant_generator import grid_search
@@ -280,8 +288,6 @@ in the ``config`` section of the experiments.
 
     run_experiments(experiment)
 
-.. _`managing a cluster with parallel ssh`: using-ray-on-a-large-cluster.html
-
 Contributing to RLlib
 ---------------------
 

examples/custom_env/README

@@ -0,0 +1 @@
+Example of using a custom gym env with RLlib.

examples/custom_env/custom_env.py

@@ -0,0 +1,54 @@
+"""Example of a custom gym environment. Run this for a demo."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import gym
+from gym.spaces import Discrete, Box
+from gym.envs.registration import EnvSpec
+
+from ray.tune import run_experiments
+from ray.tune.registry import register_env
+
+
+class SimpleCorridor(gym.Env):
+    """Example of a custom env in which you have to walk down a corridor.
+
+    You can configure the length of the corridor via the env config."""
+
+    def __init__(self, config):
+        self.end_pos = config["corridor_length"]
+        self.cur_pos = 0
+        self.action_space = Discrete(2)
+        self.observation_space = Box(0.0, self.end_pos, shape=(1,))
+        self._spec = EnvSpec("SimpleCorridor-{}-v0".format(self.end_pos))
+
+    def _reset(self):
+        self.cur_pos = 0
+        return [self.cur_pos]
+
+    def _step(self, action):
+        assert action in [0, 1]
+        if action == 0 and self.cur_pos > 0:
+            self.cur_pos -= 1
+        elif action == 1:
+            self.cur_pos += 1
+        done = self.cur_pos >= self.end_pos
+        return [self.cur_pos], 1 if done else 0, done, {}
+
+
+if __name__ == "__main__":
+    env_creator_name = "corridor"
+    register_env(env_creator_name, lambda config: SimpleCorridor(config))
+    run_experiments({
+        "demo": {
+            "run": "PPO",
+            "env": "corridor",
+            "config": {
+                "env_config": {
+                    "corridor_length": 5,
+                },
+            },
+        },
+    })

python/ray/rllib/models/preprocessors.py

@@ -128,7 +128,7 @@ def get_preprocessor(space):
     obs_shape = space.shape
     print("Observation shape is {}".format(obs_shape))
 
-    if obs_shape == ():
+    if isinstance(space, gym.spaces.Discrete):
         print("Using one-hot preprocessor for discrete envs.")
         preprocessor = OneHotPreprocessor
     elif obs_shape == ATARI_OBS_SHAPE: