diff --git a/doc/source/rllib.rst b/doc/source/rllib.rst index 2f5b20858..8a28146ee 100644 --- a/doc/source/rllib.rst +++ b/doc/source/rllib.rst @@ -56,7 +56,7 @@ be trained, checkpointed, or an action computed. You can train a simple DQN agent with the following command -:: +.. code-block:: bash python ray/python/ray/rllib/train.py --run DQN --env CartPole-v0 @@ -66,14 +66,14 @@ hyperparameters, a file ``result.json`` which contains a training summary for each episode and a TensorBoard file that can be used to visualize training process with TensorBoard by running -:: +.. code-block:: bash tensorboard --logdir=~/ray_results The ``train.py`` script has a number of options you can show by running -:: +.. code-block:: bash python ray/python/ray/rllib/train.py --help @@ -93,7 +93,10 @@ Each algorithm has specific hyperparameters that can be set with ``--config`` - `DQN `__. In an example below, we train A3C by specifying 8 workers through the config flag. -:: +function that creates the env to refer to it by name. The contents of the env_config agent config field will be passed to that function to allow the environment to be configured. The return type should be an OpenAI gym.Env. For example: + + +.. code-block:: bash python ray/python/ray/rllib/train.py --env=PongDeterministic-v4 \ --run=A3C --config '{"num_workers": 8}' @@ -108,7 +111,7 @@ when running ``train.py``. An example of evaluating a previously trained DQN agent is as follows: -:: +.. code-block:: bash python ray/python/ray/rllib/eval.py \ ~/ray_results/default/DQN_CartPole-v0_0upjmdgr0/checkpoint-1 \ @@ -134,7 +137,7 @@ The Python API provides the needed flexibility for applying RLlib to new problem Here is an example of the basic usage: -:: +.. code-block:: python import ray import ray.rllib.ppo as ppo @@ -171,15 +174,18 @@ Custom Environments To train against a custom environment, i.e. one not in the gym catalog, you can register a function that creates the env to refer to it by name. The contents of the ``env_config`` agent config field will be passed to that function to allow the -environment to be configured. For example: +environment to be configured. The return type should be an `OpenAI gym.Env `__. For example: -:: +.. code-block:: python import ray from ray.tune.registry import register_env from ray.rllib import ppo - env_creator = lambda env_config: MyCustomEnv(env_config) + def env_creator(env_config): + import gym + gym.make("CartPole-v0") # or return your own custom env + env_creator_name = "custom_env" register_env(env_creator_name, env_creator) @@ -188,6 +194,8 @@ environment to be configured. For example: "env_config": {}, # config to pass to env creator }) +For a code example of a custom env, see the `SimpleCorridor example `__. For a more complex example, also see the `Carla RLlib env `__. + Custom Preprocessors and Models ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -198,7 +206,7 @@ output a vector of the size specified in the constructor. The interfaces for these custom classes can be found in the `RLlib Developer Guide `__. -:: +.. code-block:: python import ray from ray.rllib.models import ModelCatalog, Model @@ -245,14 +253,14 @@ All Agents implemented in RLlib support the Here is an example of using the command-line interface with RLlib: -:: +.. code-block:: bash python ray/python/ray/rllib/train.py -f tuned_examples/cartpole-grid-search-example.yaml Here is an example using the Python API. The same config passed to ``Agents`` may be placed in the ``config`` section of the experiments. -:: +.. code-block:: python from ray.tune.tune import run_experiments from ray.tune.variant_generator import grid_search @@ -280,8 +288,6 @@ in the ``config`` section of the experiments. run_experiments(experiment) -.. _`managing a cluster with parallel ssh`: using-ray-on-a-large-cluster.html - Contributing to RLlib --------------------- diff --git a/examples/custom_env/README b/examples/custom_env/README new file mode 100644 index 000000000..75ffcad88 --- /dev/null +++ b/examples/custom_env/README @@ -0,0 +1 @@ +Example of using a custom gym env with RLlib. diff --git a/examples/custom_env/custom_env.py b/examples/custom_env/custom_env.py new file mode 100644 index 000000000..0c0e5a517 --- /dev/null +++ b/examples/custom_env/custom_env.py @@ -0,0 +1,54 @@ +"""Example of a custom gym environment. Run this for a demo.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gym +from gym.spaces import Discrete, Box +from gym.envs.registration import EnvSpec + +from ray.tune import run_experiments +from ray.tune.registry import register_env + + +class SimpleCorridor(gym.Env): + """Example of a custom env in which you have to walk down a corridor. + + You can configure the length of the corridor via the env config.""" + + def __init__(self, config): + self.end_pos = config["corridor_length"] + self.cur_pos = 0 + self.action_space = Discrete(2) + self.observation_space = Box(0.0, self.end_pos, shape=(1,)) + self._spec = EnvSpec("SimpleCorridor-{}-v0".format(self.end_pos)) + + def _reset(self): + self.cur_pos = 0 + return [self.cur_pos] + + def _step(self, action): + assert action in [0, 1] + if action == 0 and self.cur_pos > 0: + self.cur_pos -= 1 + elif action == 1: + self.cur_pos += 1 + done = self.cur_pos >= self.end_pos + return [self.cur_pos], 1 if done else 0, done, {} + + +if __name__ == "__main__": + env_creator_name = "corridor" + register_env(env_creator_name, lambda config: SimpleCorridor(config)) + run_experiments({ + "demo": { + "run": "PPO", + "env": "corridor", + "config": { + "env_config": { + "corridor_length": 5, + }, + }, + }, + }) diff --git a/python/ray/rllib/models/preprocessors.py b/python/ray/rllib/models/preprocessors.py index e90a0d6d8..01e9db16f 100644 --- a/python/ray/rllib/models/preprocessors.py +++ b/python/ray/rllib/models/preprocessors.py @@ -128,7 +128,7 @@ def get_preprocessor(space): obs_shape = space.shape print("Observation shape is {}".format(obs_shape)) - if obs_shape == (): + if isinstance(space, gym.spaces.Discrete): print("Using one-hot preprocessor for discrete envs.") preprocessor = OneHotPreprocessor elif obs_shape == ATARI_OBS_SHAPE: