diff --git a/doc/source/rllib-examples.rst b/doc/source/rllib-examples.rst index 832183fd1..165cff4ed 100644 --- a/doc/source/rllib-examples.rst +++ b/doc/source/rllib-examples.rst @@ -48,7 +48,7 @@ Custom Envs and Models Example of how to ensure subprocesses spawned by envs are killed when RLlib exits. - `Batch normalization `__: Example of adding batch norm layers to a custom model. -- `Parametric actions `__: +- `Parametric actions `__: Example of how to handle variable-length or parametric action spaces. - `Eager execution `__: Example of how to leverage TensorFlow eager to simplify debugging and design of custom models and policies. diff --git a/doc/source/rllib-models.rst b/doc/source/rllib-models.rst index 5583a199b..bd1d9b27a 100644 --- a/doc/source/rllib-models.rst +++ b/doc/source/rllib-models.rst @@ -274,7 +274,7 @@ Custom models can be used to work with environments where (1) the set of valid a return action_logits + inf_mask, state -Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_action_cartpole.py `__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``vf_share_layers=True``), and several other algorithms. Not all algorithms support parametric actions; see the `feature compatibility matrix `__. +Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_actions_cartpole.py `__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``vf_share_layers=True``), and several other algorithms. Not all algorithms support parametric actions; see the `feature compatibility matrix `__. Autoregressive Action Distributions diff --git a/rllib/BUILD b/rllib/BUILD index e87262f7f..3aaed4b32 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -1339,7 +1339,7 @@ py_test( tags = ["examples", "examples_C"], size = "large", srcs = ["examples/custom_keras_rnn_model.py"], - args = ["--run=PPO", "--stop=50", "--env=RepeatInitialEnv", "--num-cpus=4"] + args = ["--run=PPO", "--stop=50", "--env=RepeatInitialObsEnv", "--num-cpus=4"] ) py_test( @@ -1401,6 +1401,22 @@ py_test( args = ["--iters=2"] ) +py_test( + name = "examples/hierarchical_training_tf", + tags = ["examples", "examples_H"], + size = "small", + srcs = ["examples/hierarchical_training.py"], + args = ["--stop-reward=0.0"] +) + +py_test( + name = "examples/hierarchical_training_torch", + tags = ["examples", "examples_H"], + size = "small", + srcs = ["examples/hierarchical_training.py"], + args = ["--torch", "--stop-reward=0.0"] +) + py_test( name = "examples/multi_agent_cartpole", tags = ["examples", "examples_M"], @@ -1434,36 +1450,32 @@ py_test( ) py_test( - name = "examples/parametric_action_cartpole_pg", main="examples/parametric_action_cartpole.py", + name = "examples/parametric_actions_cartpole_pg", + main = "examples/parametric_actions_cartpole.py", tags = ["examples", "examples_P"], size = "medium", - srcs = ["examples/parametric_action_cartpole.py"], + srcs = ["examples/parametric_actions_cartpole.py"], args = ["--run=PG", "--stop=50"] ) py_test( - name = "examples/parametric_action_cartpole_ppo", main="examples/parametric_action_cartpole.py", + name = "examples/parametric_actions_cartpole_ppo", + main = "examples/parametric_actions_cartpole.py", tags = ["examples", "examples_P"], size = "medium", - srcs = ["examples/parametric_action_cartpole.py"], + srcs = ["examples/parametric_actions_cartpole.py"], args = ["--run=PPO", "--stop=50"] ) py_test( - name = "examples/parametric_action_cartpole_dqn", main="examples/parametric_action_cartpole.py", + name = "examples/parametric_actions_cartpole_dqn", + main = "examples/parametric_actions_cartpole.py", tags = ["examples", "examples_P"], size = "medium", - srcs = ["examples/parametric_action_cartpole.py"], + srcs = ["examples/parametric_actions_cartpole.py"], args = ["--run=DQN", "--stop=50"] ) -py_test( - name = "examples/random_env", main = "examples/random_env.py", - tags = ["examples", "examples_R"], - size = "large", - srcs = ["examples/random_env.py"] -) - py_test( name = "examples/rollout_worker_custom_workflow", tags = ["examples", "examples_R"], diff --git a/rllib/agents/qmix/qmix_policy.py b/rllib/agents/qmix/qmix_policy.py index 355588a2d..aa4c096b1 100644 --- a/rllib/agents/qmix/qmix_policy.py +++ b/rllib/agents/qmix/qmix_policy.py @@ -5,6 +5,7 @@ import numpy as np import ray from ray.rllib.agents.qmix.mixers import VDNMixer, QMixer from ray.rllib.agents.qmix.model import RNNModel, _get_size +from ray.rllib.env.multi_agent_env import ENV_STATE from ray.rllib.evaluation.metrics import LEARNER_STATS_KEY from ray.rllib.policy.policy import Policy from ray.rllib.policy.rnn_sequencing import chop_into_sequences @@ -20,9 +21,6 @@ torch, nn = try_import_torch(error=True) logger = logging.getLogger(__name__) -# if the obs space is Dict type, look for the global state under this key -ENV_STATE = "state" - class QMixLoss(nn.Module): def __init__(self, diff --git a/rllib/contrib/alpha_zero/README.md b/rllib/contrib/alpha_zero/README.md index 2b9807e29..7486877a5 100644 --- a/rllib/contrib/alpha_zero/README.md +++ b/rllib/contrib/alpha_zero/README.md @@ -12,7 +12,7 @@ The code is Pytorch based. It assumes that the environment is a gym environment, The model used in AlphaZero trainer should extend `ActorCriticModel` and implement the method `compute_priors_and_value`. -## Example on Cartpole +## Example on CartPole Note that both mean and max rewards are obtained with the MCTS in exploration mode: dirichlet noise is added to priors and actions are sampled from the tree policy vectors. We will add later the display of the MCTS in exploitation mode: no dirichlet noise and actions are chosen as tree policy vectors argmax. ![cartpole_plot](doc/cartpole_plot.png) @@ -21,4 +21,3 @@ Note that both mean and max rewards are obtained with the MCTS in exploration mo - AlphaZero: https://arxiv.org/abs/1712.01815 - Ranked rewards: https://arxiv.org/abs/1807.01672 - \ No newline at end of file diff --git a/rllib/env/multi_agent_env.py b/rllib/env/multi_agent_env.py index bfe147b5d..3a9fd2ef7 100644 --- a/rllib/env/multi_agent_env.py +++ b/rllib/env/multi_agent_env.py @@ -1,5 +1,8 @@ from ray.rllib.utils.annotations import PublicAPI +# If the obs space is Dict type, look for the global state under this key. +ENV_STATE = "state" + @PublicAPI class MultiAgentEnv: diff --git a/rllib/examples/autoregressive_action_dist.py b/rllib/examples/autoregressive_action_dist.py index 8a529771a..bc91f147f 100644 --- a/rllib/examples/autoregressive_action_dist.py +++ b/rllib/examples/autoregressive_action_dist.py @@ -10,13 +10,12 @@ pattern, and a custom action distribution class that leverages that model. This examples shows both. """ -import gym from gym.spaces import Discrete, Tuple import argparse -import random import ray from ray import tune +from ray.rllib.examples.env.correlated_actions_env import CorrelatedActionsEnv from ray.rllib.models import ModelCatalog from ray.rllib.models.tf.tf_action_dist import Categorical, ActionDistribution from ray.rllib.models.tf.misc import normc_initializer @@ -31,34 +30,6 @@ parser.add_argument("--stop", type=int, default=200) parser.add_argument("--num-cpus", type=int, default=0) -class CorrelatedActionsEnv(gym.Env): - """Simple env in which the policy has to emit a tuple of equal actions. - - The best score would be ~200 reward.""" - - def __init__(self, _): - self.observation_space = Discrete(2) - self.action_space = Tuple([Discrete(2), Discrete(2)]) - - def reset(self): - self.t = 0 - self.last = random.choice([0, 1]) - return self.last - - def step(self, action): - self.t += 1 - a1, a2 = action - reward = 0 - if a1 == self.last: - reward += 5 - # encourage correlation between a1 and a2 - if a1 == a2: - reward += 5 - done = self.t > 20 - self.last = random.choice([0, 1]) - return self.last, reward, done, {} - - class BinaryAutoregressiveOutput(ActionDistribution): """Action distribution P(a1, a2) = P(a1) * P(a2 | a1)""" diff --git a/rllib/examples/batch_norm_model.py b/rllib/examples/batch_norm_model.py index e58c9ff7a..22a7af8e6 100644 --- a/rllib/examples/batch_norm_model.py +++ b/rllib/examples/batch_norm_model.py @@ -155,10 +155,6 @@ if __name__ == "__main__": "num_workers": 0, } - from ray.rllib.agents.ppo import PPOTrainer - trainer = PPOTrainer(config=config) - trainer.train() - tune.run( args.run, stop={"training_iteration": args.num_iters}, diff --git a/rllib/examples/cartpole_lstm.py b/rllib/examples/cartpole_lstm.py index 7e82dee84..56ab3178d 100644 --- a/rllib/examples/cartpole_lstm.py +++ b/rllib/examples/cartpole_lstm.py @@ -1,170 +1,20 @@ -"""Partially observed variant of the CartPole gym environment. - -https://github.com/openai/gym/blob/master/gym/envs/classic_control/cartpole.py - -We delete the velocity component of the state, so that it can only be solved -by a LSTM policy.""" - import argparse -import math -import gym -from gym import spaces -from gym.utils import seeding -import numpy as np + +from ray.rllib.examples.env.stateless_cartpole import StatelessCartPole parser = argparse.ArgumentParser() parser.add_argument("--stop", type=int, default=200) +parser.add_argument("--torch", action="store_true") parser.add_argument("--use-prev-action-reward", action="store_true") parser.add_argument("--run", type=str, default="PPO") parser.add_argument("--num-cpus", type=int, default=0) - -class CartPoleStatelessEnv(gym.Env): - metadata = { - "render.modes": ["human", "rgb_array"], - "video.frames_per_second": 60 - } - - def __init__(self, config=None): - self.gravity = 9.8 - self.masscart = 1.0 - self.masspole = 0.1 - self.total_mass = (self.masspole + self.masscart) - self.length = 0.5 # actually half the pole's length - self.polemass_length = (self.masspole * self.length) - self.force_mag = 10.0 - self.tau = 0.02 # seconds between state updates - - # Angle at which to fail the episode - self.theta_threshold_radians = 12 * 2 * math.pi / 360 - self.x_threshold = 2.4 - - high = np.array([ - self.x_threshold * 2, - self.theta_threshold_radians * 2, - ]) - - self.action_space = spaces.Discrete(2) - self.observation_space = spaces.Box(-high, high) - - self.seed() - self.viewer = None - self.state = None - - self.steps_beyond_done = None - - def seed(self, seed=None): - self.np_random, seed = seeding.np_random(seed) - return [seed] - - def step(self, action): - assert self.action_space.contains( - action), "%r (%s) invalid" % (action, type(action)) - state = self.state - x, x_dot, theta, theta_dot = state - force = self.force_mag if action == 1 else -self.force_mag - costheta = math.cos(theta) - sintheta = math.sin(theta) - temp = (force + self.polemass_length * theta_dot * theta_dot * sintheta - ) / self.total_mass - thetaacc = (self.gravity * sintheta - costheta * temp) / ( - self.length * - (4.0 / 3.0 - self.masspole * costheta * costheta / self.total_mass) - ) - xacc = (temp - - self.polemass_length * thetaacc * costheta / self.total_mass) - x = x + self.tau * x_dot - x_dot = x_dot + self.tau * xacc - theta = theta + self.tau * theta_dot - theta_dot = theta_dot + self.tau * thetaacc - self.state = (x, x_dot, theta, theta_dot) - done = (x < -self.x_threshold or x > self.x_threshold - or theta < -self.theta_threshold_radians - or theta > self.theta_threshold_radians) - done = bool(done) - - if not done: - reward = 1.0 - elif self.steps_beyond_done is None: - # Pole just fell! - self.steps_beyond_done = 0 - reward = 1.0 - else: - self.steps_beyond_done += 1 - reward = 0.0 - - rv = np.r_[self.state[0], self.state[2]] - return rv, reward, done, {} - - def reset(self): - self.state = self.np_random.uniform(low=-0.05, high=0.05, size=(4, )) - self.steps_beyond_done = None - - rv = np.r_[self.state[0], self.state[2]] - return rv - - def render(self, mode="human"): - screen_width = 600 - screen_height = 400 - - world_width = self.x_threshold * 2 - scale = screen_width / world_width - carty = 100 # TOP OF CART - polewidth = 10.0 - polelen = scale * 1.0 - cartwidth = 50.0 - cartheight = 30.0 - - if self.viewer is None: - from gym.envs.classic_control import rendering - self.viewer = rendering.Viewer(screen_width, screen_height) - l, r, t, b = (-cartwidth / 2, cartwidth / 2, cartheight / 2, - -cartheight / 2) - axleoffset = cartheight / 4.0 - cart = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)]) - self.carttrans = rendering.Transform() - cart.add_attr(self.carttrans) - self.viewer.add_geom(cart) - l, r, t, b = (-polewidth / 2, polewidth / 2, - polelen - polewidth / 2, -polewidth / 2) - pole = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)]) - pole.set_color(.8, .6, .4) - self.poletrans = rendering.Transform(translation=(0, axleoffset)) - pole.add_attr(self.poletrans) - pole.add_attr(self.carttrans) - self.viewer.add_geom(pole) - self.axle = rendering.make_circle(polewidth / 2) - self.axle.add_attr(self.poletrans) - self.axle.add_attr(self.carttrans) - self.axle.set_color(.5, .5, .8) - self.viewer.add_geom(self.axle) - self.track = rendering.Line((0, carty), (screen_width, carty)) - self.track.set_color(0, 0, 0) - self.viewer.add_geom(self.track) - - if self.state is None: - return None - - x = self.state - cartx = x[0] * scale + screen_width / 2.0 # MIDDLE OF CART - self.carttrans.set_translation(cartx, carty) - self.poletrans.set_rotation(-x[2]) - - return self.viewer.render(return_rgb_array=mode == "rgb_array") - - def close(self): - if self.viewer: - self.viewer.close() - - if __name__ == "__main__": import ray from ray import tune args = parser.parse_args() - tune.register_env("cartpole_stateless", lambda _: CartPoleStatelessEnv()) - ray.init(num_cpus=args.num_cpus or None) configs = { @@ -185,10 +35,11 @@ if __name__ == "__main__": stop={"episode_reward_mean": args.stop}, config=dict( configs[args.run], **{ - "env": "cartpole_stateless", + "env": StatelessCartPole, "model": { "use_lstm": True, "lstm_use_prev_action_reward": args.use_prev_action_reward, }, + "use_pytorch": args.torch, }), ) diff --git a/rllib/examples/custom_eval.py b/rllib/examples/custom_eval.py index c6dfecce5..d95bb09b9 100644 --- a/rllib/examples/custom_eval.py +++ b/rllib/examples/custom_eval.py @@ -67,18 +67,16 @@ Result for PG_SimpleCorridor_0de4e686: """ import argparse -import numpy as np -import gym -from gym.spaces import Discrete, Box import ray from ray import tune from ray.rllib.evaluation.metrics import collect_episodes, summarize_episodes +from ray.rllib.examples.env.simple_corridor import SimpleCorridor parser = argparse.ArgumentParser() parser.add_argument("--custom-eval", action="store_true") parser.add_argument("--num-cpus", type=int, default=0) -args = parser.parse_args() +parser.add_argument("--torch", action="store_true") def custom_eval_function(trainer, eval_workers): @@ -123,36 +121,9 @@ def custom_eval_function(trainer, eval_workers): return metrics -class SimpleCorridor(gym.Env): - """Custom env we use for this example.""" - - def __init__(self, env_config): - self.end_pos = env_config["corridor_length"] - self.cur_pos = 0 - self.action_space = Discrete(2) - self.observation_space = Box(0.0, 9999, shape=(1, ), dtype=np.float32) - print("Created env for worker index", env_config.worker_index, - "with corridor length", self.end_pos) - - def set_corridor_length(self, length): - print("Update corridor length to", length) - self.end_pos = length - - def reset(self): - self.cur_pos = 0 - return [self.cur_pos] - - def step(self, action): - assert action in [0, 1], action - if action == 0 and self.cur_pos > 0: - self.cur_pos -= 1 - elif action == 1: - self.cur_pos += 1 - done = self.cur_pos >= self.end_pos - return [self.cur_pos], 1 if done else 0, done, {} - - if __name__ == "__main__": + args = parser.parse_args() + if args.custom_eval: eval_fn = custom_eval_function else: @@ -196,4 +167,5 @@ if __name__ == "__main__": "corridor_length": 5, }, }, + "use_pytorch": args.torch, }) diff --git a/rllib/examples/custom_fast_model.py b/rllib/examples/custom_fast_model.py index 857a879ca..e6ab6c1a4 100644 --- a/rllib/examples/custom_fast_model.py +++ b/rllib/examples/custom_fast_model.py @@ -4,11 +4,8 @@ Both the model and env are trivial (and super-fast), so they are useful for running perf microbenchmarks. """ -from gym.spaces import Discrete, Box -import gym -import numpy as np - import ray +from ray.rllib.examples.env.fast_image_env import FastImageEnv from ray.rllib.models import Model, ModelCatalog from ray.tune import run_experiments, sample_from from ray.rllib.utils import try_import_tf @@ -27,23 +24,6 @@ class FastModel(Model): return output, output -class FastImageEnv(gym.Env): - def __init__(self, config): - self.zeros = np.zeros((84, 84, 4)) - self.action_space = Discrete(2) - self.observation_space = Box( - 0.0, 1.0, shape=(84, 84, 4), dtype=np.float32) - self.i = 0 - - def reset(self): - self.i = 0 - return self.zeros - - def step(self, action): - self.i += 1 - return self.zeros, 1, self.i > 1000, {} - - if __name__ == "__main__": ray.init() ModelCatalog.register_custom_model("fast_model", FastModel) diff --git a/rllib/examples/custom_keras_rnn_model.py b/rllib/examples/custom_keras_rnn_model.py index ec02de923..311215ef3 100644 --- a/rllib/examples/custom_keras_rnn_model.py +++ b/rllib/examples/custom_keras_rnn_model.py @@ -1,14 +1,13 @@ """Example of using a custom RNN keras model.""" import argparse -import gym -from gym.spaces import Discrete import numpy as np -import random import ray from ray import tune from ray.tune.registry import register_env +from ray.rllib.examples.env.repeat_after_me_env import RepeatAfterMeEnv +from ray.rllib.examples.env.repeat_initial_obs_env import RepeatInitialObsEnv from ray.rllib.models import ModelCatalog from ray.rllib.models.modelv2 import ModelV2 from ray.rllib.models.tf.recurrent_tf_modelv2 import RecurrentTFModelV2 @@ -88,70 +87,12 @@ class MyKerasRNN(RecurrentTFModelV2): return tf.reshape(self._value_out, [-1]) -class RepeatInitialEnv(gym.Env): - """Simple env where policy has to always repeat the initial observation. - - Runs for 100 steps. - r=1 if action correct, -1 otherwise (max. R=100). - """ - - def __init__(self, episode_len=100): - self.observation_space = Discrete(2) - self.action_space = Discrete(2) - self.token = None - self.episode_len = episode_len - self.num_steps = 0 - - def reset(self): - self.token = random.choice([0, 1]) - self.num_steps = 0 - return self.token - - def step(self, action): - if action == self.token: - reward = 1 - else: - reward = -1 - self.num_steps += 1 - done = self.num_steps >= self.episode_len - return 0, reward, done, {} - - -class RepeatAfterMeEnv(gym.Env): - """Simple env in which the policy learns to repeat a previous observation - token after a given delay.""" - - def __init__(self, config): - self.observation_space = Discrete(2) - self.action_space = Discrete(2) - self.delay = config["repeat_delay"] - assert self.delay >= 1, "delay must be at least 1" - self.history = [] - - def reset(self): - self.history = [0] * self.delay - return self._next_obs() - - def step(self, action): - if action == self.history[-(1 + self.delay)]: - reward = 1 - else: - reward = -1 - done = len(self.history) > 100 - return self._next_obs(), reward, done, {} - - def _next_obs(self): - token = random.choice([0, 1]) - self.history.append(token) - return token - - if __name__ == "__main__": args = parser.parse_args() ray.init(num_cpus=args.num_cpus or None) ModelCatalog.register_custom_model("rnn", MyKerasRNN) register_env("RepeatAfterMeEnv", lambda c: RepeatAfterMeEnv(c)) - register_env("RepeatInitialEnv", lambda _: RepeatInitialEnv()) + register_env("RepeatInitialObsEnv", lambda _: RepeatInitialObsEnv()) config = { "env": args.env, diff --git a/rllib/examples/custom_torch_rnn_model.py b/rllib/examples/custom_torch_rnn_model.py index 87fb81343..5e676bd08 100644 --- a/rllib/examples/custom_torch_rnn_model.py +++ b/rllib/examples/custom_torch_rnn_model.py @@ -1,9 +1,9 @@ import argparse import ray -from ray.rllib.examples.cartpole_lstm import CartPoleStatelessEnv -from ray.rllib.examples.custom_keras_rnn_model import RepeatInitialEnv, \ - RepeatAfterMeEnv +from ray.rllib.examples.env.repeat_initial_obs_env import RepeatInitialObsEnv +from ray.rllib.examples.env.repeat_after_me_env import RepeatAfterMeEnv +from ray.rllib.examples.env.stateless_cartpole import StatelessCartPole from ray.rllib.models.preprocessors import get_preprocessor from ray.rllib.models.torch.recurrent_torch_model import RecurrentTorchModel from ray.rllib.models.modelv2 import ModelV2 @@ -92,10 +92,10 @@ if __name__ == "__main__": ray.init(num_cpus=args.num_cpus or None) ModelCatalog.register_custom_model("rnn", RNNModel) tune.register_env( - "repeat_initial", lambda _: RepeatInitialEnv(episode_len=100)) + "repeat_initial", lambda _: RepeatInitialObsEnv(episode_len=100)) tune.register_env( "repeat_after_me", lambda _: RepeatAfterMeEnv({"repeat_delay": 1})) - tune.register_env("cartpole_stateless", lambda _: CartPoleStatelessEnv()) + tune.register_env("stateless_cartpole", lambda _: StatelessCartPole()) config = { "env": args.env, diff --git a/rllib/examples/custom_train_fn.py b/rllib/examples/custom_train_fn.py index 9707a2dbf..2b85f2a9f 100644 --- a/rllib/examples/custom_train_fn.py +++ b/rllib/examples/custom_train_fn.py @@ -5,11 +5,15 @@ This example shows: You can visualize experiment results in ~/ray_results using TensorBoard. """ +import argparse import ray from ray import tune from ray.rllib.agents.ppo import PPOTrainer +parser = argparse.ArgumentParser() +parser.add_argument("--torch", action="store_true") + def my_train_fn(config, reporter): # Train for 100 iterations with high LR @@ -36,9 +40,11 @@ def my_train_fn(config, reporter): if __name__ == "__main__": ray.init() + args = parser.parse_args() config = { "lr": 0.01, "num_workers": 0, + "use_pytorch": args.torch, } resources = PPOTrainer.default_resource_request(config).to_json() tune.run(my_train_fn, resources_per_trial=resources, config=config) diff --git a/rllib/examples/env/__init__.py b/rllib/examples/env/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/rllib/examples/env/correlated_actions_env.py b/rllib/examples/env/correlated_actions_env.py new file mode 100644 index 000000000..4a42c5ca1 --- /dev/null +++ b/rllib/examples/env/correlated_actions_env.py @@ -0,0 +1,31 @@ +import gym +from gym.spaces import Discrete, Tuple +import random + + +class CorrelatedActionsEnv(gym.Env): + """Simple env in which the policy has to emit a tuple of equal actions. + + The best score would be ~200 reward.""" + + def __init__(self, _): + self.observation_space = Discrete(2) + self.action_space = Tuple([Discrete(2), Discrete(2)]) + + def reset(self): + self.t = 0 + self.last = random.choice([0, 1]) + return self.last + + def step(self, action): + self.t += 1 + a1, a2 = action + reward = 0 + if a1 == self.last: + reward += 5 + # encourage correlation between a1 and a2 + if a1 == a2: + reward += 5 + done = self.t > 20 + self.last = random.choice([0, 1]) + return self.last, reward, done, {} diff --git a/rllib/examples/env/env_with_subprocess.py b/rllib/examples/env/env_with_subprocess.py new file mode 100644 index 000000000..e55e8e0e7 --- /dev/null +++ b/rllib/examples/env/env_with_subprocess.py @@ -0,0 +1,44 @@ +import atexit +import gym +from gym.spaces import Discrete +import os +import subprocess +import time + + +class EnvWithSubprocess(gym.Env): + """Our env that spawns a subprocess.""" + + # Dummy command to run as a subprocess with a unique name + UNIQUE_CMD = "sleep {}".format(str(time.time())) + + def __init__(self, config): + self.UNIQUE_FILE_0 = config["tmp_file1"] + self.UNIQUE_FILE_1 = config["tmp_file2"] + self.UNIQUE_FILE_2 = config["tmp_file3"] + self.UNIQUE_FILE_3 = config["tmp_file4"] + + self.action_space = Discrete(2) + self.observation_space = Discrete(2) + # Subprocess that should be cleaned up + self.subproc = subprocess.Popen( + self.UNIQUE_CMD.split(" "), shell=False) + self.config = config + # Exit handler should be called + atexit.register(lambda: self.subproc.kill()) + if config.worker_index == 0: + atexit.register(lambda: os.unlink(self.UNIQUE_FILE_0)) + else: + atexit.register(lambda: os.unlink(self.UNIQUE_FILE_1)) + + def close(self): + if self.config.worker_index == 0: + os.unlink(self.UNIQUE_FILE_2) + else: + os.unlink(self.UNIQUE_FILE_3) + + def reset(self): + return 0 + + def step(self, action): + return 0, 0, True, {} diff --git a/rllib/examples/env/fast_image_env.py b/rllib/examples/env/fast_image_env.py new file mode 100644 index 000000000..cb50be51d --- /dev/null +++ b/rllib/examples/env/fast_image_env.py @@ -0,0 +1,20 @@ +import gym +from gym.spaces import Box, Discrete +import numpy as np + + +class FastImageEnv(gym.Env): + def __init__(self, config): + self.zeros = np.zeros((84, 84, 4)) + self.action_space = Discrete(2) + self.observation_space = Box( + 0.0, 1.0, shape=(84, 84, 4), dtype=np.float32) + self.i = 0 + + def reset(self): + self.i = 0 + return self.zeros + + def step(self, action): + self.i += 1 + return self.zeros, 1, self.i > 1000, {} diff --git a/rllib/examples/env/multi_agent.py b/rllib/examples/env/multi_agent.py new file mode 100644 index 000000000..ce5214b76 --- /dev/null +++ b/rllib/examples/env/multi_agent.py @@ -0,0 +1,162 @@ +import gym + +from ray.rllib.env.multi_agent_env import MultiAgentEnv +from ray.rllib.tests.test_rollout_worker import MockEnv, MockEnv2 + + +def make_multiagent(env_name): + class MultiEnv(MultiAgentEnv): + def __init__(self, config): + self.agents = [ + gym.make(env_name) for _ in range(config["num_agents"]) + ] + self.dones = set() + self.observation_space = self.agents[0].observation_space + self.action_space = self.agents[0].action_space + + def reset(self): + self.dones = set() + return {i: a.reset() for i, a in enumerate(self.agents)} + + def step(self, action_dict): + obs, rew, done, info = {}, {}, {}, {} + for i, action in action_dict.items(): + obs[i], rew[i], done[i], info[i] = self.agents[i].step(action) + if done[i]: + self.dones.add(i) + done["__all__"] = len(self.dones) == len(self.agents) + return obs, rew, done, info + + return MultiEnv + + +class BasicMultiAgent(MultiAgentEnv): + """Env of N independent agents, each of which exits after 25 steps.""" + + def __init__(self, num): + self.agents = [MockEnv(25) for _ in range(num)] + self.dones = set() + self.observation_space = gym.spaces.Discrete(2) + self.action_space = gym.spaces.Discrete(2) + self.resetted = False + + def reset(self): + self.resetted = True + self.dones = set() + return {i: a.reset() for i, a in enumerate(self.agents)} + + def step(self, action_dict): + obs, rew, done, info = {}, {}, {}, {} + for i, action in action_dict.items(): + obs[i], rew[i], done[i], info[i] = self.agents[i].step(action) + if done[i]: + self.dones.add(i) + done["__all__"] = len(self.dones) == len(self.agents) + return obs, rew, done, info + + +class EarlyDoneMultiAgent(MultiAgentEnv): + """Env for testing when the env terminates (after agent 0 does).""" + + def __init__(self): + self.agents = [MockEnv(3), MockEnv(5)] + self.dones = set() + self.last_obs = {} + self.last_rew = {} + self.last_done = {} + self.last_info = {} + self.i = 0 + self.observation_space = gym.spaces.Discrete(10) + self.action_space = gym.spaces.Discrete(2) + + def reset(self): + self.dones = set() + self.last_obs = {} + self.last_rew = {} + self.last_done = {} + self.last_info = {} + self.i = 0 + for i, a in enumerate(self.agents): + self.last_obs[i] = a.reset() + self.last_rew[i] = None + self.last_done[i] = False + self.last_info[i] = {} + obs_dict = {self.i: self.last_obs[self.i]} + self.i = (self.i + 1) % len(self.agents) + return obs_dict + + def step(self, action_dict): + assert len(self.dones) != len(self.agents) + for i, action in action_dict.items(): + (self.last_obs[i], self.last_rew[i], self.last_done[i], + self.last_info[i]) = self.agents[i].step(action) + obs = {self.i: self.last_obs[self.i]} + rew = {self.i: self.last_rew[self.i]} + done = {self.i: self.last_done[self.i]} + info = {self.i: self.last_info[self.i]} + if done[self.i]: + rew[self.i] = 0 + self.dones.add(self.i) + self.i = (self.i + 1) % len(self.agents) + done["__all__"] = len(self.dones) == len(self.agents) - 1 + return obs, rew, done, info + + +class RoundRobinMultiAgent(MultiAgentEnv): + """Env of N independent agents, each of which exits after 5 steps. + + On each step() of the env, only one agent takes an action.""" + + def __init__(self, num, increment_obs=False): + if increment_obs: + # Observations are 0, 1, 2, 3... etc. as time advances + self.agents = [MockEnv2(5) for _ in range(num)] + else: + # Observations are all zeros + self.agents = [MockEnv(5) for _ in range(num)] + self.dones = set() + self.last_obs = {} + self.last_rew = {} + self.last_done = {} + self.last_info = {} + self.i = 0 + self.num = num + self.observation_space = gym.spaces.Discrete(10) + self.action_space = gym.spaces.Discrete(2) + + def reset(self): + self.dones = set() + self.last_obs = {} + self.last_rew = {} + self.last_done = {} + self.last_info = {} + self.i = 0 + for i, a in enumerate(self.agents): + self.last_obs[i] = a.reset() + self.last_rew[i] = None + self.last_done[i] = False + self.last_info[i] = {} + obs_dict = {self.i: self.last_obs[self.i]} + self.i = (self.i + 1) % self.num + return obs_dict + + def step(self, action_dict): + assert len(self.dones) != len(self.agents) + for i, action in action_dict.items(): + (self.last_obs[i], self.last_rew[i], self.last_done[i], + self.last_info[i]) = self.agents[i].step(action) + obs = {self.i: self.last_obs[self.i]} + rew = {self.i: self.last_rew[self.i]} + done = {self.i: self.last_done[self.i]} + info = {self.i: self.last_info[self.i]} + if done[self.i]: + rew[self.i] = 0 + self.dones.add(self.i) + self.i = (self.i + 1) % self.num + done["__all__"] = len(self.dones) == len(self.agents) + return obs, rew, done, info + + +MultiAgentCartPole = make_multiagent("CartPole-v0") +MultiAgentMountainCar = make_multiagent("MountainCarContinuous-v0") +MultiAgentPendulum = make_multiagent("Pendulum-v0") diff --git a/rllib/examples/env/nested_space_repeat_after_me_env.py b/rllib/examples/env/nested_space_repeat_after_me_env.py new file mode 100644 index 000000000..c48505fdb --- /dev/null +++ b/rllib/examples/env/nested_space_repeat_after_me_env.py @@ -0,0 +1,52 @@ +import gym +from gym.spaces import Box, Dict, Discrete, Tuple +import numpy as np + +from ray.rllib.utils import try_import_tree +from ray.rllib.utils.space_utils import flatten_space + +tree = try_import_tree() + + +class NestedSpaceRepeatAfterMeEnv(gym.Env): + """Env for which policy has to repeat the (possibly complex) observation. + + The action space and observation spaces are always the same and may be + arbitrarily nested Dict/Tuple Spaces. + Rewards are given for exactly matching Discrete sub-actions and for being + as close as possible for Box sub-actions. + """ + + def __init__(self, config): + self.observation_space = config.get( + "space", Tuple([Discrete(2), + Dict({ + "a": Box(-1.0, 1.0, (2, )) + })])) + self.action_space = self.observation_space + self.flattened_action_space = flatten_space(self.action_space) + self.episode_len = config.get("episode_len", 100) + + def reset(self): + self.steps = 0 + return self._next_obs() + + def step(self, action): + self.steps += 1 + action = tree.flatten(action) + reward = 0.0 + for a, o, space in zip(action, self.current_obs_flattened, + self.flattened_action_space): + # Box: -abs(diff). + if isinstance(space, gym.spaces.Box): + reward -= np.abs(np.sum(a - o)) + # Discrete: +1.0 if exact match. + if isinstance(space, gym.spaces.Discrete): + reward += 1.0 if a == o else 0.0 + done = self.steps >= self.episode_len + return self._next_obs(), reward, done, {} + + def _next_obs(self): + self.current_obs = self.observation_space.sample() + self.current_obs_flattened = tree.flatten(self.current_obs) + return self.current_obs diff --git a/rllib/examples/env/parametric_actions_cartpole.py b/rllib/examples/env/parametric_actions_cartpole.py new file mode 100644 index 000000000..ec98b78e2 --- /dev/null +++ b/rllib/examples/env/parametric_actions_cartpole.py @@ -0,0 +1,77 @@ +import gym +from gym.spaces import Box, Dict, Discrete +import numpy as np +import random + + +class ParametricActionsCartPole(gym.Env): + """Parametric action version of CartPole. + + In this env there are only ever two valid actions, but we pretend there are + actually up to `max_avail_actions` actions that can be taken, and the two + valid actions are randomly hidden among this set. + + At each step, we emit a dict of: + - the actual cart observation + - a mask of valid actions (e.g., [0, 0, 1, 0, 0, 1] for 6 max avail) + - the list of action embeddings (w/ zeroes for invalid actions) (e.g., + [[0, 0], + [0, 0], + [-0.2322, -0.2569], + [0, 0], + [0, 0], + [0.7878, 1.2297]] for max_avail_actions=6) + + In a real environment, the actions embeddings would be larger than two + units of course, and also there would be a variable number of valid actions + per step instead of always [LEFT, RIGHT]. + """ + + def __init__(self, max_avail_actions): + # Use simple random 2-unit action embeddings for [LEFT, RIGHT] + self.left_action_embed = np.random.randn(2) + self.right_action_embed = np.random.randn(2) + self.action_space = Discrete(max_avail_actions) + self.wrapped = gym.make("CartPole-v0") + self.observation_space = Dict({ + "action_mask": Box(0, 1, shape=(max_avail_actions, )), + "avail_actions": Box(-10, 10, shape=(max_avail_actions, 2)), + "cart": self.wrapped.observation_space, + }) + + def update_avail_actions(self): + self.action_assignments = np.array([[0., 0.]] * self.action_space.n) + self.action_mask = np.array([0.] * self.action_space.n) + self.left_idx, self.right_idx = random.sample( + range(self.action_space.n), 2) + self.action_assignments[self.left_idx] = self.left_action_embed + self.action_assignments[self.right_idx] = self.right_action_embed + self.action_mask[self.left_idx] = 1 + self.action_mask[self.right_idx] = 1 + + def reset(self): + self.update_avail_actions() + return { + "action_mask": self.action_mask, + "avail_actions": self.action_assignments, + "cart": self.wrapped.reset(), + } + + def step(self, action): + if action == self.left_idx: + actual_action = 0 + elif action == self.right_idx: + actual_action = 1 + else: + raise ValueError( + "Chosen action was not one of the non-zero action embeddings", + action, self.action_assignments, self.action_mask, + self.left_idx, self.right_idx) + orig_obs, rew, done, info = self.wrapped.step(actual_action) + self.update_avail_actions() + obs = { + "action_mask": self.action_mask, + "avail_actions": self.action_assignments, + "cart": orig_obs, + } + return obs, rew, done, info diff --git a/rllib/examples/env/random_env.py b/rllib/examples/env/random_env.py new file mode 100644 index 000000000..66cd63b5c --- /dev/null +++ b/rllib/examples/env/random_env.py @@ -0,0 +1,45 @@ +import gym +from gym.spaces import Tuple +import numpy as np + + +class RandomEnv(gym.Env): + """A randomly acting environment. + + Can be instantiated with arbitrary action-, observation-, and reward + spaces. Observations and rewards are generated by simply sampling from the + observation/reward spaces. The probability of a `done=True` can be + configured as well. + """ + + def __init__(self, config): + # Action space. + self.action_space = config["action_space"] + # Observation space from which to sample. + self.observation_space = config["observation_space"] + # Reward space from which to sample. + self.reward_space = config.get( + "reward_space", + gym.spaces.Box(low=-1.0, high=1.0, shape=(), dtype=np.float32)) + # Chance that an episode ends at any step. + self.p_done = config.get("p_done", 0.1) + # Whether to check action bounds. + self.check_action_bounds = config.get("check_action_bounds", False) + + def reset(self): + return self.observation_space.sample() + + def step(self, action): + if self.check_action_bounds and not self.action_space.contains(action): + raise ValueError("Illegal action for {}: {}".format( + self.action_space, action)) + if (isinstance(self.action_space, Tuple) + and len(action) != len(self.action_space.spaces)): + raise ValueError("Illegal action for {}: {}".format( + self.action_space, action)) + + return self.observation_space.sample(), \ + float(self.reward_space.sample()), \ + bool(np.random.choice( + [True, False], p=[self.p_done, 1.0 - self.p_done] + )), {} diff --git a/rllib/examples/env/repeat_after_me_env.py b/rllib/examples/env/repeat_after_me_env.py new file mode 100644 index 000000000..ae028034c --- /dev/null +++ b/rllib/examples/env/repeat_after_me_env.py @@ -0,0 +1,31 @@ +import gym +from gym.spaces import Discrete +import random + + +class RepeatAfterMeEnv(gym.Env): + """Env in which the observation at timestep minus n must be repeated.""" + + def __init__(self, config): + self.observation_space = Discrete(2) + self.action_space = Discrete(2) + self.delay = config["repeat_delay"] + assert self.delay >= 1, "`repeat_delay` must be at least 1!" + self.history = [] + + def reset(self): + self.history = [0] * self.delay + return self._next_obs() + + def step(self, action): + if action == self.history[-(1 + self.delay)]: + reward = 1 + else: + reward = -1 + done = len(self.history) > 100 + return self._next_obs(), reward, done, {} + + def _next_obs(self): + token = random.choice([0, 1]) + self.history.append(token) + return token diff --git a/rllib/examples/env/repeat_initial_obs_env.py b/rllib/examples/env/repeat_initial_obs_env.py new file mode 100644 index 000000000..7e5d57d4f --- /dev/null +++ b/rllib/examples/env/repeat_initial_obs_env.py @@ -0,0 +1,32 @@ +import gym +from gym.spaces import Discrete +import random + + +class RepeatInitialObsEnv(gym.Env): + """Env in which the initial observation has to be repeated all the time. + + Runs for n steps. + r=1 if action correct, -1 otherwise (max. R=100). + """ + + def __init__(self, episode_len=100): + self.observation_space = Discrete(2) + self.action_space = Discrete(2) + self.token = None + self.episode_len = episode_len + self.num_steps = 0 + + def reset(self): + self.token = random.choice([0, 1]) + self.num_steps = 0 + return self.token + + def step(self, action): + if action == self.token: + reward = 1 + else: + reward = -1 + self.num_steps += 1 + done = self.num_steps >= self.episode_len + return 0, reward, done, {} diff --git a/rllib/examples/env/rock_paper_scissors.py b/rllib/examples/env/rock_paper_scissors.py new file mode 100644 index 000000000..a4480013c --- /dev/null +++ b/rllib/examples/env/rock_paper_scissors.py @@ -0,0 +1,82 @@ +from gym.spaces import Discrete + +from ray.rllib.env.multi_agent_env import MultiAgentEnv + + +class RockPaperScissors(MultiAgentEnv): + """Two-player environment for the famous rock paper scissors game. + + The observation is simply the last opponent action.""" + + ROCK = 0 + PAPER = 1 + SCISSORS = 2 + LIZARD = 3 + SPOCK = 4 + + def __init__(self, config): + self.action_space = Discrete(3) + self.observation_space = Discrete(3) + self.sheldon_cooper = config.get("sheldon_cooper", False) + self.player1 = "player1" + self.player2 = "player2" + self.last_move = None + self.num_moves = 0 + + def reset(self): + self.last_move = (0, 0) + self.num_moves = 0 + return { + self.player1: self.last_move[1], + self.player2: self.last_move[0], + } + + def step(self, action_dict): + move1 = action_dict[self.player1] + move2 = action_dict[self.player2] + if self.sheldon_cooper is False: + assert move1 not in [self.LIZARD, self.SPOCK] + assert move2 not in [self.LIZARD, self.SPOCK] + + self.last_move = (move1, move2) + obs = { + self.player1: self.last_move[1], + self.player2: self.last_move[0], + } + r1, r2 = { + (self.ROCK, self.ROCK): (0, 0), + (self.ROCK, self.PAPER): (-1, 1), + (self.ROCK, self.SCISSORS): (1, -1), + (self.PAPER, self.ROCK): (1, -1), + (self.PAPER, self.PAPER): (0, 0), + (self.PAPER, self.SCISSORS): (-1, 1), + (self.SCISSORS, self.ROCK): (-1, 1), + (self.SCISSORS, self.PAPER): (1, -1), + (self.SCISSORS, self.SCISSORS): (0, 0), + # Sheldon Cooper extension: + (self.LIZARD, self.LIZARD): (0, 0), + (self.LIZARD, self.SPOCK): (1, -1), # Lizard poisons Spock + (self.LIZARD, self.ROCK): (-1, 1), # Rock crushes lizard + (self.LIZARD, self.PAPER): (1, -1), # Lizard eats paper + (self.LIZARD, self.SCISSORS): (-1, 1), # Scissors decapitate Lizrd + (self.ROCK, self.LIZARD): (1, -1), # Rock crushes lizard + (self.PAPER, self.LIZARD): (-1, 1), # Lizard eats paper + (self.SCISSORS, self.LIZARD): (1, -1), # Scissors decapitate Lizrd + (self.SPOCK, self.SPOCK): (0, 0), + (self.SPOCK, self.LIZARD): (-1, 1), # Lizard poisons Spock + (self.SPOCK, self.ROCK): (1, -1), # Spock vaporizes rock + (self.SPOCK, self.PAPER): (-1, 1), # Paper disproves Spock + (self.SPOCK, self.SCISSORS): (1, -1), # Spock smashes scissors + (self.ROCK, self.SPOCK): (-1, 1), # Spock vaporizes rock + (self.PAPER, self.SPOCK): (1, -1), # Paper disproves Spock + (self.SCISSORS, self.SPOCK): (-1, 1), # Spock smashes scissors + }[move1, move2] + rew = { + self.player1: r1, + self.player2: r2, + } + self.num_moves += 1 + done = { + "__all__": self.num_moves >= 10, + } + return obs, rew, done, {} diff --git a/rllib/examples/env/simple_corridor.py b/rllib/examples/env/simple_corridor.py new file mode 100644 index 000000000..6529f9dbb --- /dev/null +++ b/rllib/examples/env/simple_corridor.py @@ -0,0 +1,35 @@ +import gym +from gym.spaces import Box, Discrete +import numpy as np + + +class SimpleCorridor(gym.Env): + """Example of a custom env in which you have to walk down a corridor. + + You can configure the length of the corridor via the env config.""" + + def __init__(self, config): + self.end_pos = config["corridor_length"] + self.cur_pos = 0 + self.action_space = Discrete(2) + self.observation_space = Box( + 0.0, self.end_pos, shape=(1, ), dtype=np.float32) + + def set_corridor_length(self, length): + self.end_pos = length + self.observation_space = Box( + 0.0, self.end_pos, shape=(1, ), dtype=np.float32) + print("Updated corridor length to {}".format(length)) + + def reset(self): + self.cur_pos = 0 + return [self.cur_pos] + + def step(self, action): + assert action in [0, 1], action + if action == 0 and self.cur_pos > 0: + self.cur_pos -= 1 + elif action == 1: + self.cur_pos += 1 + done = self.cur_pos >= self.end_pos + return [self.cur_pos], 1 if done else 0, done, {} diff --git a/rllib/examples/env/stateless_cartpole.py b/rllib/examples/env/stateless_cartpole.py new file mode 100644 index 000000000..dd720414e --- /dev/null +++ b/rllib/examples/env/stateless_cartpole.py @@ -0,0 +1,152 @@ +import math +import gym +from gym import spaces +from gym.utils import seeding +import numpy as np + + +class StatelessCartPole(gym.Env): + """Partially observable variant of the CartPole gym environment. + + https://github.com/openai/gym/blob/master/gym/envs/classic_control/ + cartpole.py + + We delete the velocity component of the state, so that it can only be + solved by a LSTM policy. + """ + + metadata = { + "render.modes": ["human", "rgb_array"], + "video.frames_per_second": 60 + } + + def __init__(self, config=None): + self.gravity = 9.8 + self.masscart = 1.0 + self.masspole = 0.1 + self.total_mass = (self.masspole + self.masscart) + self.length = 0.5 # actually half the pole's length + self.polemass_length = (self.masspole * self.length) + self.force_mag = 10.0 + self.tau = 0.02 # seconds between state updates + + # Angle at which to fail the episode + self.theta_threshold_radians = 12 * 2 * math.pi / 360 + self.x_threshold = 2.4 + + high = np.array([ + self.x_threshold * 2, + self.theta_threshold_radians * 2, + ]) + + self.action_space = spaces.Discrete(2) + self.observation_space = spaces.Box(-high, high) + + self.seed() + self.viewer = None + self.state = None + + self.steps_beyond_done = None + + def seed(self, seed=None): + self.np_random, seed = seeding.np_random(seed) + return [seed] + + def step(self, action): + assert self.action_space.contains( + action), "%r (%s) invalid" % (action, type(action)) + state = self.state + x, x_dot, theta, theta_dot = state + force = self.force_mag if action == 1 else -self.force_mag + costheta = math.cos(theta) + sintheta = math.sin(theta) + temp = (force + self.polemass_length * theta_dot * theta_dot * sintheta + ) / self.total_mass + thetaacc = (self.gravity * sintheta - costheta * temp) / ( + self.length * + (4.0 / 3.0 - self.masspole * costheta * costheta / self.total_mass) + ) + xacc = (temp - + self.polemass_length * thetaacc * costheta / self.total_mass) + x = x + self.tau * x_dot + x_dot = x_dot + self.tau * xacc + theta = theta + self.tau * theta_dot + theta_dot = theta_dot + self.tau * thetaacc + self.state = (x, x_dot, theta, theta_dot) + done = (x < -self.x_threshold or x > self.x_threshold + or theta < -self.theta_threshold_radians + or theta > self.theta_threshold_radians) + done = bool(done) + + if not done: + reward = 1.0 + elif self.steps_beyond_done is None: + # Pole just fell! + self.steps_beyond_done = 0 + reward = 1.0 + else: + self.steps_beyond_done += 1 + reward = 0.0 + + rv = np.r_[self.state[0], self.state[2]] + return rv, reward, done, {} + + def reset(self): + self.state = self.np_random.uniform(low=-0.05, high=0.05, size=(4, )) + self.steps_beyond_done = None + + rv = np.r_[self.state[0], self.state[2]] + return rv + + def render(self, mode="human"): + screen_width = 600 + screen_height = 400 + + world_width = self.x_threshold * 2 + scale = screen_width / world_width + carty = 100 # TOP OF CART + polewidth = 10.0 + polelen = scale * 1.0 + cartwidth = 50.0 + cartheight = 30.0 + + if self.viewer is None: + from gym.envs.classic_control import rendering + self.viewer = rendering.Viewer(screen_width, screen_height) + l, r, t, b = (-cartwidth / 2, cartwidth / 2, cartheight / 2, + -cartheight / 2) + axleoffset = cartheight / 4.0 + cart = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)]) + self.carttrans = rendering.Transform() + cart.add_attr(self.carttrans) + self.viewer.add_geom(cart) + l, r, t, b = (-polewidth / 2, polewidth / 2, + polelen - polewidth / 2, -polewidth / 2) + pole = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)]) + pole.set_color(.8, .6, .4) + self.poletrans = rendering.Transform(translation=(0, axleoffset)) + pole.add_attr(self.poletrans) + pole.add_attr(self.carttrans) + self.viewer.add_geom(pole) + self.axle = rendering.make_circle(polewidth / 2) + self.axle.add_attr(self.poletrans) + self.axle.add_attr(self.carttrans) + self.axle.set_color(.5, .5, .8) + self.viewer.add_geom(self.axle) + self.track = rendering.Line((0, carty), (screen_width, carty)) + self.track.set_color(0, 0, 0) + self.viewer.add_geom(self.track) + + if self.state is None: + return None + + x = self.state + cartx = x[0] * scale + screen_width / 2.0 # MIDDLE OF CART + self.carttrans.set_translation(cartx, carty) + self.poletrans.set_rotation(-x[2]) + + return self.viewer.render(return_rgb_array=mode == "rgb_array") + + def close(self): + if self.viewer: + self.viewer.close() diff --git a/rllib/examples/env/two_step_game.py b/rllib/examples/env/two_step_game.py new file mode 100644 index 000000000..c68bfad7e --- /dev/null +++ b/rllib/examples/env/two_step_game.py @@ -0,0 +1,107 @@ +from gym.spaces import MultiDiscrete, Dict, Discrete +import numpy as np + +from ray.rllib.env.multi_agent_env import MultiAgentEnv, ENV_STATE + + +class TwoStepGame(MultiAgentEnv): + action_space = Discrete(2) + + def __init__(self, env_config): + self.state = None + self.agent_1 = 0 + self.agent_2 = 1 + # MADDPG emits action logits instead of actual discrete actions + self.actions_are_logits = env_config.get("actions_are_logits", False) + self.one_hot_state_encoding = env_config.get("one_hot_state_encoding", + False) + self.with_state = env_config.get("separate_state_space", False) + + if not self.one_hot_state_encoding: + self.observation_space = Discrete(6) + self.with_state = False + else: + # Each agent gets the full state (one-hot encoding of which of the + # three states are active) as input with the receiving agent's + # ID (1 or 2) concatenated onto the end. + if self.with_state: + self.observation_space = Dict({ + "obs": MultiDiscrete([2, 2, 2, 3]), + ENV_STATE: MultiDiscrete([2, 2, 2]) + }) + else: + self.observation_space = MultiDiscrete([2, 2, 2, 3]) + + def reset(self): + self.state = np.array([1, 0, 0]) + return self._obs() + + def step(self, action_dict): + if self.actions_are_logits: + action_dict = { + k: np.random.choice([0, 1], p=v) + for k, v in action_dict.items() + } + + state_index = np.flatnonzero(self.state) + if state_index == 0: + action = action_dict[self.agent_1] + assert action in [0, 1], action + if action == 0: + self.state = np.array([0, 1, 0]) + else: + self.state = np.array([0, 0, 1]) + global_rew = 0 + done = False + elif state_index == 1: + global_rew = 7 + done = True + else: + if action_dict[self.agent_1] == 0 and action_dict[self. + agent_2] == 0: + global_rew = 0 + elif action_dict[self.agent_1] == 1 and action_dict[self. + agent_2] == 1: + global_rew = 8 + else: + global_rew = 1 + done = True + + rewards = { + self.agent_1: global_rew / 2.0, + self.agent_2: global_rew / 2.0 + } + obs = self._obs() + dones = {"__all__": done} + infos = {} + return obs, rewards, dones, infos + + def _obs(self): + if self.with_state: + return { + self.agent_1: { + "obs": self.agent_1_obs(), + ENV_STATE: self.state + }, + self.agent_2: { + "obs": self.agent_2_obs(), + ENV_STATE: self.state + } + } + else: + return { + self.agent_1: self.agent_1_obs(), + self.agent_2: self.agent_2_obs() + } + + def agent_1_obs(self): + if self.one_hot_state_encoding: + return np.concatenate([self.state, [1]]) + else: + return np.flatnonzero(self.state)[0] + + def agent_2_obs(self): + if self.one_hot_state_encoding: + return np.concatenate([self.state, [2]]) + else: + return np.flatnonzero(self.state)[0] + 3 diff --git a/rllib/examples/env/windy_maze_env.py b/rllib/examples/env/windy_maze_env.py new file mode 100644 index 000000000..8ffc4ffda --- /dev/null +++ b/rllib/examples/env/windy_maze_env.py @@ -0,0 +1,146 @@ +import gym +from gym.spaces import Box, Discrete, Tuple +import logging +import random + +from ray.rllib.env import MultiAgentEnv + +logger = logging.getLogger(__name__) + +# Agent has to traverse the maze from the starting position S -> F +# Observation space [x_pos, y_pos, wind_direction] +# Action space: stay still OR move in current wind direction +MAP_DATA = """ +######### +#S # +####### # + # # + # # +####### # +#F # +#########""" + + +class WindyMazeEnv(gym.Env): + def __init__(self, env_config): + self.map = [m for m in MAP_DATA.split("\n") if m] + self.x_dim = len(self.map) + self.y_dim = len(self.map[0]) + logger.info("Loaded map {} {}".format(self.x_dim, self.y_dim)) + for x in range(self.x_dim): + for y in range(self.y_dim): + if self.map[x][y] == "S": + self.start_pos = (x, y) + elif self.map[x][y] == "F": + self.end_pos = (x, y) + logger.info("Start pos {} end pos {}".format(self.start_pos, + self.end_pos)) + self.observation_space = Tuple([ + Box(0, 100, shape=(2, )), # (x, y) + Discrete(4), # wind direction (N, E, S, W) + ]) + self.action_space = Discrete(2) # whether to move or not + + def reset(self): + self.wind_direction = random.choice([0, 1, 2, 3]) + self.pos = self.start_pos + self.num_steps = 0 + return [[self.pos[0], self.pos[1]], self.wind_direction] + + def step(self, action): + if action == 1: + self.pos = self._get_new_pos(self.pos, self.wind_direction) + self.num_steps += 1 + self.wind_direction = random.choice([0, 1, 2, 3]) + at_goal = self.pos == self.end_pos + done = at_goal or self.num_steps >= 200 + return ([[self.pos[0], self.pos[1]], self.wind_direction], + 100 * int(at_goal), done, {}) + + def _get_new_pos(self, pos, direction): + if direction == 0: + new_pos = (pos[0] - 1, pos[1]) + elif direction == 1: + new_pos = (pos[0], pos[1] + 1) + elif direction == 2: + new_pos = (pos[0] + 1, pos[1]) + elif direction == 3: + new_pos = (pos[0], pos[1] - 1) + if (new_pos[0] >= 0 and new_pos[0] < self.x_dim and new_pos[1] >= 0 + and new_pos[1] < self.y_dim + and self.map[new_pos[0]][new_pos[1]] != "#"): + return new_pos + else: + return pos # did not move + + +class HierarchicalWindyMazeEnv(MultiAgentEnv): + def __init__(self, env_config): + self.flat_env = WindyMazeEnv(env_config) + + def reset(self): + self.cur_obs = self.flat_env.reset() + self.current_goal = None + self.steps_remaining_at_level = None + self.num_high_level_steps = 0 + # current low level agent id. This must be unique for each high level + # step since agent ids cannot be reused. + self.low_level_agent_id = "low_level_{}".format( + self.num_high_level_steps) + return { + "high_level_agent": self.cur_obs, + } + + def step(self, action_dict): + assert len(action_dict) == 1, action_dict + if "high_level_agent" in action_dict: + return self._high_level_step(action_dict["high_level_agent"]) + else: + return self._low_level_step(list(action_dict.values())[0]) + + def _high_level_step(self, action): + logger.debug("High level agent sets goal".format(action)) + self.current_goal = action + self.steps_remaining_at_level = 25 + self.num_high_level_steps += 1 + self.low_level_agent_id = "low_level_{}".format( + self.num_high_level_steps) + obs = {self.low_level_agent_id: [self.cur_obs, self.current_goal]} + rew = {self.low_level_agent_id: 0} + done = {"__all__": False} + return obs, rew, done, {} + + def _low_level_step(self, action): + logger.debug("Low level agent step {}".format(action)) + self.steps_remaining_at_level -= 1 + cur_pos = tuple(self.cur_obs[0]) + goal_pos = self.flat_env._get_new_pos(cur_pos, self.current_goal) + + # Step in the actual env + f_obs, f_rew, f_done, _ = self.flat_env.step(action) + new_pos = tuple(f_obs[0]) + self.cur_obs = f_obs + + # Calculate low-level agent observation and reward + obs = {self.low_level_agent_id: [f_obs, self.current_goal]} + if new_pos != cur_pos: + if new_pos == goal_pos: + rew = {self.low_level_agent_id: 1} + else: + rew = {self.low_level_agent_id: -1} + else: + rew = {self.low_level_agent_id: 0} + + # Handle env termination & transitions back to higher level + done = {"__all__": False} + if f_done: + done["__all__"] = True + logger.debug("high level final reward {}".format(f_rew)) + rew["high_level_agent"] = f_rew + obs["high_level_agent"] = f_obs + elif self.steps_remaining_at_level == 0: + done[self.low_level_agent_id] = True + rew["high_level_agent"] = 0 + obs["high_level_agent"] = f_obs + + return obs, rew, done, {} diff --git a/rllib/examples/hierarchical_training.py b/rllib/examples/hierarchical_training.py index 8fb09a5fe..37f33883f 100644 --- a/rllib/examples/hierarchical_training.py +++ b/rllib/examples/hierarchical_training.py @@ -23,169 +23,40 @@ using --flat in this example. """ import argparse -import random -import gym -from gym.spaces import Box, Discrete, Tuple +from gym.spaces import Discrete, Tuple import logging import ray from ray import tune +from ray.rllib.examples.env.windy_maze_env import WindyMazeEnv, \ + HierarchicalWindyMazeEnv from ray.tune import function -from ray.rllib.env import MultiAgentEnv parser = argparse.ArgumentParser() parser.add_argument("--flat", action="store_true") - -# Agent has to traverse the maze from the starting position S -> F -# Observation space [x_pos, y_pos, wind_direction] -# Action space: stay still OR move in current wind direction -MAP_DATA = """ -######### -#S # -####### # - # # - # # -####### # -#F # -#########""" +parser.add_argument("--torch", action="store_true") +parser.add_argument("--stop-reward", type=float, default=0.0) +parser.add_argument("--stop-timesteps", type=int, default=100000) logger = logging.getLogger(__name__) - -class WindyMazeEnv(gym.Env): - def __init__(self, env_config): - self.map = [m for m in MAP_DATA.split("\n") if m] - self.x_dim = len(self.map) - self.y_dim = len(self.map[0]) - logger.info("Loaded map {} {}".format(self.x_dim, self.y_dim)) - for x in range(self.x_dim): - for y in range(self.y_dim): - if self.map[x][y] == "S": - self.start_pos = (x, y) - elif self.map[x][y] == "F": - self.end_pos = (x, y) - logger.info("Start pos {} end pos {}".format(self.start_pos, - self.end_pos)) - self.observation_space = Tuple([ - Box(0, 100, shape=(2, )), # (x, y) - Discrete(4), # wind direction (N, E, S, W) - ]) - self.action_space = Discrete(2) # whether to move or not - - def reset(self): - self.wind_direction = random.choice([0, 1, 2, 3]) - self.pos = self.start_pos - self.num_steps = 0 - return [[self.pos[0], self.pos[1]], self.wind_direction] - - def step(self, action): - if action == 1: - self.pos = self._get_new_pos(self.pos, self.wind_direction) - self.num_steps += 1 - self.wind_direction = random.choice([0, 1, 2, 3]) - at_goal = self.pos == self.end_pos - done = at_goal or self.num_steps >= 200 - return ([[self.pos[0], self.pos[1]], self.wind_direction], - 100 * int(at_goal), done, {}) - - def _get_new_pos(self, pos, direction): - if direction == 0: - new_pos = (pos[0] - 1, pos[1]) - elif direction == 1: - new_pos = (pos[0], pos[1] + 1) - elif direction == 2: - new_pos = (pos[0] + 1, pos[1]) - elif direction == 3: - new_pos = (pos[0], pos[1] - 1) - if (new_pos[0] >= 0 and new_pos[0] < self.x_dim and new_pos[1] >= 0 - and new_pos[1] < self.y_dim - and self.map[new_pos[0]][new_pos[1]] != "#"): - return new_pos - else: - return pos # did not move - - -class HierarchicalWindyMazeEnv(MultiAgentEnv): - def __init__(self, env_config): - self.flat_env = WindyMazeEnv(env_config) - - def reset(self): - self.cur_obs = self.flat_env.reset() - self.current_goal = None - self.steps_remaining_at_level = None - self.num_high_level_steps = 0 - # current low level agent id. This must be unique for each high level - # step since agent ids cannot be reused. - self.low_level_agent_id = "low_level_{}".format( - self.num_high_level_steps) - return { - "high_level_agent": self.cur_obs, - } - - def step(self, action_dict): - assert len(action_dict) == 1, action_dict - if "high_level_agent" in action_dict: - return self._high_level_step(action_dict["high_level_agent"]) - else: - return self._low_level_step(list(action_dict.values())[0]) - - def _high_level_step(self, action): - logger.debug("High level agent sets goal".format(action)) - self.current_goal = action - self.steps_remaining_at_level = 25 - self.num_high_level_steps += 1 - self.low_level_agent_id = "low_level_{}".format( - self.num_high_level_steps) - obs = {self.low_level_agent_id: [self.cur_obs, self.current_goal]} - rew = {self.low_level_agent_id: 0} - done = {"__all__": False} - return obs, rew, done, {} - - def _low_level_step(self, action): - logger.debug("Low level agent step {}".format(action)) - self.steps_remaining_at_level -= 1 - cur_pos = tuple(self.cur_obs[0]) - goal_pos = self.flat_env._get_new_pos(cur_pos, self.current_goal) - - # Step in the actual env - f_obs, f_rew, f_done, _ = self.flat_env.step(action) - new_pos = tuple(f_obs[0]) - self.cur_obs = f_obs - - # Calculate low-level agent observation and reward - obs = {self.low_level_agent_id: [f_obs, self.current_goal]} - if new_pos != cur_pos: - if new_pos == goal_pos: - rew = {self.low_level_agent_id: 1} - else: - rew = {self.low_level_agent_id: -1} - else: - rew = {self.low_level_agent_id: 0} - - # Handle env termination & transitions back to higher level - done = {"__all__": False} - if f_done: - done["__all__"] = True - logger.debug("high level final reward {}".format(f_rew)) - rew["high_level_agent"] = f_rew - obs["high_level_agent"] = f_obs - elif self.steps_remaining_at_level == 0: - done[self.low_level_agent_id] = True - rew["high_level_agent"] = 0 - obs["high_level_agent"] = f_obs - - return obs, rew, done, {} - - if __name__ == "__main__": args = parser.parse_args() ray.init() + + stop = { + "episode_reward_mean": args.stop_reward, + "timesteps_total": args.stop_timesteps, + } + if args.flat: - tune.run( + results = tune.run( "PPO", + stop=stop, config={ "env": WindyMazeEnv, "num_workers": 0, + "use_pytorch": args.torch, }, ) else: @@ -197,28 +68,39 @@ if __name__ == "__main__": else: return "high_level_policy" - tune.run( - "PPO", - config={ - "env": HierarchicalWindyMazeEnv, - "num_workers": 0, - "log_level": "INFO", - "entropy_coeff": 0.01, - "multiagent": { - "policies": { - "high_level_policy": (None, maze.observation_space, - Discrete(4), { - "gamma": 0.9 - }), - "low_level_policy": (None, - Tuple([ - maze.observation_space, - Discrete(4) - ]), maze.action_space, { - "gamma": 0.0 - }), - }, - "policy_mapping_fn": function(policy_mapping_fn), + config = { + "env": HierarchicalWindyMazeEnv, + "num_workers": 0, + "log_level": "INFO", + "entropy_coeff": 0.01, + "multiagent": { + "policies": { + "high_level_policy": (None, maze.observation_space, + Discrete(4), { + "gamma": 0.9 + }), + "low_level_policy": (None, + Tuple([ + maze.observation_space, + Discrete(4) + ]), maze.action_space, { + "gamma": 0.0 + }), }, + "policy_mapping_fn": function(policy_mapping_fn), }, + "use_pytorch": args.torch, + } + + results = tune.run( + "PPO", + stop=stop, + config=config, ) + + # Error if stop-reward not reached. + if results.trials[0].last_result["episode_reward_mean"] < \ + args.stop_reward: + raise ValueError("`stop-reward` of {} not reached!".format( + args.stop_reward)) + print("ok") diff --git a/rllib/examples/multi_agent_cartpole.py b/rllib/examples/multi_agent_cartpole.py index 40ade9eef..68a931ea5 100644 --- a/rllib/examples/multi_agent_cartpole.py +++ b/rllib/examples/multi_agent_cartpole.py @@ -15,11 +15,10 @@ import random import ray from ray import tune +from ray.rllib.examples.env.multi_agent import MultiAgentCartPole from ray.rllib.models import ModelCatalog from ray.rllib.models.modelv2 import ModelV2 from ray.rllib.models.tf.tf_modelv2 import TFModelV2 -from ray.rllib.tests.test_multi_agent_env import MultiCartpole -from ray.tune.registry import register_env from ray.rllib.utils import try_import_tf from ray.rllib.utils.annotations import override @@ -105,8 +104,6 @@ if __name__ == "__main__": args = parser.parse_args() ray.init(num_cpus=args.num_cpus or None) - # Simple environment with `num_agents` independent cartpole entities - register_env("multi_cartpole", lambda _: MultiCartpole(args.num_agents)) ModelCatalog.register_custom_model("model1", CustomModel1) ModelCatalog.register_custom_model("model2", CustomModel2) single_env = gym.make("CartPole-v0") @@ -134,7 +131,10 @@ if __name__ == "__main__": "PPO", stop={"training_iteration": args.num_iters}, config={ - "env": "multi_cartpole", + "env": MultiAgentCartPole, + "env_config": { + "num_agents": args.num_agents, + }, "log_level": "DEBUG", "simple_optimizer": args.simple, "num_sgd_iter": 10, diff --git a/rllib/examples/multi_agent_custom_policy.py b/rllib/examples/multi_agent_custom_policy.py index 3342ff718..96bb48fd7 100644 --- a/rllib/examples/multi_agent_custom_policy.py +++ b/rllib/examples/multi_agent_custom_policy.py @@ -18,8 +18,8 @@ import gym import ray from ray import tune +from ray.rllib.examples.env.multi_agent import MultiAgentCartPole from ray.rllib.policy import Policy -from ray.rllib.tests.test_multi_agent_env import MultiCartpole from ray.tune.registry import register_env parser = argparse.ArgumentParser() @@ -50,7 +50,8 @@ if __name__ == "__main__": ray.init() # Simple environment with 4 independent cartpole entities - register_env("multi_cartpole", lambda _: MultiCartpole(4)) + register_env("multi_agent_cartpole", + lambda _: MultiAgentCartPole({"num_agents": 4})) single_env = gym.make("CartPole-v0") obs_space = single_env.observation_space act_space = single_env.action_space @@ -59,7 +60,7 @@ if __name__ == "__main__": "PG", stop={"training_iteration": args.num_iters}, config={ - "env": "multi_cartpole", + "env": "multi_agent_cartpole", "multiagent": { "policies": { "pg_policy": (None, obs_space, act_space, {}), diff --git a/rllib/examples/multi_agent_two_trainers.py b/rllib/examples/multi_agent_two_trainers.py index 994bbd3f1..2956bc767 100644 --- a/rllib/examples/multi_agent_two_trainers.py +++ b/rllib/examples/multi_agent_two_trainers.py @@ -16,7 +16,7 @@ from ray.rllib.agents.dqn.dqn import DQNTrainer from ray.rllib.agents.dqn.dqn_tf_policy import DQNTFPolicy from ray.rllib.agents.ppo.ppo import PPOTrainer from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy -from ray.rllib.tests.test_multi_agent_env import MultiCartpole +from ray.rllib.examples.env.multi_agent import MultiAgentCartPole from ray.tune.logger import pretty_print from ray.tune.registry import register_env @@ -28,7 +28,8 @@ if __name__ == "__main__": ray.init() # Simple environment with 4 independent cartpole entities - register_env("multi_cartpole", lambda _: MultiCartpole(4)) + register_env("multi_agent_cartpole", + lambda _: MultiAgentCartPole({"num_agents": 4})) single_env = gym.make("CartPole-v0") obs_space = single_env.observation_space act_space = single_env.action_space @@ -47,7 +48,7 @@ if __name__ == "__main__": return "dqn_policy" ppo_trainer = PPOTrainer( - env="multi_cartpole", + env="multi_agent_cartpole", config={ "multiagent": { "policies": policies, @@ -61,7 +62,7 @@ if __name__ == "__main__": }) dqn_trainer = DQNTrainer( - env="multi_cartpole", + env="multi_agent_cartpole", config={ "multiagent": { "policies": policies, diff --git a/rllib/examples/nested_action_spaces.py b/rllib/examples/nested_action_spaces.py index acaa66203..a3d8ad0ba 100644 --- a/rllib/examples/nested_action_spaces.py +++ b/rllib/examples/nested_action_spaces.py @@ -1,64 +1,24 @@ import argparse -import gym from gym.spaces import Dict, Tuple, Box, Discrete -import numpy as np import sys import ray from ray.tune.registry import register_env +from ray.rllib.examples.env.nested_space_repeat_after_me_env import \ + NestedSpaceRepeatAfterMeEnv from ray.rllib.utils import try_import_tree from ray.rllib.utils.framework import try_import_tf -from ray.rllib.utils.space_utils import flatten_space tf = try_import_tf() tree = try_import_tree() parser = argparse.ArgumentParser() parser.add_argument("--run", type=str, default="PPO") +parser.add_argument("--torch", action="store_true") parser.add_argument("--stop", type=int, default=90) parser.add_argument("--max-trainstop", type=int, default=90) parser.add_argument("--num-cpus", type=int, default=0) - -class NestedSpaceRepeatAfterMeEnv(gym.Env): - """Env for which policy has to repeat the (possibly complex) observation. - """ - - def __init__(self, config): - self.observation_space = config.get( - "space", Tuple([Discrete(2), - Dict({ - "a": Box(-1.0, 1.0, (2, )) - })])) - self.action_space = self.observation_space - self.flattened_action_space = flatten_space(self.action_space) - self.episode_len = config.get("episode_len", 100) - - def reset(self): - self.steps = 0 - return self._next_obs() - - def step(self, action): - self.steps += 1 - action = tree.flatten(action) - reward = 0.0 - for a, o, space in zip(action, self.current_obs_flattened, - self.flattened_action_space): - # Box: -abs(diff). - if isinstance(space, gym.spaces.Box): - reward -= np.abs(np.sum(a - o)) - # Discrete: +1.0 if exact match. - if isinstance(space, gym.spaces.Discrete): - reward += 1.0 if a == o else 0.0 - done = self.steps >= self.episode_len - return self._next_obs(), reward, done, {} - - def _next_obs(self): - self.current_obs = self.observation_space.sample() - self.current_obs_flattened = tree.flatten(self.current_obs) - return self.current_obs - - if __name__ == "__main__": args = parser.parse_args() ray.init(num_cpus=args.num_cpus or None) @@ -78,13 +38,14 @@ if __name__ == "__main__": "c": Discrete(4) }), }, - "gamma": 0.0, # No history in Env (bandit problem). - "num_workers": 0, - "num_envs_per_worker": 20, "entropy_coeff": 0.00005, # We don't want high entropy in this Env. + "gamma": 0.0, # No history in Env (bandit problem). + "lr": 0.0003, + "num_envs_per_worker": 20, "num_sgd_iter": 20, + "num_workers": 0, + "use_pytorch": args.torch, "vf_loss_coeff": 0.01, - "lr": 0.0003 } import ray.rllib.agents.ppo as ppo diff --git a/rllib/examples/parametric_action_cartpole.py b/rllib/examples/parametric_actions_cartpole.py similarity index 57% rename from rllib/examples/parametric_action_cartpole.py rename to rllib/examples/parametric_actions_cartpole.py index d55b10160..62e7dae54 100644 --- a/rllib/examples/parametric_action_cartpole.py +++ b/rllib/examples/parametric_actions_cartpole.py @@ -15,15 +15,14 @@ Working configurations are given below. """ import argparse -import random -import numpy as np -import gym -from gym.spaces import Box, Discrete, Dict +from gym.spaces import Box import ray from ray import tune from ray.rllib.agents.dqn.distributional_q_tf_model import \ DistributionalQTFModel +from ray.rllib.examples.env.parametric_actions_cartpole import \ + ParametricActionsCartPole from ray.rllib.models import ModelCatalog from ray.rllib.models.tf.fcnet_v2 import FullyConnectedNetwork from ray.rllib.models.tf.tf_modelv2 import TFModelV2 @@ -37,79 +36,6 @@ parser.add_argument("--stop", type=int, default=200) parser.add_argument("--run", type=str, default="PPO") -class ParametricActionCartpole(gym.Env): - """Parametric action version of CartPole. - - In this env there are only ever two valid actions, but we pretend there are - actually up to `max_avail_actions` actions that can be taken, and the two - valid actions are randomly hidden among this set. - - At each step, we emit a dict of: - - the actual cart observation - - a mask of valid actions (e.g., [0, 0, 1, 0, 0, 1] for 6 max avail) - - the list of action embeddings (w/ zeroes for invalid actions) (e.g., - [[0, 0], - [0, 0], - [-0.2322, -0.2569], - [0, 0], - [0, 0], - [0.7878, 1.2297]] for max_avail_actions=6) - - In a real environment, the actions embeddings would be larger than two - units of course, and also there would be a variable number of valid actions - per step instead of always [LEFT, RIGHT]. - """ - - def __init__(self, max_avail_actions): - # Use simple random 2-unit action embeddings for [LEFT, RIGHT] - self.left_action_embed = np.random.randn(2) - self.right_action_embed = np.random.randn(2) - self.action_space = Discrete(max_avail_actions) - self.wrapped = gym.make("CartPole-v0") - self.observation_space = Dict({ - "action_mask": Box(0, 1, shape=(max_avail_actions, )), - "avail_actions": Box(-10, 10, shape=(max_avail_actions, 2)), - "cart": self.wrapped.observation_space, - }) - - def update_avail_actions(self): - self.action_assignments = np.array([[0., 0.]] * self.action_space.n) - self.action_mask = np.array([0.] * self.action_space.n) - self.left_idx, self.right_idx = random.sample( - range(self.action_space.n), 2) - self.action_assignments[self.left_idx] = self.left_action_embed - self.action_assignments[self.right_idx] = self.right_action_embed - self.action_mask[self.left_idx] = 1 - self.action_mask[self.right_idx] = 1 - - def reset(self): - self.update_avail_actions() - return { - "action_mask": self.action_mask, - "avail_actions": self.action_assignments, - "cart": self.wrapped.reset(), - } - - def step(self, action): - if action == self.left_idx: - actual_action = 0 - elif action == self.right_idx: - actual_action = 1 - else: - raise ValueError( - "Chosen action was not one of the non-zero action embeddings", - action, self.action_assignments, self.action_mask, - self.left_idx, self.right_idx) - orig_obs, rew, done, info = self.wrapped.step(actual_action) - self.update_avail_actions() - obs = { - "action_mask": self.action_mask, - "avail_actions": self.action_assignments, - "cart": orig_obs, - } - return obs, rew, done, info - - class ParametricActionsModel(DistributionalQTFModel, TFModelV2): """Parametric action model that handles the dot product and masking. @@ -165,7 +91,7 @@ if __name__ == "__main__": ray.init() ModelCatalog.register_custom_model("pa_model", ParametricActionsModel) - register_env("pa_cartpole", lambda _: ParametricActionCartpole(10)) + register_env("pa_cartpole", lambda _: ParametricActionsCartPole(10)) if args.run == "DQN": cfg = { # TODO(ekl) we need to set these to prevent the masked values diff --git a/rllib/examples/random_env.py b/rllib/examples/random_env.py deleted file mode 100644 index dd3c06600..000000000 --- a/rllib/examples/random_env.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Example of a custom gym environment and model. Run this for a demo. - -This example shows: - - using a custom environment - - using a custom model - - using Tune for grid search - -You can visualize experiment results in ~/ray_results using TensorBoard. -""" - -import gym -from gym.spaces import Tuple, Discrete -import numpy as np - -from ray.rllib.agents.ppo import PPOTrainer -from ray.rllib.utils import try_import_tf - -tf = try_import_tf() - - -class RandomEnv(gym.Env): - """ - A randomly acting environment that can be instantiated with arbitrary - action and observation spaces. - """ - - def __init__(self, config): - # Action space. - self.action_space = config["action_space"] - # Observation space from which to sample. - self.observation_space = config["observation_space"] - # Reward space from which to sample. - self.reward_space = config.get( - "reward_space", - gym.spaces.Box(low=-1.0, high=1.0, shape=(), dtype=np.float32)) - # Chance that an episode ends at any step. - self.p_done = config.get("p_done", 0.1) - - def reset(self): - return self.observation_space.sample() - - def step(self, action): - return self.observation_space.sample(), \ - float(self.reward_space.sample()), \ - bool(np.random.choice( - [True, False], p=[self.p_done, 1.0 - self.p_done] - )), {} - - -if __name__ == "__main__": - trainer = PPOTrainer( - config={ - "model": { - "use_lstm": True, - }, - "vf_share_layers": False, - "num_workers": 0, # no parallelism - "env_config": { - "action_space": Discrete(2), - # Test a simple Tuple observation space. - "observation_space": Tuple([Discrete(3), - Discrete(2)]) - } - }, - env=RandomEnv, - ) - results = trainer.train() - print(results) diff --git a/rllib/examples/rock_paper_scissors_multiagent.py b/rllib/examples/rock_paper_scissors_multiagent.py index 6c6804541..1095b3272 100644 --- a/rllib/examples/rock_paper_scissors_multiagent.py +++ b/rllib/examples/rock_paper_scissors_multiagent.py @@ -14,8 +14,8 @@ from gym.spaces import Discrete from ray import tune from ray.rllib.agents.pg.pg import PGTrainer from ray.rllib.agents.pg.pg_tf_policy import PGTFPolicy +from ray.rllib.examples.env.rock_paper_scissors import RockPaperScissors from ray.rllib.policy.policy import Policy -from ray.rllib.env.multi_agent_env import MultiAgentEnv from ray.rllib.utils import try_import_tf parser = argparse.ArgumentParser() @@ -23,61 +23,6 @@ parser.add_argument("--stop", type=int, default=1000) tf = try_import_tf() -ROCK = 0 -PAPER = 1 -SCISSORS = 2 - - -class RockPaperScissorsEnv(MultiAgentEnv): - """Two-player environment for rock paper scissors. - - The observation is simply the last opponent action.""" - - def __init__(self, _): - self.action_space = Discrete(3) - self.observation_space = Discrete(3) - self.player1 = "player1" - self.player2 = "player2" - self.last_move = None - self.num_moves = 0 - - def reset(self): - self.last_move = (0, 0) - self.num_moves = 0 - return { - self.player1: self.last_move[1], - self.player2: self.last_move[0], - } - - def step(self, action_dict): - move1 = action_dict[self.player1] - move2 = action_dict[self.player2] - self.last_move = (move1, move2) - obs = { - self.player1: self.last_move[1], - self.player2: self.last_move[0], - } - r1, r2 = { - (ROCK, ROCK): (0, 0), - (ROCK, PAPER): (-1, 1), - (ROCK, SCISSORS): (1, -1), - (PAPER, ROCK): (1, -1), - (PAPER, PAPER): (0, 0), - (PAPER, SCISSORS): (-1, 1), - (SCISSORS, ROCK): (-1, 1), - (SCISSORS, PAPER): (1, -1), - (SCISSORS, SCISSORS): (0, 0), - }[move1, move2] - rew = { - self.player1: r1, - self.player2: r2, - } - self.num_moves += 1 - done = { - "__all__": self.num_moves >= 10, - } - return obs, rew, done, {} - class AlwaysSameHeuristic(Policy): """Pick a random move and stick with it for the entire episode.""" @@ -87,7 +32,12 @@ class AlwaysSameHeuristic(Policy): self.exploration = self._create_exploration() def get_initial_state(self): - return [random.choice([ROCK, PAPER, SCISSORS])] + return [ + random.choice([ + RockPaperScissors.ROCK, RockPaperScissors.PAPER, + RockPaperScissors.SCISSORS + ]) + ] def compute_actions(self, obs_batch, @@ -125,12 +75,12 @@ class BeatLastHeuristic(Policy): episodes=None, **kwargs): def successor(x): - if x[ROCK] == 1: - return PAPER - elif x[PAPER] == 1: - return SCISSORS - elif x[SCISSORS] == 1: - return ROCK + if x[RockPaperScissors.ROCK] == 1: + return RockPaperScissors.PAPER + elif x[RockPaperScissors.PAPER] == 1: + return RockPaperScissors.SCISSORS + elif x[RockPaperScissors.SCISSORS] == 1: + return RockPaperScissors.ROCK return [successor(x) for x in obs_batch], [], {} @@ -150,7 +100,7 @@ def run_same_policy(args): tune.run( "PG", stop={"timesteps_total": args.stop}, - config={"env": RockPaperScissorsEnv}) + config={"env": RockPaperScissors}) def run_heuristic_vs_learned(args, use_lstm=False, trainer="PG"): @@ -169,7 +119,7 @@ def run_heuristic_vs_learned(args, use_lstm=False, trainer="PG"): return random.choice(["always_same", "beat_last"]) config = { - "env": RockPaperScissorsEnv, + "env": RockPaperScissors, "gamma": 0.9, "num_workers": 0, "num_envs_per_worker": 4, diff --git a/rllib/examples/twostep_game.py b/rllib/examples/twostep_game.py index 1ed8cb7cb..4252312ca 100644 --- a/rllib/examples/twostep_game.py +++ b/rllib/examples/twostep_game.py @@ -11,123 +11,18 @@ See also: centralized_critic.py for centralized critic PPO on this game. import argparse from gym.spaces import Tuple, MultiDiscrete, Dict, Discrete -import numpy as np import ray from ray import tune from ray.tune import register_env, grid_search -from ray.rllib.env.multi_agent_env import MultiAgentEnv -from ray.rllib.agents.qmix.qmix_policy import ENV_STATE +from ray.rllib.env.multi_agent_env import ENV_STATE +from ray.rllib.examples.env.two_step_game import TwoStepGame parser = argparse.ArgumentParser() parser.add_argument("--stop", type=int, default=50000) parser.add_argument("--run", type=str, default="PG") parser.add_argument("--num-cpus", type=int, default=0) - -class TwoStepGame(MultiAgentEnv): - action_space = Discrete(2) - - def __init__(self, env_config): - self.state = None - self.agent_1 = 0 - self.agent_2 = 1 - # MADDPG emits action logits instead of actual discrete actions - self.actions_are_logits = env_config.get("actions_are_logits", False) - self.one_hot_state_encoding = env_config.get("one_hot_state_encoding", - False) - self.with_state = env_config.get("separate_state_space", False) - - if not self.one_hot_state_encoding: - self.observation_space = Discrete(6) - self.with_state = False - else: - # Each agent gets the full state (one-hot encoding of which of the - # three states are active) as input with the receiving agent's - # ID (1 or 2) concatenated onto the end. - if self.with_state: - self.observation_space = Dict({ - "obs": MultiDiscrete([2, 2, 2, 3]), - ENV_STATE: MultiDiscrete([2, 2, 2]) - }) - else: - self.observation_space = MultiDiscrete([2, 2, 2, 3]) - - def reset(self): - self.state = np.array([1, 0, 0]) - return self._obs() - - def step(self, action_dict): - if self.actions_are_logits: - action_dict = { - k: np.random.choice([0, 1], p=v) - for k, v in action_dict.items() - } - - state_index = np.flatnonzero(self.state) - if state_index == 0: - action = action_dict[self.agent_1] - assert action in [0, 1], action - if action == 0: - self.state = np.array([0, 1, 0]) - else: - self.state = np.array([0, 0, 1]) - global_rew = 0 - done = False - elif state_index == 1: - global_rew = 7 - done = True - else: - if action_dict[self.agent_1] == 0 and action_dict[self. - agent_2] == 0: - global_rew = 0 - elif action_dict[self.agent_1] == 1 and action_dict[self. - agent_2] == 1: - global_rew = 8 - else: - global_rew = 1 - done = True - - rewards = { - self.agent_1: global_rew / 2.0, - self.agent_2: global_rew / 2.0 - } - obs = self._obs() - dones = {"__all__": done} - infos = {} - return obs, rewards, dones, infos - - def _obs(self): - if self.with_state: - return { - self.agent_1: { - "obs": self.agent_1_obs(), - ENV_STATE: self.state - }, - self.agent_2: { - "obs": self.agent_2_obs(), - ENV_STATE: self.state - } - } - else: - return { - self.agent_1: self.agent_1_obs(), - self.agent_2: self.agent_2_obs() - } - - def agent_1_obs(self): - if self.one_hot_state_encoding: - return np.concatenate([self.state, [1]]) - else: - return np.flatnonzero(self.state)[0] - - def agent_2_obs(self): - if self.one_hot_state_encoding: - return np.concatenate([self.state, [2]]) - else: - return np.flatnonzero(self.state)[0] + 3 - - if __name__ == "__main__": args = parser.parse_args() diff --git a/rllib/rollout.py b/rllib/rollout.py index 44080536e..841ac2e21 100755 --- a/rllib/rollout.py +++ b/rllib/rollout.py @@ -33,7 +33,7 @@ Example Usage via executable: # Note: if you use any custom models or envs, register them here first, e.g.: # # ModelCatalog.register_custom_model("pa_model", ParametricActionsModel) -# register_env("pa_cartpole", lambda _: ParametricActionCartpole(10)) +# register_env("pa_cartpole", lambda _: ParametricActionsCartPole(10)) class RolloutSaver: diff --git a/rllib/tests/test_env_with_subprocess.py b/rllib/tests/test_env_with_subprocess.py index 3a4d92a18..ad090d87b 100644 --- a/rllib/tests/test_env_with_subprocess.py +++ b/rllib/tests/test_env_with_subprocess.py @@ -1,8 +1,4 @@ """Tests that envs clean up after themselves on agent exit.""" - -from gym.spaces import Discrete -import atexit -import gym import os import subprocess import tempfile @@ -11,58 +7,35 @@ import time import ray from ray.tune import run_experiments from ray.tune.registry import register_env - -# Dummy command to run as a subprocess with a unique name -UNIQUE_CMD = "sleep {}".format(str(time.time())) -_, UNIQUE_FILE_0 = tempfile.mkstemp("test_env_with_subprocess") -_, UNIQUE_FILE_1 = tempfile.mkstemp("test_env_with_subprocess") -_, UNIQUE_FILE_2 = tempfile.mkstemp("test_env_with_subprocess") -_, UNIQUE_FILE_3 = tempfile.mkstemp("test_env_with_subprocess") - - -class EnvWithSubprocess(gym.Env): - """Our env that spawns a subprocess.""" - - def __init__(self, config): - self.action_space = Discrete(2) - self.observation_space = Discrete(2) - # Subprocess that should be cleaned up - self.subproc = subprocess.Popen(UNIQUE_CMD.split(" "), shell=False) - self.config = config - # Exit handler should be called - atexit.register(lambda: self.subproc.kill()) - if config.worker_index == 0: - atexit.register(lambda: os.unlink(UNIQUE_FILE_0)) - else: - atexit.register(lambda: os.unlink(UNIQUE_FILE_1)) - - def close(self): - if self.config.worker_index == 0: - os.unlink(UNIQUE_FILE_2) - else: - os.unlink(UNIQUE_FILE_3) - - def reset(self): - return 0 - - def step(self, action): - return 0, 0, True, {} +from ray.rllib.examples.env.env_with_subprocess import EnvWithSubprocess def leaked_processes(): """Returns whether any subprocesses were leaked.""" result = subprocess.check_output( - "ps aux | grep '{}' | grep -v grep || true".format(UNIQUE_CMD), + "ps aux | grep '{}' | grep -v grep || true".format( + EnvWithSubprocess.UNIQUE_CMD), shell=True) return result if __name__ == "__main__": - register_env("subproc", lambda config: EnvWithSubprocess(config)) + + # Create 4 temp files, which the Env has to clean up after it's done. + _, tmp1 = tempfile.mkstemp("test_env_with_subprocess") + _, tmp2 = tempfile.mkstemp("test_env_with_subprocess") + _, tmp3 = tempfile.mkstemp("test_env_with_subprocess") + _, tmp4 = tempfile.mkstemp("test_env_with_subprocess") + register_env("subproc", lambda c: EnvWithSubprocess(c)) + ray.init() - assert os.path.exists(UNIQUE_FILE_0) - assert os.path.exists(UNIQUE_FILE_1) + # Check whether everything is ok. + assert os.path.exists(tmp1) + assert os.path.exists(tmp2) + assert os.path.exists(tmp3) + assert os.path.exists(tmp4) assert not leaked_processes() + run_experiments({ "demo": { "run": "PG", @@ -70,6 +43,12 @@ if __name__ == "__main__": "num_samples": 1, "config": { "num_workers": 1, + "env_config": { + "tmp_file1": tmp1, + "tmp_file2": tmp2, + "tmp_file3": tmp3, + "tmp_file4": tmp4, + }, }, "stop": { "training_iteration": 1 @@ -77,10 +56,12 @@ if __name__ == "__main__": }, }) time.sleep(10.0) + # Check whether processes are still running or Env has not cleaned up + # the given tmp files. leaked = leaked_processes() assert not leaked, "LEAKED PROCESSES: {}".format(leaked) - assert not os.path.exists(UNIQUE_FILE_0), "atexit handler not called" - assert not os.path.exists(UNIQUE_FILE_1), "atexit handler not called" - assert not os.path.exists(UNIQUE_FILE_2), "close not called" - assert not os.path.exists(UNIQUE_FILE_3), "close not called" + assert not os.path.exists(tmp1), "atexit handler not called" + assert not os.path.exists(tmp2), "atexit handler not called" + assert not os.path.exists(tmp3), "close not called" + assert not os.path.exists(tmp4), "close not called" print("OK") diff --git a/rllib/tests/test_external_multi_agent_env.py b/rllib/tests/test_external_multi_agent_env.py index c7df295b8..5699bdf14 100644 --- a/rllib/tests/test_external_multi_agent_env.py +++ b/rllib/tests/test_external_multi_agent_env.py @@ -5,13 +5,14 @@ import unittest import ray from ray.rllib.agents.pg.pg_tf_policy import PGTFPolicy -from ray.rllib.optimizers import SyncSamplesOptimizer +from ray.rllib.env.external_multi_agent_env import ExternalMultiAgentEnv from ray.rllib.evaluation.rollout_worker import RolloutWorker from ray.rllib.evaluation.worker_set import WorkerSet -from ray.rllib.env.external_multi_agent_env import ExternalMultiAgentEnv +from ray.rllib.examples.env.multi_agent import BasicMultiAgent, \ + MultiAgentCartPole +from ray.rllib.optimizers import SyncSamplesOptimizer from ray.rllib.tests.test_rollout_worker import MockPolicy from ray.rllib.tests.test_external_env import make_simple_serving -from ray.rllib.tests.test_multi_agent_env import BasicMultiAgent, MultiCartpole from ray.rllib.evaluation.metrics import collect_metrics SimpleMultiServing = make_simple_serving(True, ExternalMultiAgentEnv) @@ -63,7 +64,7 @@ class TestExternalMultiAgentEnv(unittest.TestCase): batch = ev.sample() self.assertEqual(batch.count, 50) - def test_train_external_multi_cartpole_many_policies(self): + def test_train_external_multi_agent_cartpole_many_policies(self): n = 20 single_env = gym.make("CartPole-v0") act_space = single_env.action_space @@ -74,7 +75,7 @@ class TestExternalMultiAgentEnv(unittest.TestCase): {}) policy_ids = list(policies.keys()) ev = RolloutWorker( - env_creator=lambda _: MultiCartpole(n), + env_creator=lambda _: MultiAgentCartPole({"num_agents": n}), policy=policies, policy_mapping_fn=lambda agent_id: random.choice(policy_ids), rollout_fragment_length=100) diff --git a/rllib/tests/test_io.py b/rllib/tests/test_io.py index cf0ce481e..b801339be 100644 --- a/rllib/tests/test_io.py +++ b/rllib/tests/test_io.py @@ -10,13 +10,13 @@ import time import unittest import ray +from ray.tune.registry import register_env from ray.rllib.agents.pg import PGTrainer from ray.rllib.agents.pg.pg_tf_policy import PGTFPolicy +from ray.rllib.examples.env.multi_agent import MultiAgentCartPole from ray.rllib.offline import IOContext, JsonWriter, JsonReader from ray.rllib.offline.json_writer import _to_json from ray.rllib.policy.sample_batch import SampleBatch -from ray.rllib.tests.test_multi_agent_env import MultiCartpole -from ray.tune.registry import register_env SAMPLES = SampleBatch({ "actions": np.array([1, 2, 3, 4]), @@ -149,7 +149,8 @@ class AgentIOTest(unittest.TestCase): self.assertTrue(not np.isnan(result["episode_reward_mean"])) def testMultiAgent(self): - register_env("multi_cartpole", lambda _: MultiCartpole(10)) + register_env("multi_agent_cartpole", + lambda _: MultiAgentCartPole({"num_agents": 10})) single_env = gym.make("CartPole-v0") def gen_policy(): @@ -158,7 +159,7 @@ class AgentIOTest(unittest.TestCase): return (PGTFPolicy, obs_space, act_space, {}) pg = PGTrainer( - env="multi_cartpole", + env="multi_agent_cartpole", config={ "num_workers": 0, "output": self.test_dir, @@ -177,7 +178,7 @@ class AgentIOTest(unittest.TestCase): pg.stop() pg = PGTrainer( - env="multi_cartpole", + env="multi_agent_cartpole", config={ "num_workers": 0, "input": self.test_dir, diff --git a/rllib/tests/test_multi_agent_env.py b/rllib/tests/test_multi_agent_env.py index 434cd3919..a94ae3207 100644 --- a/rllib/tests/test_multi_agent_env.py +++ b/rllib/tests/test_multi_agent_env.py @@ -3,19 +3,20 @@ import random import unittest import ray +from ray.tune.registry import register_env from ray.rllib.agents.pg import PGTrainer from ray.rllib.agents.pg.pg_tf_policy import PGTFPolicy from ray.rllib.agents.dqn.dqn_tf_policy import DQNTFPolicy +from ray.rllib.examples.env.multi_agent import MultiAgentCartPole, \ + BasicMultiAgent, EarlyDoneMultiAgent, RoundRobinMultiAgent from ray.rllib.optimizers import (SyncSamplesOptimizer, SyncReplayOptimizer, AsyncGradientsOptimizer) -from ray.rllib.tests.test_rollout_worker import (MockEnv, MockEnv2, MockPolicy) +from ray.rllib.tests.test_rollout_worker import MockPolicy from ray.rllib.evaluation.rollout_worker import RolloutWorker from ray.rllib.policy.tests.test_policy import TestPolicy from ray.rllib.evaluation.metrics import collect_metrics from ray.rllib.evaluation.worker_set import WorkerSet from ray.rllib.env.base_env import _MultiAgentEnvToBaseEnv -from ray.rllib.env.multi_agent_env import MultiAgentEnv -from ray.tune.registry import register_env def one_hot(i, n): @@ -24,161 +25,6 @@ def one_hot(i, n): return out -class BasicMultiAgent(MultiAgentEnv): - """Env of N independent agents, each of which exits after 25 steps.""" - - def __init__(self, num): - self.agents = [MockEnv(25) for _ in range(num)] - self.dones = set() - self.observation_space = gym.spaces.Discrete(2) - self.action_space = gym.spaces.Discrete(2) - self.resetted = False - - def reset(self): - self.resetted = True - self.dones = set() - return {i: a.reset() for i, a in enumerate(self.agents)} - - def step(self, action_dict): - obs, rew, done, info = {}, {}, {}, {} - for i, action in action_dict.items(): - obs[i], rew[i], done[i], info[i] = self.agents[i].step(action) - if done[i]: - self.dones.add(i) - done["__all__"] = len(self.dones) == len(self.agents) - return obs, rew, done, info - - -class EarlyDoneMultiAgent(MultiAgentEnv): - """Env for testing when the env terminates (after agent 0 does).""" - - def __init__(self): - self.agents = [MockEnv(3), MockEnv(5)] - self.dones = set() - self.last_obs = {} - self.last_rew = {} - self.last_done = {} - self.last_info = {} - self.i = 0 - self.observation_space = gym.spaces.Discrete(10) - self.action_space = gym.spaces.Discrete(2) - - def reset(self): - self.dones = set() - self.last_obs = {} - self.last_rew = {} - self.last_done = {} - self.last_info = {} - self.i = 0 - for i, a in enumerate(self.agents): - self.last_obs[i] = a.reset() - self.last_rew[i] = None - self.last_done[i] = False - self.last_info[i] = {} - obs_dict = {self.i: self.last_obs[self.i]} - self.i = (self.i + 1) % len(self.agents) - return obs_dict - - def step(self, action_dict): - assert len(self.dones) != len(self.agents) - for i, action in action_dict.items(): - (self.last_obs[i], self.last_rew[i], self.last_done[i], - self.last_info[i]) = self.agents[i].step(action) - obs = {self.i: self.last_obs[self.i]} - rew = {self.i: self.last_rew[self.i]} - done = {self.i: self.last_done[self.i]} - info = {self.i: self.last_info[self.i]} - if done[self.i]: - rew[self.i] = 0 - self.dones.add(self.i) - self.i = (self.i + 1) % len(self.agents) - done["__all__"] = len(self.dones) == len(self.agents) - 1 - return obs, rew, done, info - - -class RoundRobinMultiAgent(MultiAgentEnv): - """Env of N independent agents, each of which exits after 5 steps. - - On each step() of the env, only one agent takes an action.""" - - def __init__(self, num, increment_obs=False): - if increment_obs: - # Observations are 0, 1, 2, 3... etc. as time advances - self.agents = [MockEnv2(5) for _ in range(num)] - else: - # Observations are all zeros - self.agents = [MockEnv(5) for _ in range(num)] - self.dones = set() - self.last_obs = {} - self.last_rew = {} - self.last_done = {} - self.last_info = {} - self.i = 0 - self.num = num - self.observation_space = gym.spaces.Discrete(10) - self.action_space = gym.spaces.Discrete(2) - - def reset(self): - self.dones = set() - self.last_obs = {} - self.last_rew = {} - self.last_done = {} - self.last_info = {} - self.i = 0 - for i, a in enumerate(self.agents): - self.last_obs[i] = a.reset() - self.last_rew[i] = None - self.last_done[i] = False - self.last_info[i] = {} - obs_dict = {self.i: self.last_obs[self.i]} - self.i = (self.i + 1) % self.num - return obs_dict - - def step(self, action_dict): - assert len(self.dones) != len(self.agents) - for i, action in action_dict.items(): - (self.last_obs[i], self.last_rew[i], self.last_done[i], - self.last_info[i]) = self.agents[i].step(action) - obs = {self.i: self.last_obs[self.i]} - rew = {self.i: self.last_rew[self.i]} - done = {self.i: self.last_done[self.i]} - info = {self.i: self.last_info[self.i]} - if done[self.i]: - rew[self.i] = 0 - self.dones.add(self.i) - self.i = (self.i + 1) % self.num - done["__all__"] = len(self.dones) == len(self.agents) - return obs, rew, done, info - - -def make_multiagent(env_name): - class MultiEnv(MultiAgentEnv): - def __init__(self, num): - self.agents = [gym.make(env_name) for _ in range(num)] - self.dones = set() - self.observation_space = self.agents[0].observation_space - self.action_space = self.agents[0].action_space - - def reset(self): - self.dones = set() - return {i: a.reset() for i, a in enumerate(self.agents)} - - def step(self, action_dict): - obs, rew, done, info = {}, {}, {}, {} - for i, action in action_dict.items(): - obs[i], rew[i], done[i], info[i] = self.agents[i].step(action) - if done[i]: - self.dones.add(i) - done["__all__"] = len(self.dones) == len(self.agents) - return obs, rew, done, info - - return MultiEnv - - -MultiCartpole = make_multiagent("CartPole-v0") -MultiMountainCar = make_multiagent("MountainCarContinuous-v0") - - class TestMultiAgentEnv(unittest.TestCase): def setUp(self) -> None: ray.init(num_cpus=4) @@ -512,7 +358,7 @@ class TestMultiAgentEnv(unittest.TestCase): obs_space = single_env.observation_space act_space = single_env.action_space ev = RolloutWorker( - env_creator=lambda _: MultiCartpole(2), + env_creator=lambda _: MultiAgentCartPole({"num_agents": 2}), policy={ "p0": (ModelBasedPolicy, obs_space, act_space, {}), "p1": (ModelBasedPolicy, obs_space, act_space, {}), @@ -524,10 +370,11 @@ class TestMultiAgentEnv(unittest.TestCase): self.assertEqual(batch.policy_batches["p0"].count, 10) self.assertEqual(batch.policy_batches["p1"].count, 25) - def test_train_multi_cartpole_single_policy(self): + def test_train_multi_agent_cartpole_single_policy(self): n = 10 - register_env("multi_cartpole", lambda _: MultiCartpole(n)) - pg = PGTrainer(env="multi_cartpole", config={"num_workers": 0}) + register_env("multi_agent_cartpole", + lambda _: MultiAgentCartPole({"num_agents": n})) + pg = PGTrainer(env="multi_agent_cartpole", config={"num_workers": 0}) for i in range(100): result = pg.train() print("Iteration {}, reward {}, timesteps {}".format( @@ -536,9 +383,10 @@ class TestMultiAgentEnv(unittest.TestCase): return raise Exception("failed to improve reward") - def test_train_multi_cartpole_multi_policy(self): + def test_train_multi_agent_cartpole_multi_policy(self): n = 10 - register_env("multi_cartpole", lambda _: MultiCartpole(n)) + register_env("multi_agent_cartpole", + lambda _: MultiAgentCartPole({"num_agents": n})) single_env = gym.make("CartPole-v0") def gen_policy(): @@ -551,7 +399,7 @@ class TestMultiAgentEnv(unittest.TestCase): return (None, obs_space, act_space, config) pg = PGTrainer( - env="multi_cartpole", + env="multi_agent_cartpole", config={ "num_workers": 0, "multiagent": { @@ -596,7 +444,7 @@ class TestMultiAgentEnv(unittest.TestCase): "p2": (DQNTFPolicy, obs_space, act_space, dqn_config), } worker = RolloutWorker( - env_creator=lambda _: MultiCartpole(n), + env_creator=lambda _: MultiAgentCartPole({"num_agents": n}), policy=policies, policy_mapping_fn=lambda agent_id: ["p1", "p2"][agent_id % 2], rollout_fragment_length=50) @@ -607,7 +455,8 @@ class TestMultiAgentEnv(unittest.TestCase): remote_workers = [ RolloutWorker.as_remote().remote( - env_creator=lambda _: MultiCartpole(n), + env_creator=lambda _: MultiAgentCartPole( + {"num_agents": n}), policy=policies, policy_mapping_fn=policy_mapper, rollout_fragment_length=50) @@ -645,7 +494,7 @@ class TestMultiAgentEnv(unittest.TestCase): def test_multi_agent_replay_optimizer(self): self._test_with_optimizer(SyncReplayOptimizer) - def test_train_multi_cartpole_many_policies(self): + def test_train_multi_agent_cartpole_many_policies(self): n = 20 env = gym.make("CartPole-v0") act_space = env.action_space @@ -656,7 +505,7 @@ class TestMultiAgentEnv(unittest.TestCase): {}) policy_ids = list(policies.keys()) worker = RolloutWorker( - env_creator=lambda _: MultiCartpole(n), + env_creator=lambda _: MultiAgentCartPole({"num_agents": n}), policy=policies, policy_mapping_fn=lambda agent_id: random.choice(policy_ids), rollout_fragment_length=100) diff --git a/rllib/tests/test_multi_agent_pendulum.py b/rllib/tests/test_multi_agent_pendulum.py index 5ba73ee06..3f0ba7f8d 100644 --- a/rllib/tests/test_multi_agent_pendulum.py +++ b/rllib/tests/test_multi_agent_pendulum.py @@ -2,7 +2,7 @@ import unittest import ray -from ray.rllib.tests.test_multi_agent_env import make_multiagent +from ray.rllib.examples.env.multi_agent import MultiAgentPendulum from ray.tune import run_experiments from ray.tune.registry import register_env @@ -15,12 +15,12 @@ class TestMultiAgentPendulum(unittest.TestCase): ray.shutdown() def test_multi_agent_pendulum(self): - MultiPendulum = make_multiagent("Pendulum-v0") - register_env("multi_pend", lambda _: MultiPendulum(1)) + register_env("multi_agent_pendulum", + lambda _: MultiAgentPendulum({"num_agents": 1})) trials = run_experiments({ "test": { "run": "PPO", - "env": "multi_pend", + "env": "multi_agent_pendulum", "stop": { "timesteps_total": 500000, "episode_reward_mean": -200, diff --git a/rllib/tests/test_supported_spaces.py b/rllib/tests/test_supported_spaces.py index 57fc1c1b0..55c0bee58 100644 --- a/rllib/tests/test_supported_spaces.py +++ b/rllib/tests/test_supported_spaces.py @@ -1,6 +1,4 @@ -import gym from gym.spaces import Box, Dict, Discrete, Tuple, MultiDiscrete -from gym.envs.registration import EnvSpec import numpy as np import unittest import traceback @@ -8,12 +6,13 @@ import traceback import ray from ray.rllib.utils.framework import try_import_tf from ray.rllib.agents.registry import get_agent_class +from ray.rllib.examples.env.multi_agent import MultiAgentCartPole, \ + MultiAgentMountainCar +from ray.rllib.examples.env.random_env import RandomEnv from ray.rllib.models.tf.fcnet_v2 import FullyConnectedNetwork as FCNetV2 from ray.rllib.models.tf.visionnet_v2 import VisionNetwork as VisionNetV2 from ray.rllib.models.torch.visionnet import VisionNetwork as TorchVisionNetV2 from ray.rllib.models.torch.fcnet import FullyConnectedNetwork as TorchFCNetV2 -from ray.rllib.tests.test_multi_agent_env import MultiCartpole, \ - MultiMountainCar from ray.rllib.utils.error import UnsupportedSpaceException from ray.tune.registry import register_env @@ -55,30 +54,6 @@ OBSERVATION_SPACES_TO_TEST = { } -def make_stub_env(action_space, obs_space, check_action_bounds): - class StubEnv(gym.Env): - def __init__(self): - self.action_space = action_space - self.observation_space = obs_space - self.spec = EnvSpec("StubEnv-v0") - - def reset(self): - sample = self.observation_space.sample() - return sample - - def step(self, action): - if check_action_bounds and not self.action_space.contains(action): - raise ValueError("Illegal action for {}: {}".format( - self.action_space, action)) - if (isinstance(self.action_space, Tuple) - and len(action) != len(self.action_space.spaces)): - raise ValueError("Illegal action for {}: {}".format( - self.action_space, action)) - return self.observation_space.sample(), 1, True, {} - - return StubEnv - - def check_support(alg, config, stats, check_bounds=False, name=None): covered_a = set() covered_o = set() @@ -89,15 +64,21 @@ def check_support(alg, config, stats, check_bounds=False, name=None): for o_name, obs_space in OBSERVATION_SPACES_TO_TEST.items(): print("=== Testing {} (torch={}) A={} S={} ===".format( alg, torch, action_space, obs_space)) - stub_env = make_stub_env(action_space, obs_space, check_bounds) - register_env("stub_env", lambda c: stub_env()) + config.update( + dict( + env_config=dict( + action_space=action_space, + observation_space=obs_space, + reward_space=Box(1.0, 1.0, shape=(), dtype=np.float32), + p_done=1.0, + check_action_bounds=check_bounds))) stat = "ok" a = None try: if a_name in covered_a and o_name in covered_o: stat = "skip" # speed up tests by avoiding full grid else: - a = get_agent_class(alg)(config=config, env="stub_env") + a = get_agent_class(alg)(config=config, env=RandomEnv) if alg not in ["DDPG", "ES", "ARS", "SAC"]: if o_name in ["atari", "image"]: if torch: @@ -140,13 +121,15 @@ def check_support(alg, config, stats, check_bounds=False, name=None): def check_support_multiagent(alg, config): - register_env("multi_mountaincar", lambda _: MultiMountainCar(2)) - register_env("multi_cartpole", lambda _: MultiCartpole(2)) + register_env("multi_agent_mountaincar", + lambda _: MultiAgentMountainCar({"num_agents": 2})) + register_env("multi_agent_cartpole", + lambda _: MultiAgentCartPole({"num_agents": 2})) config["log_level"] = "ERROR" if "DDPG" in alg: - a = get_agent_class(alg)(config=config, env="multi_mountaincar") + a = get_agent_class(alg)(config=config, env="multi_agent_mountaincar") else: - a = get_agent_class(alg)(config=config, env="multi_cartpole") + a = get_agent_class(alg)(config=config, env="multi_agent_cartpole") try: a.train() finally: