wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-13 17:45:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

[RLlib] rllib/examples folder restructuring (#8250)

Browse Source 
Cleans up of the rllib/examples folder by moving all example Envs into rllibexamples/env (so they can be used by other scripts and tests as well).
This commit is contained in:
Sven Mika
2020-05-01 22:59:34 +02:00
committed by GitHub
parent 13f718846d
commit 42991d723f
45 changed files with 1248 additions and 1141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/source/rllib-examples.rst
+1 -1
View File
@@ -48,7 +48,7 @@ Custom Envs and Models
Example of how to ensure subprocesses spawned by envs are killed when RLlib exits.
- `Batch normalization <https://github.com/ray-project/ray/blob/master/rllib/examples/batch_norm_model.py>`__:
Example of adding batch norm layers to a custom model.
- `Parametric actions <https://github.com/ray-project/ray/blob/master/rllib/examples/parametric_action_cartpole.py>`__:
- `Parametric actions <https://github.com/ray-project/ray/blob/master/rllib/examples/parametric_actions_cartpole.py>`__:
Example of how to handle variable-length or parametric action spaces.
- `Eager execution <https://github.com/ray-project/ray/blob/master/rllib/examples/eager_execution.py>`__:
Example of how to leverage TensorFlow eager to simplify debugging and design of custom models and policies.
doc/source/rllib-models.rst
+1 -1
View File
@@ -274,7 +274,7 @@ Custom models can be used to work with environments where (1) the set of valid a
return action_logits + inf_mask, state
Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_action_cartpole.py <https://github.com/ray-project/ray/blob/master/rllib/examples/parametric_action_cartpole.py>`__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``vf_share_layers=True``), and several other algorithms. Not all algorithms support parametric actions; see the `feature compatibility matrix <rllib-env.html#feature-compatibility-matrix>`__.
Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_actions_cartpole.py <https://github.com/ray-project/ray/blob/master/rllib/examples/parametric_actions_cartpole.py>`__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``vf_share_layers=True``), and several other algorithms. Not all algorithms support parametric actions; see the `feature compatibility matrix <rllib-env.html#feature-compatibility-matrix>`__.
Autoregressive Action Distributions
rllib/BUILD
+26 -14
View File
@@ -1339,7 +1339,7 @@ py_test(
tags = ["examples", "examples_C"],
size = "large",
srcs = ["examples/custom_keras_rnn_model.py"],
args = ["--run=PPO", "--stop=50", "--env=RepeatInitialEnv", "--num-cpus=4"]
args = ["--run=PPO", "--stop=50", "--env=RepeatInitialObsEnv", "--num-cpus=4"]
)
py_test(
@@ -1401,6 +1401,22 @@ py_test(
args = ["--iters=2"]
)
py_test(
name = "examples/hierarchical_training_tf",
tags = ["examples", "examples_H"],
size = "small",
srcs = ["examples/hierarchical_training.py"],
args = ["--stop-reward=0.0"]
)
py_test(
name = "examples/hierarchical_training_torch",
tags = ["examples", "examples_H"],
size = "small",
srcs = ["examples/hierarchical_training.py"],
args = ["--torch", "--stop-reward=0.0"]
)
py_test(
name = "examples/multi_agent_cartpole",
tags = ["examples", "examples_M"],
@@ -1434,36 +1450,32 @@ py_test(
)
py_test(
name = "examples/parametric_action_cartpole_pg", main="examples/parametric_action_cartpole.py",
name = "examples/parametric_actions_cartpole_pg",
main = "examples/parametric_actions_cartpole.py",
tags = ["examples", "examples_P"],
size = "medium",
srcs = ["examples/parametric_action_cartpole.py"],
srcs = ["examples/parametric_actions_cartpole.py"],
args = ["--run=PG", "--stop=50"]
)
py_test(
name = "examples/parametric_action_cartpole_ppo", main="examples/parametric_action_cartpole.py",
name = "examples/parametric_actions_cartpole_ppo",
main = "examples/parametric_actions_cartpole.py",
tags = ["examples", "examples_P"],
size = "medium",
srcs = ["examples/parametric_action_cartpole.py"],
srcs = ["examples/parametric_actions_cartpole.py"],
args = ["--run=PPO", "--stop=50"]
)
py_test(
name = "examples/parametric_action_cartpole_dqn", main="examples/parametric_action_cartpole.py",
name = "examples/parametric_actions_cartpole_dqn",
main = "examples/parametric_actions_cartpole.py",
tags = ["examples", "examples_P"],
size = "medium",
srcs = ["examples/parametric_action_cartpole.py"],
srcs = ["examples/parametric_actions_cartpole.py"],
args = ["--run=DQN", "--stop=50"]
)
py_test(
name = "examples/random_env", main = "examples/random_env.py",
tags = ["examples", "examples_R"],
size = "large",
srcs = ["examples/random_env.py"]
)
py_test(
name = "examples/rollout_worker_custom_workflow",
tags = ["examples", "examples_R"],
rllib/agents/qmix/qmix_policy.py
+1 -3
View File
@@ -5,6 +5,7 @@ import numpy as np
import ray
from ray.rllib.agents.qmix.mixers import VDNMixer, QMixer
from ray.rllib.agents.qmix.model import RNNModel, _get_size
from ray.rllib.env.multi_agent_env import ENV_STATE
from ray.rllib.evaluation.metrics import LEARNER_STATS_KEY
from ray.rllib.policy.policy import Policy
from ray.rllib.policy.rnn_sequencing import chop_into_sequences
@@ -20,9 +21,6 @@ torch, nn = try_import_torch(error=True)
logger = logging.getLogger(__name__)
# if the obs space is Dict type, look for the global state under this key
ENV_STATE = "state"
class QMixLoss(nn.Module):
def __init__(self,
rllib/contrib/alpha_zero/README.md
+1 -2
View File
@@ -12,7 +12,7 @@ The code is Pytorch based. It assumes that the environment is a gym environment,
The model used in AlphaZero trainer should extend `ActorCriticModel` and implement the method `compute_priors_and_value`.
## Example on Cartpole
## Example on CartPole
Note that both mean and max rewards are obtained with the MCTS in exploration mode: dirichlet noise is added to priors and actions are sampled from the tree policy vectors. We will add later the display of the MCTS in exploitation mode: no dirichlet noise and actions are chosen as tree policy vectors argmax.
![cartpole_plot](doc/cartpole_plot.png)
@@ -21,4 +21,3 @@ Note that both mean and max rewards are obtained with the MCTS in exploration mo
- AlphaZero: https://arxiv.org/abs/1712.01815
- Ranked rewards: https://arxiv.org/abs/1807.01672
rllib/env/multi_agent_env.py
Vendored
+3
View File
@@ -1,5 +1,8 @@
from ray.rllib.utils.annotations import PublicAPI
# If the obs space is Dict type, look for the global state under this key.
ENV_STATE = "state"
@PublicAPI
class MultiAgentEnv:
rllib/examples/autoregressive_action_dist.py
+1 -30
View File
@@ -10,13 +10,12 @@ pattern, and a custom action distribution class that leverages that model.
This examples shows both.
"""
import gym
from gym.spaces import Discrete, Tuple
import argparse
import random
import ray
from ray import tune
from ray.rllib.examples.env.correlated_actions_env import CorrelatedActionsEnv
from ray.rllib.models import ModelCatalog
from ray.rllib.models.tf.tf_action_dist import Categorical, ActionDistribution
from ray.rllib.models.tf.misc import normc_initializer
@@ -31,34 +30,6 @@ parser.add_argument("--stop", type=int, default=200)
parser.add_argument("--num-cpus", type=int, default=0)
class CorrelatedActionsEnv(gym.Env):
"""Simple env in which the policy has to emit a tuple of equal actions.
The best score would be ~200 reward."""
def __init__(self, _):
self.observation_space = Discrete(2)
self.action_space = Tuple([Discrete(2), Discrete(2)])
def reset(self):
self.t = 0
self.last = random.choice([0, 1])
return self.last
def step(self, action):
self.t += 1
a1, a2 = action
reward = 0
if a1 == self.last:
reward += 5
# encourage correlation between a1 and a2
if a1 == a2:
reward += 5
done = self.t > 20
self.last = random.choice([0, 1])
return self.last, reward, done, {}
class BinaryAutoregressiveOutput(ActionDistribution):
"""Action distribution P(a1, a2) = P(a1) * P(a2 | a1)"""
rllib/examples/batch_norm_model.py
-4
View File
@@ -155,10 +155,6 @@ if __name__ == "__main__":
"num_workers": 0,
}
from ray.rllib.agents.ppo import PPOTrainer
trainer = PPOTrainer(config=config)
trainer.train()
tune.run(
args.run,
stop={"training_iteration": args.num_iters},
rllib/examples/cartpole_lstm.py
+5 -154
View File
@@ -1,170 +1,20 @@
"""Partially observed variant of the CartPole gym environment.
https://github.com/openai/gym/blob/master/gym/envs/classic_control/cartpole.py
We delete the velocity component of the state, so that it can only be solved
by a LSTM policy."""
import argparse
import math
import gym
from gym import spaces
from gym.utils import seeding
import numpy as np
from ray.rllib.examples.env.stateless_cartpole import StatelessCartPole
parser = argparse.ArgumentParser()
parser.add_argument("--stop", type=int, default=200)
parser.add_argument("--torch", action="store_true")
parser.add_argument("--use-prev-action-reward", action="store_true")
parser.add_argument("--run", type=str, default="PPO")
parser.add_argument("--num-cpus", type=int, default=0)
class CartPoleStatelessEnv(gym.Env):
metadata = {
"render.modes": ["human", "rgb_array"],
"video.frames_per_second": 60
}
def __init__(self, config=None):
self.gravity = 9.8
self.masscart = 1.0
self.masspole = 0.1
self.total_mass = (self.masspole + self.masscart)
self.length = 0.5 # actually half the pole's length
self.polemass_length = (self.masspole * self.length)
self.force_mag = 10.0
self.tau = 0.02 # seconds between state updates
# Angle at which to fail the episode
self.theta_threshold_radians = 12 * 2 * math.pi / 360
self.x_threshold = 2.4
high = np.array([
self.x_threshold * 2,
self.theta_threshold_radians * 2,
])
self.action_space = spaces.Discrete(2)
self.observation_space = spaces.Box(-high, high)
self.seed()
self.viewer = None
self.state = None
self.steps_beyond_done = None
def seed(self, seed=None):
self.np_random, seed = seeding.np_random(seed)
return [seed]
def step(self, action):
assert self.action_space.contains(
action), "%r (%s) invalid" % (action, type(action))
state = self.state
x, x_dot, theta, theta_dot = state
force = self.force_mag if action == 1 else -self.force_mag
costheta = math.cos(theta)
sintheta = math.sin(theta)
temp = (force + self.polemass_length * theta_dot * theta_dot * sintheta
) / self.total_mass
thetaacc = (self.gravity * sintheta - costheta * temp) / (
self.length *
(4.0 / 3.0 - self.masspole * costheta * costheta / self.total_mass)
)
xacc = (temp -
self.polemass_length * thetaacc * costheta / self.total_mass)
x = x + self.tau * x_dot
x_dot = x_dot + self.tau * xacc
theta = theta + self.tau * theta_dot
theta_dot = theta_dot + self.tau * thetaacc
self.state = (x, x_dot, theta, theta_dot)
done = (x < -self.x_threshold or x > self.x_threshold
or theta < -self.theta_threshold_radians
or theta > self.theta_threshold_radians)
done = bool(done)
if not done:
reward = 1.0
elif self.steps_beyond_done is None:
# Pole just fell!
self.steps_beyond_done = 0
reward = 1.0
else:
self.steps_beyond_done += 1
reward = 0.0
rv = np.r_[self.state[0], self.state[2]]
return rv, reward, done, {}
def reset(self):
self.state = self.np_random.uniform(low=-0.05, high=0.05, size=(4, ))
self.steps_beyond_done = None
rv = np.r_[self.state[0], self.state[2]]
return rv
def render(self, mode="human"):
screen_width = 600
screen_height = 400
world_width = self.x_threshold * 2
scale = screen_width / world_width
carty = 100 # TOP OF CART
polewidth = 10.0
polelen = scale * 1.0
cartwidth = 50.0
cartheight = 30.0
if self.viewer is None:
from gym.envs.classic_control import rendering
self.viewer = rendering.Viewer(screen_width, screen_height)
l, r, t, b = (-cartwidth / 2, cartwidth / 2, cartheight / 2,
-cartheight / 2)
axleoffset = cartheight / 4.0
cart = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)])
self.carttrans = rendering.Transform()
cart.add_attr(self.carttrans)
self.viewer.add_geom(cart)
l, r, t, b = (-polewidth / 2, polewidth / 2,
polelen - polewidth / 2, -polewidth / 2)
pole = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)])
pole.set_color(.8, .6, .4)
self.poletrans = rendering.Transform(translation=(0, axleoffset))
pole.add_attr(self.poletrans)
pole.add_attr(self.carttrans)
self.viewer.add_geom(pole)
self.axle = rendering.make_circle(polewidth / 2)
self.axle.add_attr(self.poletrans)
self.axle.add_attr(self.carttrans)
self.axle.set_color(.5, .5, .8)
self.viewer.add_geom(self.axle)
self.track = rendering.Line((0, carty), (screen_width, carty))
self.track.set_color(0, 0, 0)
self.viewer.add_geom(self.track)
if self.state is None:
return None
x = self.state
cartx = x[0] * scale + screen_width / 2.0 # MIDDLE OF CART
self.carttrans.set_translation(cartx, carty)
self.poletrans.set_rotation(-x[2])
return self.viewer.render(return_rgb_array=mode == "rgb_array")
def close(self):
if self.viewer:
self.viewer.close()
if __name__ == "__main__":
import ray
from ray import tune
args = parser.parse_args()
tune.register_env("cartpole_stateless", lambda _: CartPoleStatelessEnv())
ray.init(num_cpus=args.num_cpus or None)
configs = {
@@ -185,10 +35,11 @@ if __name__ == "__main__":
stop={"episode_reward_mean": args.stop},
config=dict(
configs[args.run], **{
"env": "cartpole_stateless",
"env": StatelessCartPole,
"model": {
"use_lstm": True,
"lstm_use_prev_action_reward": args.use_prev_action_reward,
},
"use_pytorch": args.torch,
}),
)
rllib/examples/custom_eval.py
+5 -33
View File
@@ -67,18 +67,16 @@ Result for PG_SimpleCorridor_0de4e686:
"""
import argparse
import numpy as np
import gym
from gym.spaces import Discrete, Box
import ray
from ray import tune
from ray.rllib.evaluation.metrics import collect_episodes, summarize_episodes
from ray.rllib.examples.env.simple_corridor import SimpleCorridor
parser = argparse.ArgumentParser()
parser.add_argument("--custom-eval", action="store_true")
parser.add_argument("--num-cpus", type=int, default=0)
args = parser.parse_args()
parser.add_argument("--torch", action="store_true")
def custom_eval_function(trainer, eval_workers):
@@ -123,36 +121,9 @@ def custom_eval_function(trainer, eval_workers):
return metrics
class SimpleCorridor(gym.Env):
"""Custom env we use for this example."""
def __init__(self, env_config):
self.end_pos = env_config["corridor_length"]
self.cur_pos = 0
self.action_space = Discrete(2)
self.observation_space = Box(0.0, 9999, shape=(1, ), dtype=np.float32)
print("Created env for worker index", env_config.worker_index,
"with corridor length", self.end_pos)
def set_corridor_length(self, length):
print("Update corridor length to", length)
self.end_pos = length
def reset(self):
self.cur_pos = 0
return [self.cur_pos]
def step(self, action):
assert action in [0, 1], action
if action == 0 and self.cur_pos > 0:
self.cur_pos -= 1
elif action == 1:
self.cur_pos += 1
done = self.cur_pos >= self.end_pos
return [self.cur_pos], 1 if done else 0, done, {}
if __name__ == "__main__":
args = parser.parse_args()
if args.custom_eval:
eval_fn = custom_eval_function
else:
@@ -196,4 +167,5 @@ if __name__ == "__main__":
"corridor_length": 5,
},
},
"use_pytorch": args.torch,
})
rllib/examples/custom_fast_model.py
+1 -21
View File
@@ -4,11 +4,8 @@ Both the model and env are trivial (and super-fast), so they are useful
for running perf microbenchmarks.
"""
from gym.spaces import Discrete, Box
import gym
import numpy as np
import ray
from ray.rllib.examples.env.fast_image_env import FastImageEnv
from ray.rllib.models import Model, ModelCatalog
from ray.tune import run_experiments, sample_from
from ray.rllib.utils import try_import_tf
@@ -27,23 +24,6 @@ class FastModel(Model):
return output, output
class FastImageEnv(gym.Env):
def __init__(self, config):
self.zeros = np.zeros((84, 84, 4))
self.action_space = Discrete(2)
self.observation_space = Box(
0.0, 1.0, shape=(84, 84, 4), dtype=np.float32)
self.i = 0
def reset(self):
self.i = 0
return self.zeros
def step(self, action):
self.i += 1
return self.zeros, 1, self.i > 1000, {}
if __name__ == "__main__":
ray.init()
ModelCatalog.register_custom_model("fast_model", FastModel)
rllib/examples/custom_keras_rnn_model.py
+3 -62
View File
@@ -1,14 +1,13 @@
"""Example of using a custom RNN keras model."""
import argparse
import gym
from gym.spaces import Discrete
import numpy as np
import random
import ray
from ray import tune
from ray.tune.registry import register_env
from ray.rllib.examples.env.repeat_after_me_env import RepeatAfterMeEnv
from ray.rllib.examples.env.repeat_initial_obs_env import RepeatInitialObsEnv
from ray.rllib.models import ModelCatalog
from ray.rllib.models.modelv2 import ModelV2
from ray.rllib.models.tf.recurrent_tf_modelv2 import RecurrentTFModelV2
@@ -88,70 +87,12 @@ class MyKerasRNN(RecurrentTFModelV2):
return tf.reshape(self._value_out, [-1])
class RepeatInitialEnv(gym.Env):
"""Simple env where policy has to always repeat the initial observation.
Runs for 100 steps.
r=1 if action correct, -1 otherwise (max. R=100).
"""
def __init__(self, episode_len=100):
self.observation_space = Discrete(2)
self.action_space = Discrete(2)
self.token = None
self.episode_len = episode_len
self.num_steps = 0
def reset(self):
self.token = random.choice([0, 1])
self.num_steps = 0
return self.token
def step(self, action):
if action == self.token:
reward = 1
else:
reward = -1
self.num_steps += 1
done = self.num_steps >= self.episode_len
return 0, reward, done, {}
class RepeatAfterMeEnv(gym.Env):
"""Simple env in which the policy learns to repeat a previous observation
token after a given delay."""
def __init__(self, config):
self.observation_space = Discrete(2)
self.action_space = Discrete(2)
self.delay = config["repeat_delay"]
assert self.delay >= 1, "delay must be at least 1"
self.history = []
def reset(self):
self.history = [0] * self.delay
return self._next_obs()
def step(self, action):
if action == self.history[-(1 + self.delay)]:
reward = 1
else:
reward = -1
done = len(self.history) > 100
return self._next_obs(), reward, done, {}
def _next_obs(self):
token = random.choice([0, 1])
self.history.append(token)
return token
if __name__ == "__main__":
args = parser.parse_args()
ray.init(num_cpus=args.num_cpus or None)
ModelCatalog.register_custom_model("rnn", MyKerasRNN)
register_env("RepeatAfterMeEnv", lambda c: RepeatAfterMeEnv(c))
register_env("RepeatInitialEnv", lambda _: RepeatInitialEnv())
register_env("RepeatInitialObsEnv", lambda _: RepeatInitialObsEnv())
config = {
"env": args.env,
rllib/examples/custom_torch_rnn_model.py
+5 -5
View File
@@ -1,9 +1,9 @@
import argparse
import ray
from ray.rllib.examples.cartpole_lstm import CartPoleStatelessEnv
from ray.rllib.examples.custom_keras_rnn_model import RepeatInitialEnv, \
RepeatAfterMeEnv
from ray.rllib.examples.env.repeat_initial_obs_env import RepeatInitialObsEnv
from ray.rllib.examples.env.repeat_after_me_env import RepeatAfterMeEnv
from ray.rllib.examples.env.stateless_cartpole import StatelessCartPole
from ray.rllib.models.preprocessors import get_preprocessor
from ray.rllib.models.torch.recurrent_torch_model import RecurrentTorchModel
from ray.rllib.models.modelv2 import ModelV2
@@ -92,10 +92,10 @@ if __name__ == "__main__":
ray.init(num_cpus=args.num_cpus or None)
ModelCatalog.register_custom_model("rnn", RNNModel)
tune.register_env(
"repeat_initial", lambda _: RepeatInitialEnv(episode_len=100))
"repeat_initial", lambda _: RepeatInitialObsEnv(episode_len=100))
tune.register_env(
"repeat_after_me", lambda _: RepeatAfterMeEnv({"repeat_delay": 1}))
tune.register_env("cartpole_stateless", lambda _: CartPoleStatelessEnv())
tune.register_env("stateless_cartpole", lambda _: StatelessCartPole())
config = {
"env": args.env,
rllib/examples/custom_train_fn.py
+6
View File
@@ -5,11 +5,15 @@ This example shows:
You can visualize experiment results in ~/ray_results using TensorBoard.
"""
import argparse
import ray
from ray import tune
from ray.rllib.agents.ppo import PPOTrainer
parser = argparse.ArgumentParser()
parser.add_argument("--torch", action="store_true")
def my_train_fn(config, reporter):
# Train for 100 iterations with high LR
@@ -36,9 +40,11 @@ def my_train_fn(config, reporter):
if __name__ == "__main__":
ray.init()
args = parser.parse_args()
config = {
"lr": 0.01,
"num_workers": 0,
"use_pytorch": args.torch,
}
resources = PPOTrainer.default_resource_request(config).to_json()
tune.run(my_train_fn, resources_per_trial=resources, config=config)
rllib/examples/env/__init__.py
Vendored
View File
rllib/examples/env/correlated_actions_env.py
Vendored
+31
View File
@@ -0,0 +1,31 @@
import gym
from gym.spaces import Discrete, Tuple
import random
class CorrelatedActionsEnv(gym.Env):
"""Simple env in which the policy has to emit a tuple of equal actions.
The best score would be ~200 reward."""
def __init__(self, _):
self.observation_space = Discrete(2)
self.action_space = Tuple([Discrete(2), Discrete(2)])
def reset(self):
self.t = 0
self.last = random.choice([0, 1])
return self.last
def step(self, action):
self.t += 1
a1, a2 = action
reward = 0
if a1 == self.last:
reward += 5
# encourage correlation between a1 and a2
if a1 == a2:
reward += 5
done = self.t > 20
self.last = random.choice([0, 1])
return self.last, reward, done, {}
rllib/examples/env/env_with_subprocess.py
Vendored
+44
View File
@@ -0,0 +1,44 @@
import atexit
import gym
from gym.spaces import Discrete
import os
import subprocess
import time
class EnvWithSubprocess(gym.Env):
"""Our env that spawns a subprocess."""
# Dummy command to run as a subprocess with a unique name
UNIQUE_CMD = "sleep {}".format(str(time.time()))
def __init__(self, config):
self.UNIQUE_FILE_0 = config["tmp_file1"]
self.UNIQUE_FILE_1 = config["tmp_file2"]
self.UNIQUE_FILE_2 = config["tmp_file3"]
self.UNIQUE_FILE_3 = config["tmp_file4"]
self.action_space = Discrete(2)
self.observation_space = Discrete(2)
# Subprocess that should be cleaned up
self.subproc = subprocess.Popen(
self.UNIQUE_CMD.split(" "), shell=False)
self.config = config
# Exit handler should be called
atexit.register(lambda: self.subproc.kill())
if config.worker_index == 0:
atexit.register(lambda: os.unlink(self.UNIQUE_FILE_0))
else:
atexit.register(lambda: os.unlink(self.UNIQUE_FILE_1))
def close(self):
if self.config.worker_index == 0:
os.unlink(self.UNIQUE_FILE_2)
else:
os.unlink(self.UNIQUE_FILE_3)
def reset(self):
return 0
def step(self, action):
return 0, 0, True, {}
rllib/examples/env/fast_image_env.py
Vendored
+20
View File
@@ -0,0 +1,20 @@
import gym
from gym.spaces import Box, Discrete
import numpy as np
class FastImageEnv(gym.Env):
def __init__(self, config):
self.zeros = np.zeros((84, 84, 4))
self.action_space = Discrete(2)
self.observation_space = Box(
0.0, 1.0, shape=(84, 84, 4), dtype=np.float32)
self.i = 0
def reset(self):
self.i = 0
return self.zeros
def step(self, action):
self.i += 1
return self.zeros, 1, self.i > 1000, {}
rllib/examples/env/multi_agent.py
Vendored
+162
View File
@@ -0,0 +1,162 @@
import gym
from ray.rllib.env.multi_agent_env import MultiAgentEnv
from ray.rllib.tests.test_rollout_worker import MockEnv, MockEnv2
def make_multiagent(env_name):
class MultiEnv(MultiAgentEnv):
def __init__(self, config):
self.agents = [
gym.make(env_name) for _ in range(config["num_agents"])
]
self.dones = set()
self.observation_space = self.agents[0].observation_space
self.action_space = self.agents[0].action_space
def reset(self):
self.dones = set()
return {i: a.reset() for i, a in enumerate(self.agents)}
def step(self, action_dict):
obs, rew, done, info = {}, {}, {}, {}
for i, action in action_dict.items():
obs[i], rew[i], done[i], info[i] = self.agents[i].step(action)
if done[i]:
self.dones.add(i)
done["__all__"] = len(self.dones) == len(self.agents)
return obs, rew, done, info
return MultiEnv
class BasicMultiAgent(MultiAgentEnv):
"""Env of N independent agents, each of which exits after 25 steps."""
def __init__(self, num):
self.agents = [MockEnv(25) for _ in range(num)]
self.dones = set()
self.observation_space = gym.spaces.Discrete(2)
self.action_space = gym.spaces.Discrete(2)
self.resetted = False
def reset(self):
self.resetted = True
self.dones = set()
return {i: a.reset() for i, a in enumerate(self.agents)}
def step(self, action_dict):
obs, rew, done, info = {}, {}, {}, {}
for i, action in action_dict.items():
obs[i], rew[i], done[i], info[i] = self.agents[i].step(action)
if done[i]:
self.dones.add(i)
done["__all__"] = len(self.dones) == len(self.agents)
return obs, rew, done, info
class EarlyDoneMultiAgent(MultiAgentEnv):
"""Env for testing when the env terminates (after agent 0 does)."""
def __init__(self):
self.agents = [MockEnv(3), MockEnv(5)]
self.dones = set()
self.last_obs = {}
self.last_rew = {}
self.last_done = {}
self.last_info = {}
self.i = 0
self.observation_space = gym.spaces.Discrete(10)
self.action_space = gym.spaces.Discrete(2)
def reset(self):
self.dones = set()
self.last_obs = {}
self.last_rew = {}
self.last_done = {}
self.last_info = {}
self.i = 0
for i, a in enumerate(self.agents):
self.last_obs[i] = a.reset()
self.last_rew[i] = None
self.last_done[i] = False
self.last_info[i] = {}
obs_dict = {self.i: self.last_obs[self.i]}
self.i = (self.i + 1) % len(self.agents)
return obs_dict
def step(self, action_dict):
assert len(self.dones) != len(self.agents)
for i, action in action_dict.items():
(self.last_obs[i], self.last_rew[i], self.last_done[i],
self.last_info[i]) = self.agents[i].step(action)
obs = {self.i: self.last_obs[self.i]}
rew = {self.i: self.last_rew[self.i]}
done = {self.i: self.last_done[self.i]}
info = {self.i: self.last_info[self.i]}
if done[self.i]:
rew[self.i] = 0
self.dones.add(self.i)
self.i = (self.i + 1) % len(self.agents)
done["__all__"] = len(self.dones) == len(self.agents) - 1
return obs, rew, done, info
class RoundRobinMultiAgent(MultiAgentEnv):
"""Env of N independent agents, each of which exits after 5 steps.
On each step() of the env, only one agent takes an action."""
def __init__(self, num, increment_obs=False):
if increment_obs:
# Observations are 0, 1, 2, 3... etc. as time advances
self.agents = [MockEnv2(5) for _ in range(num)]
else:
# Observations are all zeros
self.agents = [MockEnv(5) for _ in range(num)]
self.dones = set()
self.last_obs = {}
self.last_rew = {}
self.last_done = {}
self.last_info = {}
self.i = 0
self.num = num
self.observation_space = gym.spaces.Discrete(10)
self.action_space = gym.spaces.Discrete(2)
def reset(self):
self.dones = set()
self.last_obs = {}
self.last_rew = {}
self.last_done = {}
self.last_info = {}
self.i = 0
for i, a in enumerate(self.agents):
self.last_obs[i] = a.reset()
self.last_rew[i] = None
self.last_done[i] = False
self.last_info[i] = {}
obs_dict = {self.i: self.last_obs[self.i]}
self.i = (self.i + 1) % self.num
return obs_dict
def step(self, action_dict):
assert len(self.dones) != len(self.agents)
for i, action in action_dict.items():
(self.last_obs[i], self.last_rew[i], self.last_done[i],
self.last_info[i]) = self.agents[i].step(action)
obs = {self.i: self.last_obs[self.i]}
rew = {self.i: self.last_rew[self.i]}
done = {self.i: self.last_done[self.i]}
info = {self.i: self.last_info[self.i]}
if done[self.i]:
rew[self.i] = 0
self.dones.add(self.i)
self.i = (self.i + 1) % self.num
done["__all__"] = len(self.dones) == len(self.agents)
return obs, rew, done, info
MultiAgentCartPole = make_multiagent("CartPole-v0")
MultiAgentMountainCar = make_multiagent("MountainCarContinuous-v0")
MultiAgentPendulum = make_multiagent("Pendulum-v0")
rllib/examples/env/nested_space_repeat_after_me_env.py
Vendored
+52
View File
@@ -0,0 +1,52 @@
import gym
from gym.spaces import Box, Dict, Discrete, Tuple
import numpy as np
from ray.rllib.utils import try_import_tree
from ray.rllib.utils.space_utils import flatten_space
tree = try_import_tree()
class NestedSpaceRepeatAfterMeEnv(gym.Env):
"""Env for which policy has to repeat the (possibly complex) observation.
The action space and observation spaces are always the same and may be
arbitrarily nested Dict/Tuple Spaces.
Rewards are given for exactly matching Discrete sub-actions and for being
as close as possible for Box sub-actions.
"""
def __init__(self, config):
self.observation_space = config.get(
"space", Tuple([Discrete(2),
Dict({
"a": Box(-1.0, 1.0, (2, ))
})]))
self.action_space = self.observation_space
self.flattened_action_space = flatten_space(self.action_space)
self.episode_len = config.get("episode_len", 100)
def reset(self):
self.steps = 0
return self._next_obs()
def step(self, action):
self.steps += 1
action = tree.flatten(action)
reward = 0.0
for a, o, space in zip(action, self.current_obs_flattened,
self.flattened_action_space):
# Box: -abs(diff).
if isinstance(space, gym.spaces.Box):
reward -= np.abs(np.sum(a - o))
# Discrete: +1.0 if exact match.
if isinstance(space, gym.spaces.Discrete):
reward += 1.0 if a == o else 0.0
done = self.steps >= self.episode_len
return self._next_obs(), reward, done, {}
def _next_obs(self):
self.current_obs = self.observation_space.sample()
self.current_obs_flattened = tree.flatten(self.current_obs)
return self.current_obs
rllib/examples/env/parametric_actions_cartpole.py
Vendored
+77
View File
@@ -0,0 +1,77 @@
import gym
from gym.spaces import Box, Dict, Discrete
import numpy as np
import random
class ParametricActionsCartPole(gym.Env):
"""Parametric action version of CartPole.
In this env there are only ever two valid actions, but we pretend there are
actually up to `max_avail_actions` actions that can be taken, and the two
valid actions are randomly hidden among this set.
At each step, we emit a dict of:
- the actual cart observation
- a mask of valid actions (e.g., [0, 0, 1, 0, 0, 1] for 6 max avail)
- the list of action embeddings (w/ zeroes for invalid actions) (e.g.,
[[0, 0],
[0, 0],
[-0.2322, -0.2569],
[0, 0],
[0, 0],
[0.7878, 1.2297]] for max_avail_actions=6)
In a real environment, the actions embeddings would be larger than two
units of course, and also there would be a variable number of valid actions
per step instead of always [LEFT, RIGHT].
"""
def __init__(self, max_avail_actions):
# Use simple random 2-unit action embeddings for [LEFT, RIGHT]
self.left_action_embed = np.random.randn(2)
self.right_action_embed = np.random.randn(2)
self.action_space = Discrete(max_avail_actions)
self.wrapped = gym.make("CartPole-v0")
self.observation_space = Dict({
"action_mask": Box(0, 1, shape=(max_avail_actions, )),
"avail_actions": Box(-10, 10, shape=(max_avail_actions, 2)),
"cart": self.wrapped.observation_space,
})
def update_avail_actions(self):
self.action_assignments = np.array([[0., 0.]] * self.action_space.n)
self.action_mask = np.array([0.] * self.action_space.n)
self.left_idx, self.right_idx = random.sample(
range(self.action_space.n), 2)
self.action_assignments[self.left_idx] = self.left_action_embed
self.action_assignments[self.right_idx] = self.right_action_embed
self.action_mask[self.left_idx] = 1
self.action_mask[self.right_idx] = 1
def reset(self):
self.update_avail_actions()
return {
"action_mask": self.action_mask,
"avail_actions": self.action_assignments,
"cart": self.wrapped.reset(),
}
def step(self, action):
if action == self.left_idx:
actual_action = 0
elif action == self.right_idx:
actual_action = 1
else:
raise ValueError(
"Chosen action was not one of the non-zero action embeddings",
action, self.action_assignments, self.action_mask,
self.left_idx, self.right_idx)
orig_obs, rew, done, info = self.wrapped.step(actual_action)
self.update_avail_actions()
obs = {
"action_mask": self.action_mask,
"avail_actions": self.action_assignments,
"cart": orig_obs,
}
return obs, rew, done, info
rllib/examples/env/random_env.py
Vendored
+45
View File
@@ -0,0 +1,45 @@
import gym
from gym.spaces import Tuple
import numpy as np
class RandomEnv(gym.Env):
"""A randomly acting environment.
Can be instantiated with arbitrary action-, observation-, and reward
spaces. Observations and rewards are generated by simply sampling from the
observation/reward spaces. The probability of a `done=True` can be
configured as well.
"""
def __init__(self, config):
# Action space.
self.action_space = config["action_space"]
# Observation space from which to sample.
self.observation_space = config["observation_space"]
# Reward space from which to sample.
self.reward_space = config.get(
"reward_space",
gym.spaces.Box(low=-1.0, high=1.0, shape=(), dtype=np.float32))
# Chance that an episode ends at any step.
self.p_done = config.get("p_done", 0.1)
# Whether to check action bounds.
self.check_action_bounds = config.get("check_action_bounds", False)
def reset(self):
return self.observation_space.sample()
def step(self, action):
if self.check_action_bounds and not self.action_space.contains(action):
raise ValueError("Illegal action for {}: {}".format(
self.action_space, action))
if (isinstance(self.action_space, Tuple)
and len(action) != len(self.action_space.spaces)):
raise ValueError("Illegal action for {}: {}".format(
self.action_space, action))
return self.observation_space.sample(), \
float(self.reward_space.sample()), \
bool(np.random.choice(
[True, False], p=[self.p_done, 1.0 - self.p_done]
)), {}
rllib/examples/env/repeat_after_me_env.py
Vendored
+31
View File
@@ -0,0 +1,31 @@
import gym
from gym.spaces import Discrete
import random
class RepeatAfterMeEnv(gym.Env):
"""Env in which the observation at timestep minus n must be repeated."""
def __init__(self, config):
self.observation_space = Discrete(2)
self.action_space = Discrete(2)
self.delay = config["repeat_delay"]
assert self.delay >= 1, "`repeat_delay` must be at least 1!"
self.history = []
def reset(self):
self.history = [0] * self.delay
return self._next_obs()
def step(self, action):
if action == self.history[-(1 + self.delay)]:
reward = 1
else:
reward = -1
done = len(self.history) > 100
return self._next_obs(), reward, done, {}
def _next_obs(self):
token = random.choice([0, 1])
self.history.append(token)
return token
rllib/examples/env/repeat_initial_obs_env.py
Vendored
+32
View File
@@ -0,0 +1,32 @@
import gym
from gym.spaces import Discrete
import random
class RepeatInitialObsEnv(gym.Env):
"""Env in which the initial observation has to be repeated all the time.
Runs for n steps.
r=1 if action correct, -1 otherwise (max. R=100).
"""
def __init__(self, episode_len=100):
self.observation_space = Discrete(2)
self.action_space = Discrete(2)
self.token = None
self.episode_len = episode_len
self.num_steps = 0
def reset(self):
self.token = random.choice([0, 1])
self.num_steps = 0
return self.token
def step(self, action):
if action == self.token:
reward = 1
else:
reward = -1
self.num_steps += 1
done = self.num_steps >= self.episode_len
return 0, reward, done, {}
rllib/examples/env/rock_paper_scissors.py
Vendored
+82
View File
@@ -0,0 +1,82 @@
from gym.spaces import Discrete
from ray.rllib.env.multi_agent_env import MultiAgentEnv
class RockPaperScissors(MultiAgentEnv):
"""Two-player environment for the famous rock paper scissors game.
The observation is simply the last opponent action."""
ROCK = 0
PAPER = 1
SCISSORS = 2
LIZARD = 3
SPOCK = 4
def __init__(self, config):
self.action_space = Discrete(3)
self.observation_space = Discrete(3)
self.sheldon_cooper = config.get("sheldon_cooper", False)
self.player1 = "player1"
self.player2 = "player2"
self.last_move = None
self.num_moves = 0
def reset(self):
self.last_move = (0, 0)
self.num_moves = 0
return {
self.player1: self.last_move[1],
self.player2: self.last_move[0],
}
def step(self, action_dict):
move1 = action_dict[self.player1]
move2 = action_dict[self.player2]
if self.sheldon_cooper is False:
assert move1 not in [self.LIZARD, self.SPOCK]
assert move2 not in [self.LIZARD, self.SPOCK]
self.last_move = (move1, move2)
obs = {
self.player1: self.last_move[1],
self.player2: self.last_move[0],
}
r1, r2 = {
(self.ROCK, self.ROCK): (0, 0),
(self.ROCK, self.PAPER): (-1, 1),
(self.ROCK, self.SCISSORS): (1, -1),
(self.PAPER, self.ROCK): (1, -1),
(self.PAPER, self.PAPER): (0, 0),
(self.PAPER, self.SCISSORS): (-1, 1),
(self.SCISSORS, self.ROCK): (-1, 1),
(self.SCISSORS, self.PAPER): (1, -1),
(self.SCISSORS, self.SCISSORS): (0, 0),
# Sheldon Cooper extension:
(self.LIZARD, self.LIZARD): (0, 0),
(self.LIZARD, self.SPOCK): (1, -1), # Lizard poisons Spock
(self.LIZARD, self.ROCK): (-1, 1), # Rock crushes lizard
(self.LIZARD, self.PAPER): (1, -1), # Lizard eats paper
(self.LIZARD, self.SCISSORS): (-1, 1), # Scissors decapitate Lizrd
(self.ROCK, self.LIZARD): (1, -1), # Rock crushes lizard
(self.PAPER, self.LIZARD): (-1, 1), # Lizard eats paper
(self.SCISSORS, self.LIZARD): (1, -1), # Scissors decapitate Lizrd
(self.SPOCK, self.SPOCK): (0, 0),
(self.SPOCK, self.LIZARD): (-1, 1), # Lizard poisons Spock
(self.SPOCK, self.ROCK): (1, -1), # Spock vaporizes rock
(self.SPOCK, self.PAPER): (-1, 1), # Paper disproves Spock
(self.SPOCK, self.SCISSORS): (1, -1), # Spock smashes scissors
(self.ROCK, self.SPOCK): (-1, 1), # Spock vaporizes rock
(self.PAPER, self.SPOCK): (1, -1), # Paper disproves Spock
(self.SCISSORS, self.SPOCK): (-1, 1), # Spock smashes scissors
}[move1, move2]
rew = {
self.player1: r1,
self.player2: r2,
}
self.num_moves += 1
done = {
"__all__": self.num_moves >= 10,
}
return obs, rew, done, {}
rllib/examples/env/simple_corridor.py
Vendored
+35
View File
@@ -0,0 +1,35 @@
import gym
from gym.spaces import Box, Discrete
import numpy as np
class SimpleCorridor(gym.Env):
"""Example of a custom env in which you have to walk down a corridor.
You can configure the length of the corridor via the env config."""
def __init__(self, config):
self.end_pos = config["corridor_length"]
self.cur_pos = 0
self.action_space = Discrete(2)
self.observation_space = Box(
0.0, self.end_pos, shape=(1, ), dtype=np.float32)
def set_corridor_length(self, length):
self.end_pos = length
self.observation_space = Box(
0.0, self.end_pos, shape=(1, ), dtype=np.float32)
print("Updated corridor length to {}".format(length))
def reset(self):
self.cur_pos = 0
return [self.cur_pos]
def step(self, action):
assert action in [0, 1], action
if action == 0 and self.cur_pos > 0:
self.cur_pos -= 1
elif action == 1:
self.cur_pos += 1
done = self.cur_pos >= self.end_pos
return [self.cur_pos], 1 if done else 0, done, {}
rllib/examples/env/stateless_cartpole.py
Vendored
+152
View File
@@ -0,0 +1,152 @@
import math
import gym
from gym import spaces
from gym.utils import seeding
import numpy as np
class StatelessCartPole(gym.Env):
"""Partially observable variant of the CartPole gym environment.
https://github.com/openai/gym/blob/master/gym/envs/classic_control/
cartpole.py
We delete the velocity component of the state, so that it can only be
solved by a LSTM policy.
"""
metadata = {
"render.modes": ["human", "rgb_array"],
"video.frames_per_second": 60
}
def __init__(self, config=None):
self.gravity = 9.8
self.masscart = 1.0
self.masspole = 0.1
self.total_mass = (self.masspole + self.masscart)
self.length = 0.5 # actually half the pole's length
self.polemass_length = (self.masspole * self.length)
self.force_mag = 10.0
self.tau = 0.02 # seconds between state updates
# Angle at which to fail the episode
self.theta_threshold_radians = 12 * 2 * math.pi / 360
self.x_threshold = 2.4
high = np.array([
self.x_threshold * 2,
self.theta_threshold_radians * 2,
])
self.action_space = spaces.Discrete(2)
self.observation_space = spaces.Box(-high, high)
self.seed()
self.viewer = None
self.state = None
self.steps_beyond_done = None
def seed(self, seed=None):
self.np_random, seed = seeding.np_random(seed)
return [seed]
def step(self, action):
assert self.action_space.contains(
action), "%r (%s) invalid" % (action, type(action))
state = self.state
x, x_dot, theta, theta_dot = state
force = self.force_mag if action == 1 else -self.force_mag
costheta = math.cos(theta)
sintheta = math.sin(theta)
temp = (force + self.polemass_length * theta_dot * theta_dot * sintheta
) / self.total_mass
thetaacc = (self.gravity * sintheta - costheta * temp) / (
self.length *
(4.0 / 3.0 - self.masspole * costheta * costheta / self.total_mass)
)
xacc = (temp -
self.polemass_length * thetaacc * costheta / self.total_mass)
x = x + self.tau * x_dot
x_dot = x_dot + self.tau * xacc
theta = theta + self.tau * theta_dot
theta_dot = theta_dot + self.tau * thetaacc
self.state = (x, x_dot, theta, theta_dot)
done = (x < -self.x_threshold or x > self.x_threshold
or theta < -self.theta_threshold_radians
or theta > self.theta_threshold_radians)
done = bool(done)
if not done:
reward = 1.0
elif self.steps_beyond_done is None:
# Pole just fell!
self.steps_beyond_done = 0
reward = 1.0
else:
self.steps_beyond_done += 1
reward = 0.0
rv = np.r_[self.state[0], self.state[2]]
return rv, reward, done, {}
def reset(self):
self.state = self.np_random.uniform(low=-0.05, high=0.05, size=(4, ))
self.steps_beyond_done = None
rv = np.r_[self.state[0], self.state[2]]
return rv
def render(self, mode="human"):
screen_width = 600
screen_height = 400
world_width = self.x_threshold * 2
scale = screen_width / world_width
carty = 100 # TOP OF CART
polewidth = 10.0
polelen = scale * 1.0
cartwidth = 50.0
cartheight = 30.0
if self.viewer is None:
from gym.envs.classic_control import rendering
self.viewer = rendering.Viewer(screen_width, screen_height)
l, r, t, b = (-cartwidth / 2, cartwidth / 2, cartheight / 2,
-cartheight / 2)
axleoffset = cartheight / 4.0
cart = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)])
self.carttrans = rendering.Transform()
cart.add_attr(self.carttrans)
self.viewer.add_geom(cart)
l, r, t, b = (-polewidth / 2, polewidth / 2,
polelen - polewidth / 2, -polewidth / 2)
pole = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)])
pole.set_color(.8, .6, .4)
self.poletrans = rendering.Transform(translation=(0, axleoffset))
pole.add_attr(self.poletrans)
pole.add_attr(self.carttrans)
self.viewer.add_geom(pole)
self.axle = rendering.make_circle(polewidth / 2)
self.axle.add_attr(self.poletrans)
self.axle.add_attr(self.carttrans)
self.axle.set_color(.5, .5, .8)
self.viewer.add_geom(self.axle)
self.track = rendering.Line((0, carty), (screen_width, carty))
self.track.set_color(0, 0, 0)
self.viewer.add_geom(self.track)
if self.state is None:
return None
x = self.state
cartx = x[0] * scale + screen_width / 2.0 # MIDDLE OF CART
self.carttrans.set_translation(cartx, carty)
self.poletrans.set_rotation(-x[2])
return self.viewer.render(return_rgb_array=mode == "rgb_array")
def close(self):
if self.viewer:
self.viewer.close()
rllib/examples/env/two_step_game.py
Vendored
+107
View File
@@ -0,0 +1,107 @@
from gym.spaces import MultiDiscrete, Dict, Discrete
import numpy as np
from ray.rllib.env.multi_agent_env import MultiAgentEnv, ENV_STATE
class TwoStepGame(MultiAgentEnv):
action_space = Discrete(2)
def __init__(self, env_config):
self.state = None
self.agent_1 = 0
self.agent_2 = 1
# MADDPG emits action logits instead of actual discrete actions
self.actions_are_logits = env_config.get("actions_are_logits", False)
self.one_hot_state_encoding = env_config.get("one_hot_state_encoding",
False)
self.with_state = env_config.get("separate_state_space", False)
if not self.one_hot_state_encoding:
self.observation_space = Discrete(6)
self.with_state = False
else:
# Each agent gets the full state (one-hot encoding of which of the
# three states are active) as input with the receiving agent's
# ID (1 or 2) concatenated onto the end.
if self.with_state:
self.observation_space = Dict({
"obs": MultiDiscrete([2, 2, 2, 3]),
ENV_STATE: MultiDiscrete([2, 2, 2])
})
else:
self.observation_space = MultiDiscrete([2, 2, 2, 3])
def reset(self):
self.state = np.array([1, 0, 0])
return self._obs()
def step(self, action_dict):
if self.actions_are_logits:
action_dict = {
k: np.random.choice([0, 1], p=v)
for k, v in action_dict.items()
}
state_index = np.flatnonzero(self.state)
if state_index == 0:
action = action_dict[self.agent_1]
assert action in [0, 1], action
if action == 0:
self.state = np.array([0, 1, 0])
else:
self.state = np.array([0, 0, 1])
global_rew = 0
done = False
elif state_index == 1:
global_rew = 7
done = True
else:
if action_dict[self.agent_1] == 0 and action_dict[self.
agent_2] == 0:
global_rew = 0
elif action_dict[self.agent_1] == 1 and action_dict[self.
agent_2] == 1:
global_rew = 8
else:
global_rew = 1
done = True
rewards = {
self.agent_1: global_rew / 2.0,
self.agent_2: global_rew / 2.0
}
obs = self._obs()
dones = {"__all__": done}
infos = {}
return obs, rewards, dones, infos
def _obs(self):
if self.with_state:
return {
self.agent_1: {
"obs": self.agent_1_obs(),
ENV_STATE: self.state
},
self.agent_2: {
"obs": self.agent_2_obs(),
ENV_STATE: self.state
}
}
else:
return {
self.agent_1: self.agent_1_obs(),
self.agent_2: self.agent_2_obs()
}
def agent_1_obs(self):
if self.one_hot_state_encoding:
return np.concatenate([self.state, [1]])
else:
return np.flatnonzero(self.state)[0]
def agent_2_obs(self):
if self.one_hot_state_encoding:
return np.concatenate([self.state, [2]])
else:
return np.flatnonzero(self.state)[0] + 3
rllib/examples/env/windy_maze_env.py
Vendored
+146
View File
@@ -0,0 +1,146 @@
import gym
from gym.spaces import Box, Discrete, Tuple
import logging
import random
from ray.rllib.env import MultiAgentEnv
logger = logging.getLogger(__name__)
# Agent has to traverse the maze from the starting position S -> F
# Observation space [x_pos, y_pos, wind_direction]
# Action space: stay still OR move in current wind direction
MAP_DATA = """
#########
#S #
####### #
# #
# #
####### #
#F #
#########"""
class WindyMazeEnv(gym.Env):
def __init__(self, env_config):
self.map = [m for m in MAP_DATA.split("\n") if m]
self.x_dim = len(self.map)
self.y_dim = len(self.map[0])
logger.info("Loaded map {} {}".format(self.x_dim, self.y_dim))
for x in range(self.x_dim):
for y in range(self.y_dim):
if self.map[x][y] == "S":
self.start_pos = (x, y)
elif self.map[x][y] == "F":
self.end_pos = (x, y)
logger.info("Start pos {} end pos {}".format(self.start_pos,
self.end_pos))
self.observation_space = Tuple([
Box(0, 100, shape=(2, )), # (x, y)
Discrete(4), # wind direction (N, E, S, W)
])
self.action_space = Discrete(2) # whether to move or not
def reset(self):
self.wind_direction = random.choice([0, 1, 2, 3])
self.pos = self.start_pos
self.num_steps = 0
return [[self.pos[0], self.pos[1]], self.wind_direction]
def step(self, action):
if action == 1:
self.pos = self._get_new_pos(self.pos, self.wind_direction)
self.num_steps += 1
self.wind_direction = random.choice([0, 1, 2, 3])
at_goal = self.pos == self.end_pos
done = at_goal or self.num_steps >= 200
return ([[self.pos[0], self.pos[1]], self.wind_direction],
100 * int(at_goal), done, {})
def _get_new_pos(self, pos, direction):
if direction == 0:
new_pos = (pos[0] - 1, pos[1])
elif direction == 1:
new_pos = (pos[0], pos[1] + 1)
elif direction == 2:
new_pos = (pos[0] + 1, pos[1])
elif direction == 3:
new_pos = (pos[0], pos[1] - 1)
if (new_pos[0] >= 0 and new_pos[0] < self.x_dim and new_pos[1] >= 0
and new_pos[1] < self.y_dim
and self.map[new_pos[0]][new_pos[1]] != "#"):
return new_pos
else:
return pos # did not move
class HierarchicalWindyMazeEnv(MultiAgentEnv):
def __init__(self, env_config):
self.flat_env = WindyMazeEnv(env_config)
def reset(self):
self.cur_obs = self.flat_env.reset()
self.current_goal = None
self.steps_remaining_at_level = None
self.num_high_level_steps = 0
# current low level agent id. This must be unique for each high level
# step since agent ids cannot be reused.
self.low_level_agent_id = "low_level_{}".format(
self.num_high_level_steps)
return {
"high_level_agent": self.cur_obs,
}
def step(self, action_dict):
assert len(action_dict) == 1, action_dict
if "high_level_agent" in action_dict:
return self._high_level_step(action_dict["high_level_agent"])
else:
return self._low_level_step(list(action_dict.values())[0])
def _high_level_step(self, action):
logger.debug("High level agent sets goal".format(action))
self.current_goal = action
self.steps_remaining_at_level = 25
self.num_high_level_steps += 1
self.low_level_agent_id = "low_level_{}".format(
self.num_high_level_steps)
obs = {self.low_level_agent_id: [self.cur_obs, self.current_goal]}
rew = {self.low_level_agent_id: 0}
done = {"__all__": False}
return obs, rew, done, {}
def _low_level_step(self, action):
logger.debug("Low level agent step {}".format(action))
self.steps_remaining_at_level -= 1
cur_pos = tuple(self.cur_obs[0])
goal_pos = self.flat_env._get_new_pos(cur_pos, self.current_goal)
# Step in the actual env
f_obs, f_rew, f_done, _ = self.flat_env.step(action)
new_pos = tuple(f_obs[0])
self.cur_obs = f_obs
# Calculate low-level agent observation and reward
obs = {self.low_level_agent_id: [f_obs, self.current_goal]}
if new_pos != cur_pos:
if new_pos == goal_pos:
rew = {self.low_level_agent_id: 1}
else:
rew = {self.low_level_agent_id: -1}
else:
rew = {self.low_level_agent_id: 0}
# Handle env termination & transitions back to higher level
done = {"__all__": False}
if f_done:
done["__all__"] = True
logger.debug("high level final reward {}".format(f_rew))
rew["high_level_agent"] = f_rew
obs["high_level_agent"] = f_obs
elif self.steps_remaining_at_level == 0:
done[self.low_level_agent_id] = True
rew["high_level_agent"] = 0
obs["high_level_agent"] = f_obs
return obs, rew, done, {}
rllib/examples/hierarchical_training.py
+48 -166
View File
@@ -23,169 +23,40 @@ using --flat in this example.
"""
import argparse
import random
import gym
from gym.spaces import Box, Discrete, Tuple
from gym.spaces import Discrete, Tuple
import logging
import ray
from ray import tune
from ray.rllib.examples.env.windy_maze_env import WindyMazeEnv, \
HierarchicalWindyMazeEnv
from ray.tune import function
from ray.rllib.env import MultiAgentEnv
parser = argparse.ArgumentParser()
parser.add_argument("--flat", action="store_true")
# Agent has to traverse the maze from the starting position S -> F
# Observation space [x_pos, y_pos, wind_direction]
# Action space: stay still OR move in current wind direction
MAP_DATA = """
#########
#S #
####### #
# #
# #
####### #
#F #
#########"""
parser.add_argument("--torch", action="store_true")
parser.add_argument("--stop-reward", type=float, default=0.0)
parser.add_argument("--stop-timesteps", type=int, default=100000)
logger = logging.getLogger(__name__)
class WindyMazeEnv(gym.Env):
def __init__(self, env_config):
self.map = [m for m in MAP_DATA.split("\n") if m]
self.x_dim = len(self.map)
self.y_dim = len(self.map[0])
logger.info("Loaded map {} {}".format(self.x_dim, self.y_dim))
for x in range(self.x_dim):
for y in range(self.y_dim):
if self.map[x][y] == "S":
self.start_pos = (x, y)
elif self.map[x][y] == "F":
self.end_pos = (x, y)
logger.info("Start pos {} end pos {}".format(self.start_pos,
self.end_pos))
self.observation_space = Tuple([
Box(0, 100, shape=(2, )), # (x, y)
Discrete(4), # wind direction (N, E, S, W)
])
self.action_space = Discrete(2) # whether to move or not
def reset(self):
self.wind_direction = random.choice([0, 1, 2, 3])
self.pos = self.start_pos
self.num_steps = 0
return [[self.pos[0], self.pos[1]], self.wind_direction]
def step(self, action):
if action == 1:
self.pos = self._get_new_pos(self.pos, self.wind_direction)
self.num_steps += 1
self.wind_direction = random.choice([0, 1, 2, 3])
at_goal = self.pos == self.end_pos
done = at_goal or self.num_steps >= 200
return ([[self.pos[0], self.pos[1]], self.wind_direction],
100 * int(at_goal), done, {})
def _get_new_pos(self, pos, direction):
if direction == 0:
new_pos = (pos[0] - 1, pos[1])
elif direction == 1:
new_pos = (pos[0], pos[1] + 1)
elif direction == 2:
new_pos = (pos[0] + 1, pos[1])
elif direction == 3:
new_pos = (pos[0], pos[1] - 1)
if (new_pos[0] >= 0 and new_pos[0] < self.x_dim and new_pos[1] >= 0
and new_pos[1] < self.y_dim
and self.map[new_pos[0]][new_pos[1]] != "#"):
return new_pos
else:
return pos # did not move
class HierarchicalWindyMazeEnv(MultiAgentEnv):
def __init__(self, env_config):
self.flat_env = WindyMazeEnv(env_config)
def reset(self):
self.cur_obs = self.flat_env.reset()
self.current_goal = None
self.steps_remaining_at_level = None
self.num_high_level_steps = 0
# current low level agent id. This must be unique for each high level
# step since agent ids cannot be reused.
self.low_level_agent_id = "low_level_{}".format(
self.num_high_level_steps)
return {
"high_level_agent": self.cur_obs,
}
def step(self, action_dict):
assert len(action_dict) == 1, action_dict
if "high_level_agent" in action_dict:
return self._high_level_step(action_dict["high_level_agent"])
else:
return self._low_level_step(list(action_dict.values())[0])
def _high_level_step(self, action):
logger.debug("High level agent sets goal".format(action))
self.current_goal = action
self.steps_remaining_at_level = 25
self.num_high_level_steps += 1
self.low_level_agent_id = "low_level_{}".format(
self.num_high_level_steps)
obs = {self.low_level_agent_id: [self.cur_obs, self.current_goal]}
rew = {self.low_level_agent_id: 0}
done = {"__all__": False}
return obs, rew, done, {}
def _low_level_step(self, action):
logger.debug("Low level agent step {}".format(action))
self.steps_remaining_at_level -= 1
cur_pos = tuple(self.cur_obs[0])
goal_pos = self.flat_env._get_new_pos(cur_pos, self.current_goal)
# Step in the actual env
f_obs, f_rew, f_done, _ = self.flat_env.step(action)
new_pos = tuple(f_obs[0])
self.cur_obs = f_obs
# Calculate low-level agent observation and reward
obs = {self.low_level_agent_id: [f_obs, self.current_goal]}
if new_pos != cur_pos:
if new_pos == goal_pos:
rew = {self.low_level_agent_id: 1}
else:
rew = {self.low_level_agent_id: -1}
else:
rew = {self.low_level_agent_id: 0}
# Handle env termination & transitions back to higher level
done = {"__all__": False}
if f_done:
done["__all__"] = True
logger.debug("high level final reward {}".format(f_rew))
rew["high_level_agent"] = f_rew
obs["high_level_agent"] = f_obs
elif self.steps_remaining_at_level == 0:
done[self.low_level_agent_id] = True
rew["high_level_agent"] = 0
obs["high_level_agent"] = f_obs
return obs, rew, done, {}
if __name__ == "__main__":
args = parser.parse_args()
ray.init()
stop = {
"episode_reward_mean": args.stop_reward,
"timesteps_total": args.stop_timesteps,
}
if args.flat:
tune.run(
results = tune.run(
"PPO",
stop=stop,
config={
"env": WindyMazeEnv,
"num_workers": 0,
"use_pytorch": args.torch,
},
)
else:
@@ -197,28 +68,39 @@ if __name__ == "__main__":
else:
return "high_level_policy"
tune.run(
"PPO",
config={
"env": HierarchicalWindyMazeEnv,
"num_workers": 0,
"log_level": "INFO",
"entropy_coeff": 0.01,
"multiagent": {
"policies": {
"high_level_policy": (None, maze.observation_space,
Discrete(4), {
"gamma": 0.9
}),
"low_level_policy": (None,
Tuple([
maze.observation_space,
Discrete(4)
]), maze.action_space, {
"gamma": 0.0
}),
},
"policy_mapping_fn": function(policy_mapping_fn),
config = {
"env": HierarchicalWindyMazeEnv,
"num_workers": 0,
"log_level": "INFO",
"entropy_coeff": 0.01,
"multiagent": {
"policies": {
"high_level_policy": (None, maze.observation_space,
Discrete(4), {
"gamma": 0.9
}),
"low_level_policy": (None,
Tuple([
maze.observation_space,
Discrete(4)
]), maze.action_space, {
"gamma": 0.0
}),
},
"policy_mapping_fn": function(policy_mapping_fn),
},
"use_pytorch": args.torch,
}
results = tune.run(
"PPO",
stop=stop,
config=config,
)
# Error if stop-reward not reached.
if results.trials[0].last_result["episode_reward_mean"] < \
args.stop_reward:
raise ValueError("`stop-reward` of {} not reached!".format(
args.stop_reward))
print("ok")
rllib/examples/multi_agent_cartpole.py
+5 -5
View File
@@ -15,11 +15,10 @@ import random
import ray
from ray import tune
from ray.rllib.examples.env.multi_agent import MultiAgentCartPole
from ray.rllib.models import ModelCatalog
from ray.rllib.models.modelv2 import ModelV2
from ray.rllib.models.tf.tf_modelv2 import TFModelV2
from ray.rllib.tests.test_multi_agent_env import MultiCartpole
from ray.tune.registry import register_env
from ray.rllib.utils import try_import_tf
from ray.rllib.utils.annotations import override
@@ -105,8 +104,6 @@ if __name__ == "__main__":
args = parser.parse_args()
ray.init(num_cpus=args.num_cpus or None)
# Simple environment with `num_agents` independent cartpole entities
register_env("multi_cartpole", lambda _: MultiCartpole(args.num_agents))
ModelCatalog.register_custom_model("model1", CustomModel1)
ModelCatalog.register_custom_model("model2", CustomModel2)
single_env = gym.make("CartPole-v0")
@@ -134,7 +131,10 @@ if __name__ == "__main__":
"PPO",
stop={"training_iteration": args.num_iters},
config={
"env": "multi_cartpole",
"env": MultiAgentCartPole,
"env_config": {
"num_agents": args.num_agents,
},
"log_level": "DEBUG",
"simple_optimizer": args.simple,
"num_sgd_iter": 10,
rllib/examples/multi_agent_custom_policy.py
+4 -3
View File
@@ -18,8 +18,8 @@ import gym
import ray
from ray import tune
from ray.rllib.examples.env.multi_agent import MultiAgentCartPole
from ray.rllib.policy import Policy
from ray.rllib.tests.test_multi_agent_env import MultiCartpole
from ray.tune.registry import register_env
parser = argparse.ArgumentParser()
@@ -50,7 +50,8 @@ if __name__ == "__main__":
ray.init()
# Simple environment with 4 independent cartpole entities
register_env("multi_cartpole", lambda _: MultiCartpole(4))
register_env("multi_agent_cartpole",
lambda _: MultiAgentCartPole({"num_agents": 4}))
single_env = gym.make("CartPole-v0")
obs_space = single_env.observation_space
act_space = single_env.action_space
@@ -59,7 +60,7 @@ if __name__ == "__main__":
"PG",
stop={"training_iteration": args.num_iters},
config={
"env": "multi_cartpole",
"env": "multi_agent_cartpole",
"multiagent": {
"policies": {
"pg_policy": (None, obs_space, act_space, {}),
rllib/examples/multi_agent_two_trainers.py
+5 -4
View File
@@ -16,7 +16,7 @@ from ray.rllib.agents.dqn.dqn import DQNTrainer
from ray.rllib.agents.dqn.dqn_tf_policy import DQNTFPolicy
from ray.rllib.agents.ppo.ppo import PPOTrainer
from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy
from ray.rllib.tests.test_multi_agent_env import MultiCartpole
from ray.rllib.examples.env.multi_agent import MultiAgentCartPole
from ray.tune.logger import pretty_print
from ray.tune.registry import register_env
@@ -28,7 +28,8 @@ if __name__ == "__main__":
ray.init()
# Simple environment with 4 independent cartpole entities
register_env("multi_cartpole", lambda _: MultiCartpole(4))
register_env("multi_agent_cartpole",
lambda _: MultiAgentCartPole({"num_agents": 4}))
single_env = gym.make("CartPole-v0")
obs_space = single_env.observation_space
act_space = single_env.action_space
@@ -47,7 +48,7 @@ if __name__ == "__main__":
return "dqn_policy"
ppo_trainer = PPOTrainer(
env="multi_cartpole",
env="multi_agent_cartpole",
config={
"multiagent": {
"policies": policies,
@@ -61,7 +62,7 @@ if __name__ == "__main__":
})
dqn_trainer = DQNTrainer(
env="multi_cartpole",
env="multi_agent_cartpole",
config={
"multiagent": {
"policies": policies,
rllib/examples/nested_action_spaces.py
+8 -47
View File
@@ -1,64 +1,24 @@
import argparse
import gym
from gym.spaces import Dict, Tuple, Box, Discrete
import numpy as np
import sys
import ray
from ray.tune.registry import register_env
from ray.rllib.examples.env.nested_space_repeat_after_me_env import \
NestedSpaceRepeatAfterMeEnv
from ray.rllib.utils import try_import_tree
from ray.rllib.utils.framework import try_import_tf
from ray.rllib.utils.space_utils import flatten_space
tf = try_import_tf()
tree = try_import_tree()
parser = argparse.ArgumentParser()
parser.add_argument("--run", type=str, default="PPO")
parser.add_argument("--torch", action="store_true")
parser.add_argument("--stop", type=int, default=90)
parser.add_argument("--max-trainstop", type=int, default=90)
parser.add_argument("--num-cpus", type=int, default=0)
class NestedSpaceRepeatAfterMeEnv(gym.Env):
"""Env for which policy has to repeat the (possibly complex) observation.
"""
def __init__(self, config):
self.observation_space = config.get(
"space", Tuple([Discrete(2),
Dict({
"a": Box(-1.0, 1.0, (2, ))
})]))
self.action_space = self.observation_space
self.flattened_action_space = flatten_space(self.action_space)
self.episode_len = config.get("episode_len", 100)
def reset(self):
self.steps = 0
return self._next_obs()
def step(self, action):
self.steps += 1
action = tree.flatten(action)
reward = 0.0
for a, o, space in zip(action, self.current_obs_flattened,
self.flattened_action_space):
# Box: -abs(diff).
if isinstance(space, gym.spaces.Box):
reward -= np.abs(np.sum(a - o))
# Discrete: +1.0 if exact match.
if isinstance(space, gym.spaces.Discrete):
reward += 1.0 if a == o else 0.0
done = self.steps >= self.episode_len
return self._next_obs(), reward, done, {}
def _next_obs(self):
self.current_obs = self.observation_space.sample()
self.current_obs_flattened = tree.flatten(self.current_obs)
return self.current_obs
if __name__ == "__main__":
args = parser.parse_args()
ray.init(num_cpus=args.num_cpus or None)
@@ -78,13 +38,14 @@ if __name__ == "__main__":
"c": Discrete(4)
}),
},
"gamma": 0.0, # No history in Env (bandit problem).
"num_workers": 0,
"num_envs_per_worker": 20,
"entropy_coeff": 0.00005, # We don't want high entropy in this Env.
"gamma": 0.0, # No history in Env (bandit problem).
"lr": 0.0003,
"num_envs_per_worker": 20,
"num_sgd_iter": 20,
"num_workers": 0,
"use_pytorch": args.torch,
"vf_loss_coeff": 0.01,
"lr": 0.0003
}
import ray.rllib.agents.ppo as ppo
rllib/examples/parametric_action_cartpole.py → rllib/examples/parametric_actions_cartpole.py
+4 -78
View File
@@ -15,15 +15,14 @@ Working configurations are given below.
"""
import argparse
import random
import numpy as np
import gym
from gym.spaces import Box, Discrete, Dict
from gym.spaces import Box
import ray
from ray import tune
from ray.rllib.agents.dqn.distributional_q_tf_model import \
DistributionalQTFModel
from ray.rllib.examples.env.parametric_actions_cartpole import \
ParametricActionsCartPole
from ray.rllib.models import ModelCatalog
from ray.rllib.models.tf.fcnet_v2 import FullyConnectedNetwork
from ray.rllib.models.tf.tf_modelv2 import TFModelV2
@@ -37,79 +36,6 @@ parser.add_argument("--stop", type=int, default=200)
parser.add_argument("--run", type=str, default="PPO")
class ParametricActionCartpole(gym.Env):
"""Parametric action version of CartPole.
In this env there are only ever two valid actions, but we pretend there are
actually up to `max_avail_actions` actions that can be taken, and the two
valid actions are randomly hidden among this set.
At each step, we emit a dict of:
- the actual cart observation
- a mask of valid actions (e.g., [0, 0, 1, 0, 0, 1] for 6 max avail)
- the list of action embeddings (w/ zeroes for invalid actions) (e.g.,
[[0, 0],
[0, 0],
[-0.2322, -0.2569],
[0, 0],
[0, 0],
[0.7878, 1.2297]] for max_avail_actions=6)
In a real environment, the actions embeddings would be larger than two
units of course, and also there would be a variable number of valid actions
per step instead of always [LEFT, RIGHT].
"""
def __init__(self, max_avail_actions):
# Use simple random 2-unit action embeddings for [LEFT, RIGHT]
self.left_action_embed = np.random.randn(2)
self.right_action_embed = np.random.randn(2)
self.action_space = Discrete(max_avail_actions)
self.wrapped = gym.make("CartPole-v0")
self.observation_space = Dict({
"action_mask": Box(0, 1, shape=(max_avail_actions, )),
"avail_actions": Box(-10, 10, shape=(max_avail_actions, 2)),
"cart": self.wrapped.observation_space,
})
def update_avail_actions(self):
self.action_assignments = np.array([[0., 0.]] * self.action_space.n)
self.action_mask = np.array([0.] * self.action_space.n)
self.left_idx, self.right_idx = random.sample(
range(self.action_space.n), 2)
self.action_assignments[self.left_idx] = self.left_action_embed
self.action_assignments[self.right_idx] = self.right_action_embed
self.action_mask[self.left_idx] = 1
self.action_mask[self.right_idx] = 1
def reset(self):
self.update_avail_actions()
return {
"action_mask": self.action_mask,
"avail_actions": self.action_assignments,
"cart": self.wrapped.reset(),
}
def step(self, action):
if action == self.left_idx:
actual_action = 0
elif action == self.right_idx:
actual_action = 1
else:
raise ValueError(
"Chosen action was not one of the non-zero action embeddings",
action, self.action_assignments, self.action_mask,
self.left_idx, self.right_idx)
orig_obs, rew, done, info = self.wrapped.step(actual_action)
self.update_avail_actions()
obs = {
"action_mask": self.action_mask,
"avail_actions": self.action_assignments,
"cart": orig_obs,
}
return obs, rew, done, info
class ParametricActionsModel(DistributionalQTFModel, TFModelV2):
"""Parametric action model that handles the dot product and masking.
@@ -165,7 +91,7 @@ if __name__ == "__main__":
ray.init()
ModelCatalog.register_custom_model("pa_model", ParametricActionsModel)
register_env("pa_cartpole", lambda _: ParametricActionCartpole(10))
register_env("pa_cartpole", lambda _: ParametricActionsCartPole(10))
if args.run == "DQN":
cfg = {
# TODO(ekl) we need to set these to prevent the masked values
rllib/examples/random_env.py
-69
View File
@@ -1,69 +0,0 @@
"""
Example of a custom gym environment and model. Run this for a demo.
This example shows:
- using a custom environment
- using a custom model
- using Tune for grid search
You can visualize experiment results in ~/ray_results using TensorBoard.
"""
import gym
from gym.spaces import Tuple, Discrete
import numpy as np
from ray.rllib.agents.ppo import PPOTrainer
from ray.rllib.utils import try_import_tf
tf = try_import_tf()
class RandomEnv(gym.Env):
"""
A randomly acting environment that can be instantiated with arbitrary
action and observation spaces.
"""
def __init__(self, config):
# Action space.
self.action_space = config["action_space"]
# Observation space from which to sample.
self.observation_space = config["observation_space"]
# Reward space from which to sample.
self.reward_space = config.get(
"reward_space",
gym.spaces.Box(low=-1.0, high=1.0, shape=(), dtype=np.float32))
# Chance that an episode ends at any step.
self.p_done = config.get("p_done", 0.1)
def reset(self):
return self.observation_space.sample()
def step(self, action):
return self.observation_space.sample(), \
float(self.reward_space.sample()), \
bool(np.random.choice(
[True, False], p=[self.p_done, 1.0 - self.p_done]
)), {}
if __name__ == "__main__":
trainer = PPOTrainer(
config={
"model": {
"use_lstm": True,
},
"vf_share_layers": False,
"num_workers": 0, # no parallelism
"env_config": {
"action_space": Discrete(2),
# Test a simple Tuple observation space.
"observation_space": Tuple([Discrete(3),
Discrete(2)])
}
},
env=RandomEnv,
)
results = trainer.train()
print(results)
rllib/examples/rock_paper_scissors_multiagent.py
+15 -65
View File
@@ -14,8 +14,8 @@ from gym.spaces import Discrete
from ray import tune
from ray.rllib.agents.pg.pg import PGTrainer
from ray.rllib.agents.pg.pg_tf_policy import PGTFPolicy
from ray.rllib.examples.env.rock_paper_scissors import RockPaperScissors
from ray.rllib.policy.policy import Policy
from ray.rllib.env.multi_agent_env import MultiAgentEnv
from ray.rllib.utils import try_import_tf
parser = argparse.ArgumentParser()
@@ -23,61 +23,6 @@ parser.add_argument("--stop", type=int, default=1000)
tf = try_import_tf()
ROCK = 0
PAPER = 1
SCISSORS = 2
class RockPaperScissorsEnv(MultiAgentEnv):
"""Two-player environment for rock paper scissors.
The observation is simply the last opponent action."""
def __init__(self, _):
self.action_space = Discrete(3)
self.observation_space = Discrete(3)
self.player1 = "player1"
self.player2 = "player2"
self.last_move = None
self.num_moves = 0
def reset(self):
self.last_move = (0, 0)
self.num_moves = 0
return {
self.player1: self.last_move[1],
self.player2: self.last_move[0],
}
def step(self, action_dict):
move1 = action_dict[self.player1]
move2 = action_dict[self.player2]
self.last_move = (move1, move2)
obs = {
self.player1: self.last_move[1],
self.player2: self.last_move[0],
}
r1, r2 = {
(ROCK, ROCK): (0, 0),
(ROCK, PAPER): (-1, 1),
(ROCK, SCISSORS): (1, -1),
(PAPER, ROCK): (1, -1),
(PAPER, PAPER): (0, 0),
(PAPER, SCISSORS): (-1, 1),
(SCISSORS, ROCK): (-1, 1),
(SCISSORS, PAPER): (1, -1),
(SCISSORS, SCISSORS): (0, 0),
}[move1, move2]
rew = {
self.player1: r1,
self.player2: r2,
}
self.num_moves += 1
done = {
"__all__": self.num_moves >= 10,
}
return obs, rew, done, {}
class AlwaysSameHeuristic(Policy):
"""Pick a random move and stick with it for the entire episode."""
@@ -87,7 +32,12 @@ class AlwaysSameHeuristic(Policy):
self.exploration = self._create_exploration()
def get_initial_state(self):
return [random.choice([ROCK, PAPER, SCISSORS])]
return [
random.choice([
RockPaperScissors.ROCK, RockPaperScissors.PAPER,
RockPaperScissors.SCISSORS
])
]
def compute_actions(self,
obs_batch,
@@ -125,12 +75,12 @@ class BeatLastHeuristic(Policy):
episodes=None,
**kwargs):
def successor(x):
if x[ROCK] == 1:
return PAPER
elif x[PAPER] == 1:
return SCISSORS
elif x[SCISSORS] == 1:
return ROCK
if x[RockPaperScissors.ROCK] == 1:
return RockPaperScissors.PAPER
elif x[RockPaperScissors.PAPER] == 1:
return RockPaperScissors.SCISSORS
elif x[RockPaperScissors.SCISSORS] == 1:
return RockPaperScissors.ROCK
return [successor(x) for x in obs_batch], [], {}
@@ -150,7 +100,7 @@ def run_same_policy(args):
tune.run(
"PG",
stop={"timesteps_total": args.stop},
config={"env": RockPaperScissorsEnv})
config={"env": RockPaperScissors})
def run_heuristic_vs_learned(args, use_lstm=False, trainer="PG"):
@@ -169,7 +119,7 @@ def run_heuristic_vs_learned(args, use_lstm=False, trainer="PG"):
return random.choice(["always_same", "beat_last"])
config = {
"env": RockPaperScissorsEnv,
"env": RockPaperScissors,
"gamma": 0.9,
"num_workers": 0,
"num_envs_per_worker": 4,
rllib/examples/twostep_game.py
+2 -107
View File
@@ -11,123 +11,18 @@ See also: centralized_critic.py for centralized critic PPO on this game.
import argparse
from gym.spaces import Tuple, MultiDiscrete, Dict, Discrete
import numpy as np
import ray
from ray import tune
from ray.tune import register_env, grid_search
from ray.rllib.env.multi_agent_env import MultiAgentEnv
from ray.rllib.agents.qmix.qmix_policy import ENV_STATE
from ray.rllib.env.multi_agent_env import ENV_STATE
from ray.rllib.examples.env.two_step_game import TwoStepGame
parser = argparse.ArgumentParser()
parser.add_argument("--stop", type=int, default=50000)
parser.add_argument("--run", type=str, default="PG")
parser.add_argument("--num-cpus", type=int, default=0)
class TwoStepGame(MultiAgentEnv):
action_space = Discrete(2)
def __init__(self, env_config):
self.state = None
self.agent_1 = 0
self.agent_2 = 1
# MADDPG emits action logits instead of actual discrete actions
self.actions_are_logits = env_config.get("actions_are_logits", False)
self.one_hot_state_encoding = env_config.get("one_hot_state_encoding",
False)
self.with_state = env_config.get("separate_state_space", False)
if not self.one_hot_state_encoding:
self.observation_space = Discrete(6)
self.with_state = False
else:
# Each agent gets the full state (one-hot encoding of which of the
# three states are active) as input with the receiving agent's
# ID (1 or 2) concatenated onto the end.
if self.with_state:
self.observation_space = Dict({
"obs": MultiDiscrete([2, 2, 2, 3]),
ENV_STATE: MultiDiscrete([2, 2, 2])
})
else:
self.observation_space = MultiDiscrete([2, 2, 2, 3])
def reset(self):
self.state = np.array([1, 0, 0])
return self._obs()
def step(self, action_dict):
if self.actions_are_logits:
action_dict = {
k: np.random.choice([0, 1], p=v)
for k, v in action_dict.items()
}
state_index = np.flatnonzero(self.state)
if state_index == 0:
action = action_dict[self.agent_1]
assert action in [0, 1], action
if action == 0:
self.state = np.array([0, 1, 0])
else:
self.state = np.array([0, 0, 1])
global_rew = 0
done = False
elif state_index == 1:
global_rew = 7
done = True
else:
if action_dict[self.agent_1] == 0 and action_dict[self.
agent_2] == 0:
global_rew = 0
elif action_dict[self.agent_1] == 1 and action_dict[self.
agent_2] == 1:
global_rew = 8
else:
global_rew = 1
done = True
rewards = {
self.agent_1: global_rew / 2.0,
self.agent_2: global_rew / 2.0
}
obs = self._obs()
dones = {"__all__": done}
infos = {}
return obs, rewards, dones, infos
def _obs(self):
if self.with_state:
return {
self.agent_1: {
"obs": self.agent_1_obs(),
ENV_STATE: self.state
},
self.agent_2: {
"obs": self.agent_2_obs(),
ENV_STATE: self.state
}
}
else:
return {
self.agent_1: self.agent_1_obs(),
self.agent_2: self.agent_2_obs()
}
def agent_1_obs(self):
if self.one_hot_state_encoding:
return np.concatenate([self.state, [1]])
else:
return np.flatnonzero(self.state)[0]
def agent_2_obs(self):
if self.one_hot_state_encoding:
return np.concatenate([self.state, [2]])
else:
return np.flatnonzero(self.state)[0] + 3
if __name__ == "__main__":
args = parser.parse_args()
rllib/rollout.py
+1 -1
View File
@@ -33,7 +33,7 @@ Example Usage via executable:
# Note: if you use any custom models or envs, register them here first, e.g.:
#
# ModelCatalog.register_custom_model("pa_model", ParametricActionsModel)
# register_env("pa_cartpole", lambda _: ParametricActionCartpole(10))
# register_env("pa_cartpole", lambda _: ParametricActionsCartPole(10))
class RolloutSaver:
rllib/tests/test_env_with_subprocess.py
+29 -48
View File
@@ -1,8 +1,4 @@
"""Tests that envs clean up after themselves on agent exit."""
from gym.spaces import Discrete
import atexit
import gym
import os
import subprocess
import tempfile
@@ -11,58 +7,35 @@ import time
import ray
from ray.tune import run_experiments
from ray.tune.registry import register_env
# Dummy command to run as a subprocess with a unique name
UNIQUE_CMD = "sleep {}".format(str(time.time()))
_, UNIQUE_FILE_0 = tempfile.mkstemp("test_env_with_subprocess")
_, UNIQUE_FILE_1 = tempfile.mkstemp("test_env_with_subprocess")
_, UNIQUE_FILE_2 = tempfile.mkstemp("test_env_with_subprocess")
_, UNIQUE_FILE_3 = tempfile.mkstemp("test_env_with_subprocess")
class EnvWithSubprocess(gym.Env):
"""Our env that spawns a subprocess."""
def __init__(self, config):
self.action_space = Discrete(2)
self.observation_space = Discrete(2)
# Subprocess that should be cleaned up
self.subproc = subprocess.Popen(UNIQUE_CMD.split(" "), shell=False)
self.config = config
# Exit handler should be called
atexit.register(lambda: self.subproc.kill())
if config.worker_index == 0:
atexit.register(lambda: os.unlink(UNIQUE_FILE_0))
else:
atexit.register(lambda: os.unlink(UNIQUE_FILE_1))
def close(self):
if self.config.worker_index == 0:
os.unlink(UNIQUE_FILE_2)
else:
os.unlink(UNIQUE_FILE_3)
def reset(self):
return 0
def step(self, action):
return 0, 0, True, {}
from ray.rllib.examples.env.env_with_subprocess import EnvWithSubprocess
def leaked_processes():
"""Returns whether any subprocesses were leaked."""
result = subprocess.check_output(
"ps aux | grep '{}' | grep -v grep || true".format(UNIQUE_CMD),
"ps aux | grep '{}' | grep -v grep || true".format(
EnvWithSubprocess.UNIQUE_CMD),
shell=True)
return result
if __name__ == "__main__":
register_env("subproc", lambda config: EnvWithSubprocess(config))
# Create 4 temp files, which the Env has to clean up after it's done.
_, tmp1 = tempfile.mkstemp("test_env_with_subprocess")
_, tmp2 = tempfile.mkstemp("test_env_with_subprocess")
_, tmp3 = tempfile.mkstemp("test_env_with_subprocess")
_, tmp4 = tempfile.mkstemp("test_env_with_subprocess")
register_env("subproc", lambda c: EnvWithSubprocess(c))
ray.init()
assert os.path.exists(UNIQUE_FILE_0)
assert os.path.exists(UNIQUE_FILE_1)
# Check whether everything is ok.
assert os.path.exists(tmp1)
assert os.path.exists(tmp2)
assert os.path.exists(tmp3)
assert os.path.exists(tmp4)
assert not leaked_processes()
run_experiments({
"demo": {
"run": "PG",
@@ -70,6 +43,12 @@ if __name__ == "__main__":
"num_samples": 1,
"config": {
"num_workers": 1,
"env_config": {
"tmp_file1": tmp1,
"tmp_file2": tmp2,
"tmp_file3": tmp3,
"tmp_file4": tmp4,
},
},
"stop": {
"training_iteration": 1
@@ -77,10 +56,12 @@ if __name__ == "__main__":
},
})
time.sleep(10.0)
# Check whether processes are still running or Env has not cleaned up
# the given tmp files.
leaked = leaked_processes()
assert not leaked, "LEAKED PROCESSES: {}".format(leaked)
assert not os.path.exists(UNIQUE_FILE_0), "atexit handler not called"
assert not os.path.exists(UNIQUE_FILE_1), "atexit handler not called"
assert not os.path.exists(UNIQUE_FILE_2), "close not called"
assert not os.path.exists(UNIQUE_FILE_3), "close not called"
assert not os.path.exists(tmp1), "atexit handler not called"
assert not os.path.exists(tmp2), "atexit handler not called"
assert not os.path.exists(tmp3), "close not called"
assert not os.path.exists(tmp4), "close not called"
print("OK")
rllib/tests/test_external_multi_agent_env.py
+6 -5
View File
@@ -5,13 +5,14 @@ import unittest
import ray
from ray.rllib.agents.pg.pg_tf_policy import PGTFPolicy
from ray.rllib.optimizers import SyncSamplesOptimizer
from ray.rllib.env.external_multi_agent_env import ExternalMultiAgentEnv
from ray.rllib.evaluation.rollout_worker import RolloutWorker
from ray.rllib.evaluation.worker_set import WorkerSet
from ray.rllib.env.external_multi_agent_env import ExternalMultiAgentEnv
from ray.rllib.examples.env.multi_agent import BasicMultiAgent, \
MultiAgentCartPole
from ray.rllib.optimizers import SyncSamplesOptimizer
from ray.rllib.tests.test_rollout_worker import MockPolicy
from ray.rllib.tests.test_external_env import make_simple_serving
from ray.rllib.tests.test_multi_agent_env import BasicMultiAgent, MultiCartpole
from ray.rllib.evaluation.metrics import collect_metrics
SimpleMultiServing = make_simple_serving(True, ExternalMultiAgentEnv)
@@ -63,7 +64,7 @@ class TestExternalMultiAgentEnv(unittest.TestCase):
batch = ev.sample()
self.assertEqual(batch.count, 50)
def test_train_external_multi_cartpole_many_policies(self):
def test_train_external_multi_agent_cartpole_many_policies(self):
n = 20
single_env = gym.make("CartPole-v0")
act_space = single_env.action_space
@@ -74,7 +75,7 @@ class TestExternalMultiAgentEnv(unittest.TestCase):
{})
policy_ids = list(policies.keys())
ev = RolloutWorker(
env_creator=lambda _: MultiCartpole(n),
env_creator=lambda _: MultiAgentCartPole({"num_agents": n}),
policy=policies,
policy_mapping_fn=lambda agent_id: random.choice(policy_ids),
rollout_fragment_length=100)
rllib/tests/test_io.py
+6 -5
View File
@@ -10,13 +10,13 @@ import time
import unittest
import ray
from ray.tune.registry import register_env
from ray.rllib.agents.pg import PGTrainer
from ray.rllib.agents.pg.pg_tf_policy import PGTFPolicy
from ray.rllib.examples.env.multi_agent import MultiAgentCartPole
from ray.rllib.offline import IOContext, JsonWriter, JsonReader
from ray.rllib.offline.json_writer import _to_json
from ray.rllib.policy.sample_batch import SampleBatch
from ray.rllib.tests.test_multi_agent_env import MultiCartpole
from ray.tune.registry import register_env
SAMPLES = SampleBatch({
"actions": np.array([1, 2, 3, 4]),
@@ -149,7 +149,8 @@ class AgentIOTest(unittest.TestCase):
self.assertTrue(not np.isnan(result["episode_reward_mean"]))
def testMultiAgent(self):
register_env("multi_cartpole", lambda _: MultiCartpole(10))
register_env("multi_agent_cartpole",
lambda _: MultiAgentCartPole({"num_agents": 10}))
single_env = gym.make("CartPole-v0")
def gen_policy():
@@ -158,7 +159,7 @@ class AgentIOTest(unittest.TestCase):
return (PGTFPolicy, obs_space, act_space, {})
pg = PGTrainer(
env="multi_cartpole",
env="multi_agent_cartpole",
config={
"num_workers": 0,
"output": self.test_dir,
@@ -177,7 +178,7 @@ class AgentIOTest(unittest.TestCase):
pg.stop()
pg = PGTrainer(
env="multi_cartpole",
env="multi_agent_cartpole",
config={
"num_workers": 0,
"input": self.test_dir,
rllib/tests/test_multi_agent_env.py
+18 -169
View File
@@ -3,19 +3,20 @@ import random
import unittest
import ray
from ray.tune.registry import register_env
from ray.rllib.agents.pg import PGTrainer
from ray.rllib.agents.pg.pg_tf_policy import PGTFPolicy
from ray.rllib.agents.dqn.dqn_tf_policy import DQNTFPolicy
from ray.rllib.examples.env.multi_agent import MultiAgentCartPole, \
BasicMultiAgent, EarlyDoneMultiAgent, RoundRobinMultiAgent
from ray.rllib.optimizers import (SyncSamplesOptimizer, SyncReplayOptimizer,
AsyncGradientsOptimizer)
from ray.rllib.tests.test_rollout_worker import (MockEnv, MockEnv2, MockPolicy)
from ray.rllib.tests.test_rollout_worker import MockPolicy
from ray.rllib.evaluation.rollout_worker import RolloutWorker
from ray.rllib.policy.tests.test_policy import TestPolicy
from ray.rllib.evaluation.metrics import collect_metrics
from ray.rllib.evaluation.worker_set import WorkerSet
from ray.rllib.env.base_env import _MultiAgentEnvToBaseEnv
from ray.rllib.env.multi_agent_env import MultiAgentEnv
from ray.tune.registry import register_env
def one_hot(i, n):
@@ -24,161 +25,6 @@ def one_hot(i, n):
return out
class BasicMultiAgent(MultiAgentEnv):
"""Env of N independent agents, each of which exits after 25 steps."""
def __init__(self, num):
self.agents = [MockEnv(25) for _ in range(num)]
self.dones = set()
self.observation_space = gym.spaces.Discrete(2)
self.action_space = gym.spaces.Discrete(2)
self.resetted = False
def reset(self):
self.resetted = True
self.dones = set()
return {i: a.reset() for i, a in enumerate(self.agents)}
def step(self, action_dict):
obs, rew, done, info = {}, {}, {}, {}
for i, action in action_dict.items():
obs[i], rew[i], done[i], info[i] = self.agents[i].step(action)
if done[i]:
self.dones.add(i)
done["__all__"] = len(self.dones) == len(self.agents)
return obs, rew, done, info
class EarlyDoneMultiAgent(MultiAgentEnv):
"""Env for testing when the env terminates (after agent 0 does)."""
def __init__(self):
self.agents = [MockEnv(3), MockEnv(5)]
self.dones = set()
self.last_obs = {}
self.last_rew = {}
self.last_done = {}
self.last_info = {}
self.i = 0
self.observation_space = gym.spaces.Discrete(10)
self.action_space = gym.spaces.Discrete(2)
def reset(self):
self.dones = set()
self.last_obs = {}
self.last_rew = {}
self.last_done = {}
self.last_info = {}
self.i = 0
for i, a in enumerate(self.agents):
self.last_obs[i] = a.reset()
self.last_rew[i] = None
self.last_done[i] = False
self.last_info[i] = {}
obs_dict = {self.i: self.last_obs[self.i]}
self.i = (self.i + 1) % len(self.agents)
return obs_dict
def step(self, action_dict):
assert len(self.dones) != len(self.agents)
for i, action in action_dict.items():
(self.last_obs[i], self.last_rew[i], self.last_done[i],
self.last_info[i]) = self.agents[i].step(action)
obs = {self.i: self.last_obs[self.i]}
rew = {self.i: self.last_rew[self.i]}
done = {self.i: self.last_done[self.i]}
info = {self.i: self.last_info[self.i]}
if done[self.i]:
rew[self.i] = 0
self.dones.add(self.i)
self.i = (self.i + 1) % len(self.agents)
done["__all__"] = len(self.dones) == len(self.agents) - 1
return obs, rew, done, info
class RoundRobinMultiAgent(MultiAgentEnv):
"""Env of N independent agents, each of which exits after 5 steps.
On each step() of the env, only one agent takes an action."""
def __init__(self, num, increment_obs=False):
if increment_obs:
# Observations are 0, 1, 2, 3... etc. as time advances
self.agents = [MockEnv2(5) for _ in range(num)]
else:
# Observations are all zeros
self.agents = [MockEnv(5) for _ in range(num)]
self.dones = set()
self.last_obs = {}
self.last_rew = {}
self.last_done = {}
self.last_info = {}
self.i = 0
self.num = num
self.observation_space = gym.spaces.Discrete(10)
self.action_space = gym.spaces.Discrete(2)
def reset(self):
self.dones = set()
self.last_obs = {}
self.last_rew = {}
self.last_done = {}
self.last_info = {}
self.i = 0
for i, a in enumerate(self.agents):
self.last_obs[i] = a.reset()
self.last_rew[i] = None
self.last_done[i] = False
self.last_info[i] = {}
obs_dict = {self.i: self.last_obs[self.i]}
self.i = (self.i + 1) % self.num
return obs_dict
def step(self, action_dict):
assert len(self.dones) != len(self.agents)
for i, action in action_dict.items():
(self.last_obs[i], self.last_rew[i], self.last_done[i],
self.last_info[i]) = self.agents[i].step(action)
obs = {self.i: self.last_obs[self.i]}
rew = {self.i: self.last_rew[self.i]}
done = {self.i: self.last_done[self.i]}
info = {self.i: self.last_info[self.i]}
if done[self.i]:
rew[self.i] = 0
self.dones.add(self.i)
self.i = (self.i + 1) % self.num
done["__all__"] = len(self.dones) == len(self.agents)
return obs, rew, done, info
def make_multiagent(env_name):
class MultiEnv(MultiAgentEnv):
def __init__(self, num):
self.agents = [gym.make(env_name) for _ in range(num)]
self.dones = set()
self.observation_space = self.agents[0].observation_space
self.action_space = self.agents[0].action_space
def reset(self):
self.dones = set()
return {i: a.reset() for i, a in enumerate(self.agents)}
def step(self, action_dict):
obs, rew, done, info = {}, {}, {}, {}
for i, action in action_dict.items():
obs[i], rew[i], done[i], info[i] = self.agents[i].step(action)
if done[i]:
self.dones.add(i)
done["__all__"] = len(self.dones) == len(self.agents)
return obs, rew, done, info
return MultiEnv
MultiCartpole = make_multiagent("CartPole-v0")
MultiMountainCar = make_multiagent("MountainCarContinuous-v0")
class TestMultiAgentEnv(unittest.TestCase):
def setUp(self) -> None:
ray.init(num_cpus=4)
@@ -512,7 +358,7 @@ class TestMultiAgentEnv(unittest.TestCase):
obs_space = single_env.observation_space
act_space = single_env.action_space
ev = RolloutWorker(
env_creator=lambda _: MultiCartpole(2),
env_creator=lambda _: MultiAgentCartPole({"num_agents": 2}),
policy={
"p0": (ModelBasedPolicy, obs_space, act_space, {}),
"p1": (ModelBasedPolicy, obs_space, act_space, {}),
@@ -524,10 +370,11 @@ class TestMultiAgentEnv(unittest.TestCase):
self.assertEqual(batch.policy_batches["p0"].count, 10)
self.assertEqual(batch.policy_batches["p1"].count, 25)
def test_train_multi_cartpole_single_policy(self):
def test_train_multi_agent_cartpole_single_policy(self):
n = 10
register_env("multi_cartpole", lambda _: MultiCartpole(n))
pg = PGTrainer(env="multi_cartpole", config={"num_workers": 0})
register_env("multi_agent_cartpole",
lambda _: MultiAgentCartPole({"num_agents": n}))
pg = PGTrainer(env="multi_agent_cartpole", config={"num_workers": 0})
for i in range(100):
result = pg.train()
print("Iteration {}, reward {}, timesteps {}".format(
@@ -536,9 +383,10 @@ class TestMultiAgentEnv(unittest.TestCase):
return
raise Exception("failed to improve reward")
def test_train_multi_cartpole_multi_policy(self):
def test_train_multi_agent_cartpole_multi_policy(self):
n = 10
register_env("multi_cartpole", lambda _: MultiCartpole(n))
register_env("multi_agent_cartpole",
lambda _: MultiAgentCartPole({"num_agents": n}))
single_env = gym.make("CartPole-v0")
def gen_policy():
@@ -551,7 +399,7 @@ class TestMultiAgentEnv(unittest.TestCase):
return (None, obs_space, act_space, config)
pg = PGTrainer(
env="multi_cartpole",
env="multi_agent_cartpole",
config={
"num_workers": 0,
"multiagent": {
@@ -596,7 +444,7 @@ class TestMultiAgentEnv(unittest.TestCase):
"p2": (DQNTFPolicy, obs_space, act_space, dqn_config),
}
worker = RolloutWorker(
env_creator=lambda _: MultiCartpole(n),
env_creator=lambda _: MultiAgentCartPole({"num_agents": n}),
policy=policies,
policy_mapping_fn=lambda agent_id: ["p1", "p2"][agent_id % 2],
rollout_fragment_length=50)
@@ -607,7 +455,8 @@ class TestMultiAgentEnv(unittest.TestCase):
remote_workers = [
RolloutWorker.as_remote().remote(
env_creator=lambda _: MultiCartpole(n),
env_creator=lambda _: MultiAgentCartPole(
{"num_agents": n}),
policy=policies,
policy_mapping_fn=policy_mapper,
rollout_fragment_length=50)
@@ -645,7 +494,7 @@ class TestMultiAgentEnv(unittest.TestCase):
def test_multi_agent_replay_optimizer(self):
self._test_with_optimizer(SyncReplayOptimizer)
def test_train_multi_cartpole_many_policies(self):
def test_train_multi_agent_cartpole_many_policies(self):
n = 20
env = gym.make("CartPole-v0")
act_space = env.action_space
@@ -656,7 +505,7 @@ class TestMultiAgentEnv(unittest.TestCase):
{})
policy_ids = list(policies.keys())
worker = RolloutWorker(
env_creator=lambda _: MultiCartpole(n),
env_creator=lambda _: MultiAgentCartPole({"num_agents": n}),
policy=policies,
policy_mapping_fn=lambda agent_id: random.choice(policy_ids),
rollout_fragment_length=100)
rllib/tests/test_multi_agent_pendulum.py
+4 -4
View File
@@ -2,7 +2,7 @@
import unittest
import ray
from ray.rllib.tests.test_multi_agent_env import make_multiagent
from ray.rllib.examples.env.multi_agent import MultiAgentPendulum
from ray.tune import run_experiments
from ray.tune.registry import register_env
@@ -15,12 +15,12 @@ class TestMultiAgentPendulum(unittest.TestCase):
ray.shutdown()
def test_multi_agent_pendulum(self):
MultiPendulum = make_multiagent("Pendulum-v0")
register_env("multi_pend", lambda _: MultiPendulum(1))
register_env("multi_agent_pendulum",
lambda _: MultiAgentPendulum({"num_agents": 1}))
trials = run_experiments({
"test": {
"run": "PPO",
"env": "multi_pend",
"env": "multi_agent_pendulum",
"stop": {
"timesteps_total": 500000,
"episode_reward_mean": -200,
rllib/tests/test_supported_spaces.py
+18 -35
View File
@@ -1,6 +1,4 @@
import gym
from gym.spaces import Box, Dict, Discrete, Tuple, MultiDiscrete
from gym.envs.registration import EnvSpec
import numpy as np
import unittest
import traceback
@@ -8,12 +6,13 @@ import traceback
import ray
from ray.rllib.utils.framework import try_import_tf
from ray.rllib.agents.registry import get_agent_class
from ray.rllib.examples.env.multi_agent import MultiAgentCartPole, \
MultiAgentMountainCar
from ray.rllib.examples.env.random_env import RandomEnv
from ray.rllib.models.tf.fcnet_v2 import FullyConnectedNetwork as FCNetV2
from ray.rllib.models.tf.visionnet_v2 import VisionNetwork as VisionNetV2
from ray.rllib.models.torch.visionnet import VisionNetwork as TorchVisionNetV2
from ray.rllib.models.torch.fcnet import FullyConnectedNetwork as TorchFCNetV2
from ray.rllib.tests.test_multi_agent_env import MultiCartpole, \
MultiMountainCar
from ray.rllib.utils.error import UnsupportedSpaceException
from ray.tune.registry import register_env
@@ -55,30 +54,6 @@ OBSERVATION_SPACES_TO_TEST = {
}
def make_stub_env(action_space, obs_space, check_action_bounds):
class StubEnv(gym.Env):
def __init__(self):
self.action_space = action_space
self.observation_space = obs_space
self.spec = EnvSpec("StubEnv-v0")
def reset(self):
sample = self.observation_space.sample()
return sample
def step(self, action):
if check_action_bounds and not self.action_space.contains(action):
raise ValueError("Illegal action for {}: {}".format(
self.action_space, action))
if (isinstance(self.action_space, Tuple)
and len(action) != len(self.action_space.spaces)):
raise ValueError("Illegal action for {}: {}".format(
self.action_space, action))
return self.observation_space.sample(), 1, True, {}
return StubEnv
def check_support(alg, config, stats, check_bounds=False, name=None):
covered_a = set()
covered_o = set()
@@ -89,15 +64,21 @@ def check_support(alg, config, stats, check_bounds=False, name=None):
for o_name, obs_space in OBSERVATION_SPACES_TO_TEST.items():
print("=== Testing {} (torch={}) A={} S={} ===".format(
alg, torch, action_space, obs_space))
stub_env = make_stub_env(action_space, obs_space, check_bounds)
register_env("stub_env", lambda c: stub_env())
config.update(
dict(
env_config=dict(
action_space=action_space,
observation_space=obs_space,
reward_space=Box(1.0, 1.0, shape=(), dtype=np.float32),
p_done=1.0,
check_action_bounds=check_bounds)))
stat = "ok"
a = None
try:
if a_name in covered_a and o_name in covered_o:
stat = "skip" # speed up tests by avoiding full grid
else:
a = get_agent_class(alg)(config=config, env="stub_env")
a = get_agent_class(alg)(config=config, env=RandomEnv)
if alg not in ["DDPG", "ES", "ARS", "SAC"]:
if o_name in ["atari", "image"]:
if torch:
@@ -140,13 +121,15 @@ def check_support(alg, config, stats, check_bounds=False, name=None):
def check_support_multiagent(alg, config):
register_env("multi_mountaincar", lambda _: MultiMountainCar(2))
register_env("multi_cartpole", lambda _: MultiCartpole(2))
register_env("multi_agent_mountaincar",
lambda _: MultiAgentMountainCar({"num_agents": 2}))
register_env("multi_agent_cartpole",
lambda _: MultiAgentCartPole({"num_agents": 2}))
config["log_level"] = "ERROR"
if "DDPG" in alg:
a = get_agent_class(alg)(config=config, env="multi_mountaincar")
a = get_agent_class(alg)(config=config, env="multi_agent_mountaincar")
else:
a = get_agent_class(alg)(config=config, env="multi_cartpole")
a = get_agent_class(alg)(config=config, env="multi_agent_cartpole")
try:
a.train()
finally: