[RLlib] Unity3D integration (n Unity3D clients vs learning server). (#8590)

rllib/examples/attention_net.py

@@ -28,7 +28,7 @@ if __name__ == "__main__":
 
     assert not args.torch, "PyTorch not supported for AttentionNets yet!"
 
-    ray.init(num_cpus=args.num_cpus or None, local_mode=True)
+    ray.init(num_cpus=args.num_cpus or None)
 
     registry.register_env("RepeatAfterMeEnv", lambda c: RepeatAfterMeEnv(c))
     registry.register_env("RepeatInitialObsEnv",

rllib/examples/env/multi_agent.py

@@ -4,12 +4,16 @@ from ray.rllib.env.multi_agent_env import MultiAgentEnv
 from ray.rllib.tests.test_rollout_worker import MockEnv, MockEnv2
 
 
-def make_multiagent(env_name):
+def make_multiagent(env_name_or_creator):
     class MultiEnv(MultiAgentEnv):
         def __init__(self, config):
-            self.agents = [
-                gym.make(env_name) for _ in range(config["num_agents"])
-            ]
+            num = config.pop("num_agents", 1)
+            if isinstance(env_name_or_creator, str):
+                self.agents = [
+                    gym.make(env_name_or_creator) for _ in range(num)
+                ]
+            else:
+                self.agents = [env_name_or_creator(config) for _ in range(num)]
             self.dones = set()
             self.observation_space = self.agents[0].observation_space
             self.action_space = self.agents[0].action_space

rllib/examples/env/random_env.py

@@ -1,7 +1,9 @@
 import gym
-from gym.spaces import Tuple
+from gym.spaces import Discrete, Tuple
 import numpy as np
 
+from ray.rllib.examples.env.multi_agent import make_multiagent
+
 
 class RandomEnv(gym.Env):
     """A randomly acting environment.
@@ -14,9 +16,9 @@ class RandomEnv(gym.Env):
 
     def __init__(self, config):
         # Action space.
-        self.action_space = config["action_space"]
+        self.action_space = config.get("action_space", Discrete(2))
         # Observation space from which to sample.
-        self.observation_space = config["observation_space"]
+        self.observation_space = config.get("observation_space", Discrete(2))
         # Reward space from which to sample.
         self.reward_space = config.get(
             "reward_space",
@@ -43,3 +45,7 @@ class RandomEnv(gym.Env):
             bool(np.random.choice(
                 [True, False], p=[self.p_done, 1.0 - self.p_done]
             )), {}
+
+
+# Multi-agent version of the RandomEnv.
+RandomMultiAgentEnv = make_multiagent(lambda c: RandomEnv(c))

rllib/examples/models/shared_weights_model.py

@@ -85,7 +85,12 @@ class SharedWeightsModel2(TFModelV2):
 
 TORCH_GLOBAL_SHARED_LAYER = None
 if torch:
-    TORCH_GLOBAL_SHARED_LAYER = SlimFC(32, 32)
+    TORCH_GLOBAL_SHARED_LAYER = SlimFC(
+        64,
+        64,
+        activation_fn=nn.ReLU,
+        initializer=torch.nn.init.xavier_uniform_,
+    )
 
 
 class TorchSharedWeightsModel(TorchModelV2, nn.Module):
@@ -104,12 +109,22 @@ class TorchSharedWeightsModel(TorchModelV2, nn.Module):
         # Non-shared initial layer.
         self.first_layer = SlimFC(
             int(np.product(observation_space.shape)),
-            32,
-            activation_fn=nn.ReLU)
+            64,
+            activation_fn=nn.ReLU,
+            initializer=torch.nn.init.xavier_uniform_)
 
         # Non-shared final layer.
-        self.last_layer = SlimFC(32, self.num_outputs, activation_fn=nn.ReLU)
-        self.vf = SlimFC(32, 1, activation_fn=None)
+        self.last_layer = SlimFC(
+            64,
+            self.num_outputs,
+            activation_fn=None,
+            initializer=torch.nn.init.xavier_uniform_)
+        self.vf = SlimFC(
+            64,
+            1,
+            activation_fn=None,
+            initializer=torch.nn.init.xavier_uniform_,
+        )
         self._output = None
 
     @override(ModelV2)

rllib/examples/multi_agent_cartpole.py

@@ -28,7 +28,7 @@ parser = argparse.ArgumentParser()
 
 parser.add_argument("--num-agents", type=int, default=4)
 parser.add_argument("--num-policies", type=int, default=2)
-parser.add_argument("--stop-iters", type=int, default=20)
+parser.add_argument("--stop-iters", type=int, default=200)
 parser.add_argument("--stop-reward", type=float, default=150)
 parser.add_argument("--stop-timesteps", type=int, default=100000)
 parser.add_argument("--simple", action="store_true")
@@ -74,7 +74,6 @@ if __name__ == "__main__":
         "env_config": {
             "num_agents": args.num_agents,
         },
-        "log_level": "DEBUG",
         "simple_optimizer": args.simple,
         "num_sgd_iter": 10,
         "multiagent": {
@@ -89,7 +88,7 @@ if __name__ == "__main__":
         "training_iteration": args.stop_iters,
     }
 
-    results = tune.run("PPO", stop=stop, config=config)
+    results = tune.run("PPO", stop=stop, config=config, verbose=1)
 
     if args.as_test:
         check_learning_achieved(results, args.stop_reward)

rllib/examples/serving/cartpole_client.py

@@ -23,7 +23,7 @@ parser.add_argument(
     action="store_true",
     help="Whether to take random instead of on-policy actions.")
 parser.add_argument(
-    "--stop-at-reward",
+    "--stop-reward",
     type=int,
     default=9999,
     help="Stop once the specified reward is reached.")
@@ -49,7 +49,7 @@ if __name__ == "__main__":
         client.log_returns(eid, reward, info=info)
         if done:
             print("Total reward:", rewards)
-            if rewards >= args.stop_at_reward:
+            if rewards >= args.stop_reward:
                 print("Target reward achieved, exiting")
                 exit(0)
             rewards = 0

rllib/examples/serving/cartpole_server.py

@@ -32,8 +32,7 @@ if __name__ == "__main__":
     connector_config = {
         # Use the connector server to generate experiences.
         "input": (
-            lambda ioctx: PolicyServerInput( \
-                ioctx, SERVER_ADDRESS, SERVER_PORT)
+            lambda ioctx: PolicyServerInput(ioctx, SERVER_ADDRESS, SERVER_PORT)
         ),
         # Use a single worker process to run the server.
         "num_workers": 0,

rllib/examples/serving/test_local_inference.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-rm -f last_checkpoint.out
-pkill -f cartpole_server.py
-sleep 1
-
-if [ -f cartpole_server.py ]; then
-    basedir="."
-else
-    basedir="rllib/examples/serving"  # In bazel.
-fi
-
-(python $basedir/cartpole_server.py --run=PPO 2>&1 | grep -v 200) &
-pid=$!
-
-echo "Waiting for server to start"
-while ! curl localhost:9900; do   
-  sleep 1
-done
-
-sleep 2
-python $basedir/cartpole_client.py --stop-at-reward=100 --inference-mode=local
-kill $pid

rllib/examples/serving/test_remote_inference.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-rm -f last_checkpoint.out
-pkill -f cartpole_server.py
-sleep 1
-
-if [ -f cartpole_server.py ]; then
-    basedir="."
-else
-    basedir="rllib/examples/serving"  # In bazel.
-fi
-
-(python $basedir/cartpole_server.py --run=DQN 2>&1 | grep -v 200) &
-pid=$!
-
-echo "Waiting for server to start"
-while ! curl localhost:9900; do   
-  sleep 1
-done
-
-sleep 2
-python $basedir/cartpole_client.py --stop-at-reward=100 --inference-mode=remote
-kill $pid
-

rllib/examples/serving/unity3d_client.py

@@ -0,0 +1,120 @@
+"""
+Example of running a Unity3D client instance against an RLlib Policy server.
+Unity3D clients can be run in distributed fashion on n nodes in the cloud
+and all connect to the same RLlib server for faster sample collection.
+For a locally running Unity3D example, see:
+`examples/unity3d_env_local.py`
+
+To run this script on possibly different machines
+against a central Policy server:
+1) Install Unity3D and `pip install mlagents`.
+
+2) Compile a Unity3D example game with MLAgents support (e.g. 3DBall or any
+   other one that you created yourself) and place the compiled binary
+   somewhere, where your RLlib client script (see below) can access it.
+
+2.1) To find Unity3D MLAgent examples, first `pip install mlagents`,
+     then check out the `.../ml-agents/Project/Assets/ML-Agents/Examples/`
+     folder.
+
+3) Change your RLlib Policy server code so it knows the observation- and
+   action Spaces, the different Policies (called "behaviors" in Unity3D
+   MLAgents), and Agent-to-Policy mappings for your particular game.
+   Alternatively, use one of the two already existing setups (3DBall or
+   SoccerStrikersVsGoalie).
+
+4) Then run (two separate shells/machines):
+$ python unity3d_server.py --env 3DBall
+$ python unity3d_client.py --inference-mode=local --game [path to game binary]
+"""
+
+import argparse
+
+from ray.rllib.env.policy_client import PolicyClient
+from ray.rllib.env.unity3d_env import Unity3DEnv
+
+SERVER_ADDRESS = "localhost"
+SERVER_PORT = 9900
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--game",
+    type=str,
+    default=None,
+    help="The game executable to run as RL env. If not provided, uses local "
+    "Unity3D editor instance.")
+parser.add_argument(
+    "--horizon",
+    type=int,
+    default=200,
+    help="The max. number of `step()`s for any episode (per agent) before "
+    "it'll be reset again automatically.")
+parser.add_argument(
+    "--server",
+    type=str,
+    default=SERVER_ADDRESS + ":" + str(SERVER_PORT),
+    help="The Policy server's address and port to connect to from this client."
+)
+parser.add_argument(
+    "--no-train",
+    action="store_true",
+    help="Whether to disable training (on the server side).")
+parser.add_argument(
+    "--inference-mode",
+    type=str,
+    default="local",
+    choices=["local", "remote"],
+    help="Whether to compute actions `local`ly or `remote`ly. Note that "
+    "`local` is much faster b/c observations/actions do not have to be "
+    "sent via the network.")
+parser.add_argument(
+    "--update-interval-local-mode",
+    type=float,
+    default=10.0,
+    help="For `inference-mode=local`, every how many seconds do we update "
+    "learnt policy weights from the server?")
+parser.add_argument(
+    "--stop-reward",
+    type=int,
+    default=9999,
+    help="Stop once the specified reward is reached.")
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    # Start the client for sending environment information (e.g. observations,
+    # actions) to a policy server (listening on port 9900).
+    client = PolicyClient(
+        "http://" + args.server,
+        inference_mode=args.inference_mode,
+        update_interval=args.update_interval_local_mode)
+
+    # Start and reset the actual Unity3DEnv (either already running Unity3D
+    # editor or a binary (game) to be started automatically).
+    env = Unity3DEnv(file_name=args.game, episode_horizon=args.horizon)
+    obs = env.reset()
+    eid = client.start_episode(training_enabled=not args.no_train)
+
+    # Keep track of the total reward per episode.
+    total_rewards_this_episode = 0.0
+
+    # Loop infinitely through the env.
+    while True:
+        # Get actions from the Policy server given our current obs.
+        actions = client.get_action(eid, obs)
+        # Apply actions to our env.
+        obs, rewards, dones, infos = env.step(actions)
+        total_rewards_this_episode += sum(rewards.values())
+        # Log rewards and single-agent dones.
+        client.log_returns(eid, rewards, infos, multiagent_done_dict=dones)
+        # Check whether all agents are done and end the episode, if necessary.
+        if dones["__all__"]:
+            print("Episode done: Reward={}".format(total_rewards_this_episode))
+            if total_rewards_this_episode >= args.stop_reward:
+                quit(0)
+            # End the episode and reset Unity Env.
+            total_rewards_this_episode = 0.0
+            client.end_episode(eid, obs)
+            obs = env.reset()
+            # Start a new episode.
+            eid = client.start_episode(training_enabled=not args.no_train)

rllib/examples/serving/unity3d_server.py

@@ -0,0 +1,129 @@
+"""
+Example of running a Unity3D (MLAgents) Policy server that can learn
+Policies via sampling inside many connected Unity game clients (possibly
+running in the cloud on n nodes).
+For a locally running Unity3D example, see:
+`examples/unity3d_env_local.py`
+
+To run this script against one or more possibly cloud-based clients:
+1) Install Unity3D and `pip install mlagents`.
+
+2) Compile a Unity3D example game with MLAgents support (e.g. 3DBall or any
+   other one that you created yourself) and place the compiled binary
+   somewhere, where your RLlib client script (see below) can access it.
+
+2.1) To find Unity3D MLAgent examples, first `pip install mlagents`,
+     then check out the `.../ml-agents/Project/Assets/ML-Agents/Examples/`
+     folder.
+
+3) Change this RLlib Policy server code so it knows the observation- and
+   action Spaces, the different Policies (called "behaviors" in Unity3D
+   MLAgents), and Agent-to-Policy mappings for your particular game.
+   Alternatively, use one of the two already existing setups (3DBall or
+   SoccerStrikersVsGoalie).
+
+4) Then run (two separate shells/machines):
+$ python unity3d_server.py --env 3DBall
+$ python unity3d_client.py --inference-mode=local --game [path to game binary]
+"""
+
+import argparse
+import os
+
+import ray
+from ray.tune import register_env
+from ray.rllib.agents.ppo import PPOTrainer
+from ray.rllib.env.policy_server_input import PolicyServerInput
+from ray.rllib.examples.env.random_env import RandomMultiAgentEnv
+from ray.rllib.examples.env.unity3d_env import Unity3DEnv
+
+SERVER_ADDRESS = "localhost"
+SERVER_PORT = 9900
+CHECKPOINT_FILE = "last_checkpoint_{}.out"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--env",
+    type=str,
+    default="3DBall",
+    choices=["3DBall", "SoccerStrikersVsGoalie"],
+    help="The name of the Env to run in the Unity3D editor. Either `3DBall` "
+    "or `SoccerStrikersVsGoalie` (feel free to add more to this script!)")
+parser.add_argument(
+    "--port",
+    type=int,
+    default=SERVER_PORT,
+    help="The Policy server's port to listen on for ExternalEnv client "
+    "conections.")
+parser.add_argument(
+    "--checkpoint-freq",
+    type=int,
+    default=10,
+    help="The frequency with which to create checkpoint files of the learnt "
+    "Policies.")
+parser.add_argument(
+    "--no-restore",
+    action="store_true",
+    help="Whether to load the Policy "
+    "weights from a previous checkpoint")
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    ray.init(local_mode=True)
+
+    # Create a fake-env for the server. This env will never be used (neither
+    # for sampling, nor for evaluation) and its obs/action Spaces do not
+    # matter either (multi-agent config below defines Spaces per Policy).
+    register_env("fake_unity", lambda c: RandomMultiAgentEnv(c))
+
+    policies, policy_mapping_fn = \
+        Unity3DEnv.get_policy_configs_for_game(args.env)
+
+    # The entire config will be sent to connecting clients so they can
+    # build their own samplers (and also Policy objects iff
+    # `inference_mode=local` on clients' command line).
+    config = {
+        # Use the connector server to generate experiences.
+        "input": (
+            lambda ioctx: PolicyServerInput(ioctx, SERVER_ADDRESS, args.port)),
+        # Use a single worker process (w/ SyncSampler) to run the server.
+        "num_workers": 0,
+        # Disable OPE, since the rollouts are coming from online clients.
+        "input_evaluation": [],
+
+        # Other settings.
+        "sample_batch_size": 64,
+        "train_batch_size": 256,
+        "rollout_fragment_length": 20,
+        # Multi-agent setup for the particular env.
+        "multiagent": {
+            "policies": policies,
+            "policy_mapping_fn": policy_mapping_fn,
+        },
+        "framework": "tf",
+    }
+
+    # Create the Trainer used for Policy serving.
+    trainer = PPOTrainer(env="fake_unity", config=config)
+
+    # Attempt to restore from checkpoint if possible.
+    checkpoint_path = CHECKPOINT_FILE.format(args.env)
+    if not args.no_restore and os.path.exists(checkpoint_path):
+        checkpoint_path = open(checkpoint_path).read()
+        print("Restoring from checkpoint path", checkpoint_path)
+        trainer.restore(checkpoint_path)
+
+    # Serving and training loop.
+    count = 0
+    while True:
+        # Calls to train() will block on the configured `input` in the Trainer
+        # config above (PolicyServerInput).
+        print(trainer.train())
+        if count % args.checkpoint_freq == 0:
+            print("Saving learning progress to checkpoint file.")
+            checkpoint = trainer.save()
+            # Write the latest checkpoint location to CHECKPOINT_FILE,
+            # so we can pick up from the latest one after a server re-start.
+            with open(checkpoint_path, "w") as f:
+                f.write(checkpoint)
+        count += 1

rllib/examples/unity3d_env_local.py

@@ -0,0 +1,95 @@
+"""
+Example of running an RLlib Trainer against a locally running Unity3D editor
+instance (available as Unity3DEnv inside RLlib).
+For a distributed cloud setup example with Unity,
+see `examples/serving/unity3d_[server|client].py`
+
+To run this script against a local Unity3D engine:
+1) Install Unity3D and `pip install mlagents`.
+
+2) Open the Unity3D Editor and load an example scene from the following
+   ml-agents pip package location:
+   `.../ml-agents/Project/Assets/ML-Agents/Examples/`
+   This script supports the `3DBall` and `SoccerStrikersVsGoalie` examples.
+   Specify the game you chose on your command line via e.g. `--env 3DBall`.
+   Feel free to add more supported examples here.
+
+3) Then run this script (you will have to press Play in your Unity editor
+   at some point to start the game and the learning process):
+$ python unity3d_env_local.py --env 3DBall --stop-reward [..] [--torch]?
+"""
+
+import argparse
+
+import ray
+from ray import tune
+from ray.rllib.env.unity3d_env import Unity3DEnv
+from ray.rllib.utils.test_utils import check_learning_achieved
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--env",
+    type=str,
+    default="3DBall",
+    choices=["3DBall", "SoccerStrikersVsGoalie"],
+    help="The name of the Env to run in the Unity3D editor. Either `3DBall` "
+    "or `SoccerStrikersVsGoalie` (feel free to add more to this script!)")
+parser.add_argument("--as-test", action="store_true")
+parser.add_argument("--stop-iters", type=int, default=150)
+parser.add_argument("--stop-reward", type=float, default=9999.0)
+parser.add_argument("--stop-timesteps", type=int, default=100000)
+parser.add_argument(
+    "--horizon",
+    type=int,
+    default=200,
+    help="The max. number of `step()`s for any episode (per agent) before "
+    "it'll be reset again automatically.")
+parser.add_argument("--torch", action="store_true")
+
+if __name__ == "__main__":
+    ray.init(local_mode=True)
+
+    args = parser.parse_args()
+
+    tune.register_env(
+        "unity3d",
+        lambda c: Unity3DEnv(episode_horizon=c.get("episode_horizon", 1000)))
+
+    # Get policies (different agent types; "behaviors" in MLAgents) and
+    # the mappings from individual agents to Policies.
+    policies, policy_mapping_fn = \
+        Unity3DEnv.get_policy_configs_for_game(args.env)
+
+    config = {
+        "env": "unity3d",
+        "env_config": {
+            "episode_horizon": args.horizon,
+        },
+        # IMPORTANT: Just use one Worker (we only have one Unity running)!
+        "num_workers": 0,
+        # Other settings.
+        "sample_batch_size": 64,
+        "train_batch_size": 256,
+        "rollout_fragment_length": 20,
+        # Multi-agent setup for the particular env.
+        "multiagent": {
+            "policies": policies,
+            "policy_mapping_fn": policy_mapping_fn,
+        },
+        "framework": "tf",
+    }
+
+    stop = {
+        "training_iteration": args.stop_iters,
+        "timesteps_total": args.stop_timesteps,
+        "episode_reward_mean": args.stop_reward,
+    }
+
+    # Run the experiment.
+    results = tune.run("PPO", config=config, stop=stop, verbose=1)
+
+    # And check the results.
+    if args.as_test:
+        check_learning_achieved(results, args.stop_reward)
+
+    ray.shutdown()