[RLlib] Unity3D integration (n Unity3D clients vs learning server). (#8590)

rllib/examples/serving/cartpole_client.py

@@ -23,7 +23,7 @@ parser.add_argument(
     action="store_true",
     help="Whether to take random instead of on-policy actions.")
 parser.add_argument(
-    "--stop-at-reward",
+    "--stop-reward",
     type=int,
     default=9999,
     help="Stop once the specified reward is reached.")
@@ -49,7 +49,7 @@ if __name__ == "__main__":
         client.log_returns(eid, reward, info=info)
         if done:
             print("Total reward:", rewards)
-            if rewards >= args.stop_at_reward:
+            if rewards >= args.stop_reward:
                 print("Target reward achieved, exiting")
                 exit(0)
             rewards = 0

rllib/examples/serving/cartpole_server.py

@@ -32,8 +32,7 @@ if __name__ == "__main__":
     connector_config = {
         # Use the connector server to generate experiences.
         "input": (
-            lambda ioctx: PolicyServerInput( \
-                ioctx, SERVER_ADDRESS, SERVER_PORT)
+            lambda ioctx: PolicyServerInput(ioctx, SERVER_ADDRESS, SERVER_PORT)
         ),
         # Use a single worker process to run the server.
         "num_workers": 0,

rllib/examples/serving/test_local_inference.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-rm -f last_checkpoint.out
-pkill -f cartpole_server.py
-sleep 1
-
-if [ -f cartpole_server.py ]; then
-    basedir="."
-else
-    basedir="rllib/examples/serving"  # In bazel.
-fi
-
-(python $basedir/cartpole_server.py --run=PPO 2>&1 | grep -v 200) &
-pid=$!
-
-echo "Waiting for server to start"
-while ! curl localhost:9900; do   
-  sleep 1
-done
-
-sleep 2
-python $basedir/cartpole_client.py --stop-at-reward=100 --inference-mode=local
-kill $pid

rllib/examples/serving/test_remote_inference.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-rm -f last_checkpoint.out
-pkill -f cartpole_server.py
-sleep 1
-
-if [ -f cartpole_server.py ]; then
-    basedir="."
-else
-    basedir="rllib/examples/serving"  # In bazel.
-fi
-
-(python $basedir/cartpole_server.py --run=DQN 2>&1 | grep -v 200) &
-pid=$!
-
-echo "Waiting for server to start"
-while ! curl localhost:9900; do   
-  sleep 1
-done
-
-sleep 2
-python $basedir/cartpole_client.py --stop-at-reward=100 --inference-mode=remote
-kill $pid
-

rllib/examples/serving/unity3d_client.py

@@ -0,0 +1,120 @@
+"""
+Example of running a Unity3D client instance against an RLlib Policy server.
+Unity3D clients can be run in distributed fashion on n nodes in the cloud
+and all connect to the same RLlib server for faster sample collection.
+For a locally running Unity3D example, see:
+`examples/unity3d_env_local.py`
+
+To run this script on possibly different machines
+against a central Policy server:
+1) Install Unity3D and `pip install mlagents`.
+
+2) Compile a Unity3D example game with MLAgents support (e.g. 3DBall or any
+   other one that you created yourself) and place the compiled binary
+   somewhere, where your RLlib client script (see below) can access it.
+
+2.1) To find Unity3D MLAgent examples, first `pip install mlagents`,
+     then check out the `.../ml-agents/Project/Assets/ML-Agents/Examples/`
+     folder.
+
+3) Change your RLlib Policy server code so it knows the observation- and
+   action Spaces, the different Policies (called "behaviors" in Unity3D
+   MLAgents), and Agent-to-Policy mappings for your particular game.
+   Alternatively, use one of the two already existing setups (3DBall or
+   SoccerStrikersVsGoalie).
+
+4) Then run (two separate shells/machines):
+$ python unity3d_server.py --env 3DBall
+$ python unity3d_client.py --inference-mode=local --game [path to game binary]
+"""
+
+import argparse
+
+from ray.rllib.env.policy_client import PolicyClient
+from ray.rllib.env.unity3d_env import Unity3DEnv
+
+SERVER_ADDRESS = "localhost"
+SERVER_PORT = 9900
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--game",
+    type=str,
+    default=None,
+    help="The game executable to run as RL env. If not provided, uses local "
+    "Unity3D editor instance.")
+parser.add_argument(
+    "--horizon",
+    type=int,
+    default=200,
+    help="The max. number of `step()`s for any episode (per agent) before "
+    "it'll be reset again automatically.")
+parser.add_argument(
+    "--server",
+    type=str,
+    default=SERVER_ADDRESS + ":" + str(SERVER_PORT),
+    help="The Policy server's address and port to connect to from this client."
+)
+parser.add_argument(
+    "--no-train",
+    action="store_true",
+    help="Whether to disable training (on the server side).")
+parser.add_argument(
+    "--inference-mode",
+    type=str,
+    default="local",
+    choices=["local", "remote"],
+    help="Whether to compute actions `local`ly or `remote`ly. Note that "
+    "`local` is much faster b/c observations/actions do not have to be "
+    "sent via the network.")
+parser.add_argument(
+    "--update-interval-local-mode",
+    type=float,
+    default=10.0,
+    help="For `inference-mode=local`, every how many seconds do we update "
+    "learnt policy weights from the server?")
+parser.add_argument(
+    "--stop-reward",
+    type=int,
+    default=9999,
+    help="Stop once the specified reward is reached.")
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    # Start the client for sending environment information (e.g. observations,
+    # actions) to a policy server (listening on port 9900).
+    client = PolicyClient(
+        "http://" + args.server,
+        inference_mode=args.inference_mode,
+        update_interval=args.update_interval_local_mode)
+
+    # Start and reset the actual Unity3DEnv (either already running Unity3D
+    # editor or a binary (game) to be started automatically).
+    env = Unity3DEnv(file_name=args.game, episode_horizon=args.horizon)
+    obs = env.reset()
+    eid = client.start_episode(training_enabled=not args.no_train)
+
+    # Keep track of the total reward per episode.
+    total_rewards_this_episode = 0.0
+
+    # Loop infinitely through the env.
+    while True:
+        # Get actions from the Policy server given our current obs.
+        actions = client.get_action(eid, obs)
+        # Apply actions to our env.
+        obs, rewards, dones, infos = env.step(actions)
+        total_rewards_this_episode += sum(rewards.values())
+        # Log rewards and single-agent dones.
+        client.log_returns(eid, rewards, infos, multiagent_done_dict=dones)
+        # Check whether all agents are done and end the episode, if necessary.
+        if dones["__all__"]:
+            print("Episode done: Reward={}".format(total_rewards_this_episode))
+            if total_rewards_this_episode >= args.stop_reward:
+                quit(0)
+            # End the episode and reset Unity Env.
+            total_rewards_this_episode = 0.0
+            client.end_episode(eid, obs)
+            obs = env.reset()
+            # Start a new episode.
+            eid = client.start_episode(training_enabled=not args.no_train)

rllib/examples/serving/unity3d_server.py

@@ -0,0 +1,129 @@
+"""
+Example of running a Unity3D (MLAgents) Policy server that can learn
+Policies via sampling inside many connected Unity game clients (possibly
+running in the cloud on n nodes).
+For a locally running Unity3D example, see:
+`examples/unity3d_env_local.py`
+
+To run this script against one or more possibly cloud-based clients:
+1) Install Unity3D and `pip install mlagents`.
+
+2) Compile a Unity3D example game with MLAgents support (e.g. 3DBall or any
+   other one that you created yourself) and place the compiled binary
+   somewhere, where your RLlib client script (see below) can access it.
+
+2.1) To find Unity3D MLAgent examples, first `pip install mlagents`,
+     then check out the `.../ml-agents/Project/Assets/ML-Agents/Examples/`
+     folder.
+
+3) Change this RLlib Policy server code so it knows the observation- and
+   action Spaces, the different Policies (called "behaviors" in Unity3D
+   MLAgents), and Agent-to-Policy mappings for your particular game.
+   Alternatively, use one of the two already existing setups (3DBall or
+   SoccerStrikersVsGoalie).
+
+4) Then run (two separate shells/machines):
+$ python unity3d_server.py --env 3DBall
+$ python unity3d_client.py --inference-mode=local --game [path to game binary]
+"""
+
+import argparse
+import os
+
+import ray
+from ray.tune import register_env
+from ray.rllib.agents.ppo import PPOTrainer
+from ray.rllib.env.policy_server_input import PolicyServerInput
+from ray.rllib.examples.env.random_env import RandomMultiAgentEnv
+from ray.rllib.examples.env.unity3d_env import Unity3DEnv
+
+SERVER_ADDRESS = "localhost"
+SERVER_PORT = 9900
+CHECKPOINT_FILE = "last_checkpoint_{}.out"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--env",
+    type=str,
+    default="3DBall",
+    choices=["3DBall", "SoccerStrikersVsGoalie"],
+    help="The name of the Env to run in the Unity3D editor. Either `3DBall` "
+    "or `SoccerStrikersVsGoalie` (feel free to add more to this script!)")
+parser.add_argument(
+    "--port",
+    type=int,
+    default=SERVER_PORT,
+    help="The Policy server's port to listen on for ExternalEnv client "
+    "conections.")
+parser.add_argument(
+    "--checkpoint-freq",
+    type=int,
+    default=10,
+    help="The frequency with which to create checkpoint files of the learnt "
+    "Policies.")
+parser.add_argument(
+    "--no-restore",
+    action="store_true",
+    help="Whether to load the Policy "
+    "weights from a previous checkpoint")
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    ray.init(local_mode=True)
+
+    # Create a fake-env for the server. This env will never be used (neither
+    # for sampling, nor for evaluation) and its obs/action Spaces do not
+    # matter either (multi-agent config below defines Spaces per Policy).
+    register_env("fake_unity", lambda c: RandomMultiAgentEnv(c))
+
+    policies, policy_mapping_fn = \
+        Unity3DEnv.get_policy_configs_for_game(args.env)
+
+    # The entire config will be sent to connecting clients so they can
+    # build their own samplers (and also Policy objects iff
+    # `inference_mode=local` on clients' command line).
+    config = {
+        # Use the connector server to generate experiences.
+        "input": (
+            lambda ioctx: PolicyServerInput(ioctx, SERVER_ADDRESS, args.port)),
+        # Use a single worker process (w/ SyncSampler) to run the server.
+        "num_workers": 0,
+        # Disable OPE, since the rollouts are coming from online clients.
+        "input_evaluation": [],
+
+        # Other settings.
+        "sample_batch_size": 64,
+        "train_batch_size": 256,
+        "rollout_fragment_length": 20,
+        # Multi-agent setup for the particular env.
+        "multiagent": {
+            "policies": policies,
+            "policy_mapping_fn": policy_mapping_fn,
+        },
+        "framework": "tf",
+    }
+
+    # Create the Trainer used for Policy serving.
+    trainer = PPOTrainer(env="fake_unity", config=config)
+
+    # Attempt to restore from checkpoint if possible.
+    checkpoint_path = CHECKPOINT_FILE.format(args.env)
+    if not args.no_restore and os.path.exists(checkpoint_path):
+        checkpoint_path = open(checkpoint_path).read()
+        print("Restoring from checkpoint path", checkpoint_path)
+        trainer.restore(checkpoint_path)
+
+    # Serving and training loop.
+    count = 0
+    while True:
+        # Calls to train() will block on the configured `input` in the Trainer
+        # config above (PolicyServerInput).
+        print(trainer.train())
+        if count % args.checkpoint_freq == 0:
+            print("Saving learning progress to checkpoint file.")
+            checkpoint = trainer.save()
+            # Write the latest checkpoint location to CHECKPOINT_FILE,
+            # so we can pick up from the latest one after a server re-start.
+            with open(checkpoint_path, "w") as f:
+                f.write(checkpoint)
+        count += 1