[RLlib] Curiosity enhancements. (#10373)

rllib/agents/a3c/a3c.py

@@ -53,7 +53,7 @@ def get_policy_class(config):
 
 def validate_config(config):
     if config["entropy_coeff"] < 0:
-        raise DeprecationWarning("entropy_coeff must be >= 0")
+        raise DeprecationWarning("`entropy_coeff` must be >= 0")
     if config["sample_async"] and config["framework"] == "torch":
         config["sample_async"] = False
         logger.warning("`sample_async=True` is not supported for PyTorch! "

rllib/agents/a3c/tests/test_a3c.py

@@ -27,6 +27,7 @@ class TestA3C(unittest.TestCase):
         for fw in framework_iterator(config):
             config["sample_async"] = fw == "tf"
             for env in ["CartPole-v0", "Pendulum-v0", "PongDeterministic-v0"]:
+                print("env={}".format(env))
                 trainer = a3c.A3CTrainer(config=config, env=env)
                 for i in range(num_iterations):
                     results = trainer.train()

rllib/env/unity3d_env.py

@@ -1,7 +1,8 @@
 from gym.spaces import Box, MultiDiscrete, Tuple as TupleSpace
 import logging
 import numpy as np
-from typing import Callable, Tuple
+import time
+from typing import Callable, Optional, Tuple
 
 from ray.rllib.env.multi_agent_env import MultiAgentEnv
 from ray.rllib.utils.annotations import override
@@ -25,13 +26,14 @@ class Unity3DEnv(MultiAgentEnv):
     inside an RLlib PolicyClient for cloud/distributed training of Unity games.
     """
 
+    _BASE_PORT = 5004
+
     def __init__(self,
                  file_name: str = None,
-                 worker_id: int = 0,
-                 base_port: int = 5004,
+                 port: Optional[int] = None,
                  seed: int = 0,
                  no_graphics: bool = False,
-                 timeout_wait: int = 60,
+                 timeout_wait: int = 120,
                  episode_horizon: int = 1000):
         """Initializes a Unity3DEnv object.
 
@@ -39,12 +41,7 @@ class Unity3DEnv(MultiAgentEnv):
             file_name (Optional[str]): Name of the Unity game binary.
                 If None, will assume a locally running Unity3D editor
                 to be used, instead.
-            worker_id (int): Number to add to `base_port`. Used when more than
-                one Unity3DEnv (games) are running on the same machine. This
-                will be determined automatically, if possible, so a value
-                of 0 should always suffice here.
-            base_port (int): Port number to connect to Unity environment.
-                `worker_id` increments on top of this.
+            port (Optional[int]): Port number to connect to Unity environment.
             seed (int): A random seed value to use for the Unity3D game.
             no_graphics (bool): Whether to run the Unity3D simulator in
                 no-graphics mode. Default: False.
@@ -69,23 +66,23 @@ class Unity3DEnv(MultiAgentEnv):
         import mlagents_envs
         from mlagents_envs.environment import UnityEnvironment
 
-        # Try connecting to the Unity3D game instance. If a port
+        # Try connecting to the Unity3D game instance. If a port is blocked
         while True:
-            self.worker_id = worker_id
+            time.sleep(2)
+            port_ = port or self._BASE_PORT
+            self._BASE_PORT += 1
             try:
                 self.unity_env = UnityEnvironment(
                     file_name=file_name,
-                    worker_id=worker_id,
-                    base_port=base_port,
+                    worker_id=0,
+                    base_port=port_,
                     seed=seed,
                     no_graphics=no_graphics,
                     timeout_wait=timeout_wait,
                 )
-            except mlagents_envs.exception.UnityWorkerInUseException as e:
-                worker_id += 1
-                # Hard limit.
-                if worker_id > 100:
-                    raise e
+                print("Created UnityEnvironment for port {}".format(port_))
+            except mlagents_envs.exception.UnityWorkerInUseException:
+                pass
             else:
                 break
 
@@ -203,6 +200,13 @@ class Unity3DEnv(MultiAgentEnv):
             "3DBall": Box(float("-inf"), float("inf"), (8, )),
             # 3DBallHard.
             "3DBallHard": Box(float("-inf"), float("inf"), (45, )),
+            # Pyramids.
+            "Pyramids": TupleSpace([
+                Box(float("-inf"), float("inf"), (56, )),
+                Box(float("-inf"), float("inf"), (56, )),
+                Box(float("-inf"), float("inf"), (56, )),
+                Box(float("-inf"), float("inf"), (4, )),
+            ]),
             # SoccerStrikersVsGoalie.
             "Goalie": Box(float("-inf"), float("inf"), (738, )),
             "Striker": TupleSpace([
@@ -223,6 +227,8 @@ class Unity3DEnv(MultiAgentEnv):
             # 3DBallHard.
             "3DBallHard": Box(
                 float("-inf"), float("inf"), (2, ), dtype=np.float32),
+            # Pyramids.
+            "Pyramids": MultiDiscrete([5]),
             # SoccerStrikersVsGoalie.
             "Goalie": MultiDiscrete([3, 3, 3]),
             "Striker": MultiDiscrete([3, 3, 3]),

rllib/evaluation/sampler.py

@@ -986,17 +986,18 @@ def _process_observations_w_trajectory_view_api(
             logger.warning(
                 "More than {} observations for {} env steps ".format(
                     _sample_collector.total_env_steps(),
-                    _sample_collector.count) + "are buffered in "
-                "the sampler. If this is more than you expected, check that "
-                "that you set a horizon on your environment correctly and that"
-                " it terminates at some point. "
+                    _sample_collector.count) +
+                "are buffered in the sampler. If this is more than you "
+                "expected, check that that you set a horizon on your "
+                "environment correctly and that it terminates at some point. "
                 "Note: In multi-agent environments, `rollout_fragment_length` "
-                "sets the batch size based on environment steps, not the "
-                "steps of "
-                "individual agents, which can result in unexpectedly large "
-                "batches. Also, you may be in evaluation waiting for your Env "
-                "to terminate (batch_mode=`complete_episodes`). Make sure it "
-                "does at some point.")
+                "sets the batch size based on (across-agents) environment "
+                "steps, not the steps of individual agents, which can result "
+                "in unexpectedly large batches." +
+                ("Also, you may be in evaluation waiting for your Env to "
+                 "terminate (batch_mode=`complete_episodes`). Make sure it "
+                 "does at some point."
+                 if not multiple_episodes_in_batch else ""))
 
         # Check episode termination conditions.
         if dones[env_id]["__all__"] or episode.length >= horizon: