diff --git a/rllib/env/unity3d_env.py b/rllib/env/unity3d_env.py
index f437de008..36c6c0347 100644
--- a/rllib/env/unity3d_env.py
+++ b/rllib/env/unity3d_env.py
@@ -1,6 +1,7 @@
 from gym.spaces import Box, MultiDiscrete, Tuple as TupleSpace
 import logging
 import numpy as np
+import random
 import time
 from typing import Callable, Optional, Tuple
 
@@ -33,7 +34,7 @@ class Unity3DEnv(MultiAgentEnv):
                  port: Optional[int] = None,
                  seed: int = 0,
                  no_graphics: bool = False,
-                 timeout_wait: int = 120,
+                 timeout_wait: int = 300,
                  episode_horizon: int = 1000):
         """Initializes a Unity3DEnv object.
 
@@ -68,7 +69,10 @@ class Unity3DEnv(MultiAgentEnv):
 
         # Try connecting to the Unity3D game instance. If a port is blocked
         while True:
-            time.sleep(2)
+            # Sleep for random time to allow for concurrent startup of many
+            # environments (num_workers >> 1). Otherwise, would lead to port
+            # conflicts sometimes.
+            time.sleep(random.randint(1, 10))
             port_ = port or self._BASE_PORT
             self._BASE_PORT += 1
             try:
diff --git a/rllib/examples/unity3d_env_local.py b/rllib/examples/unity3d_env_local.py
index 1c4c7c45b..baf0b8410 100644
--- a/rllib/examples/unity3d_env_local.py
+++ b/rllib/examples/unity3d_env_local.py
@@ -74,6 +74,8 @@ if __name__ == "__main__":
         "unity3d",
         lambda c: Unity3DEnv(
             file_name=c["file_name"],
+            no_graphics=(args.env != "VisualHallway" and
+                         c["file_name"] is not None),
             episode_horizon=c["episode_horizon"],
         ))
 
diff --git a/rllib/policy/torch_policy.py b/rllib/policy/torch_policy.py
index 2521afda9..c7dc5861d 100644
--- a/rllib/policy/torch_policy.py
+++ b/rllib/policy/torch_policy.py
@@ -1,5 +1,6 @@
 import functools
 import gym
+import logging
 import numpy as np
 import time
 from typing import Callable, Dict, List, Optional, Tuple, Type, Union
@@ -23,6 +24,8 @@ from ray.rllib.utils.typing import ModelGradients, ModelWeights, \
 
 torch, _ = try_import_torch()
 
+logger = logging.getLogger(__name__)
+
 
 @DeveloperAPI
 class TorchPolicy(Policy):
@@ -104,8 +107,10 @@ class TorchPolicy(Policy):
         self.framework = "torch"
         super().__init__(observation_space, action_space, config)
         if torch.cuda.is_available():
+            logger.info("TorchPolicy running on GPU.")
             self.device = torch.device("cuda")
         else:
+            logger.info("TorchPolicy running on CPU.")
             self.device = torch.device("cpu")
         self.model = model.to(self.device)
         # Combine view_requirements for Model and Policy.