[rllib] Clean up agent resource configurations (#3296)

Closes #3284

python/ray/tune/ray_trial_executor.py

@@ -29,6 +29,8 @@ class RayTrialExecutor(TrialExecutor):
         self._avail_resources = Resources(cpu=0, gpu=0)
         self._committed_resources = Resources(cpu=0, gpu=0)
         self._resources_initialized = False
+        if ray.is_initialized():
+            self._update_avail_resources()
 
     def _setup_runner(self, trial):
         cls = ray.remote(
@@ -257,7 +259,16 @@ class RayTrialExecutor(TrialExecutor):
                 self._committed_resources.cpu, self._avail_resources.cpu,
                 self._committed_resources.gpu, self._avail_resources.gpu)
         else:
-            return ""
+            return "Resources requested: ?"
+
+    def resource_string(self):
+        """Returns a string describing the total resources available."""
+
+        if self._resources_initialized:
+            return "{} CPUs, {} GPUs".format(self._avail_resources.cpu,
+                                             self._avail_resources.gpu)
+        else:
+            return "? CPUs, ? GPUs"
 
     def on_step_begin(self):
         """Before step() called, update the available resources."""

python/ray/tune/trial_executor.py

@@ -158,7 +158,11 @@ class TrialExecutor(object):
 
     def debug_string(self):
         """Returns a human readable message for printing to the console."""
-        pass
+        raise NotImplementedError
+
+    def resource_string(self):
+        """Returns a string describing the total resources available."""
+        raise NotImplementedError
 
     def restore(self, trial, checkpoint=None):
         """Restores training state from a checkpoint.

python/ray/tune/trial_runner.py

@@ -122,13 +122,13 @@ class TrialRunner(object):
                     if not self.has_resources(trial.resources):
                         raise TuneError(
                             ("Insufficient cluster resources to launch trial: "
-                             "trial requested {} but the cluster summary: {} "
+                             "trial requested {} but the cluster has only {}. "
                              "Pass `queue_trials=True` in "
                              "ray.tune.run_experiments() or on the command "
                              "line to queue trials until the cluster scales "
                              "up. {}").format(
                                  trial.resources.summary_string(),
-                                 self.trial_executor.debug_string(),
+                                 self.trial_executor.resource_string(),
                                  trial._get_trainable_cls().resource_help(
                                      trial.config)))
                 elif trial.status == Trial.PAUSED: