[tune] CPU-Only Head Node support (#5900)

* trialqueue

* add tests

python/ray/tune/tests/test_ray_trial_executor.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import json
 import unittest
 
 import ray
@@ -13,6 +14,7 @@ from ray.tune.registry import _global_registry, TRAINABLE_CLASS
 from ray.tune.suggest import BasicVariantGenerator
 from ray.tune.trial import Trial, Checkpoint
 from ray.tune.resources import Resources
+from ray.tests.cluster_utils import Cluster
 
 
 class RayTrialExecutorTest(unittest.TestCase):
@@ -112,6 +114,71 @@ class RayTrialExecutorTest(unittest.TestCase):
         return suggester.next_trials()
 
 
+class RayExecutorQueueTest(unittest.TestCase):
+    def setUp(self):
+        self.trial_executor = RayTrialExecutor(
+            queue_trials=True, refresh_period=0)
+        self.cluster = Cluster(
+            initialize_head=True,
+            connect=True,
+            head_node_args={
+                "num_cpus": 1,
+                "_internal_config": json.dumps({
+                    "num_heartbeats_timeout": 10
+                })
+            })
+        # Pytest doesn't play nicely with imports
+        _register_all()
+
+    def tearDown(self):
+        ray.shutdown()
+        self.cluster.shutdown()
+        _register_all()  # re-register the evicted objects
+
+    def testQueueTrial(self):
+        """Tests that reset handles NotImplemented properly."""
+
+        def create_trial(cpu, gpu=0):
+            return Trial("__fake", resources=Resources(cpu=cpu, gpu=gpu))
+
+        cpu_only = create_trial(1, 0)
+        self.assertTrue(self.trial_executor.has_resources(cpu_only.resources))
+        self.trial_executor.start_trial(cpu_only)
+
+        gpu_only = create_trial(0, 1)
+        self.assertTrue(self.trial_executor.has_resources(gpu_only.resources))
+
+    def testHeadBlocking(self):
+        def create_trial(cpu, gpu=0):
+            return Trial("__fake", resources=Resources(cpu=cpu, gpu=gpu))
+
+        gpu_trial = create_trial(1, 1)
+        self.assertTrue(self.trial_executor.has_resources(gpu_trial.resources))
+        self.trial_executor.start_trial(gpu_trial)
+
+        # TODO(rliaw): This behavior is probably undesirable, but right now
+        # trials with different resource requirements is not often used.
+        cpu_only_trial = create_trial(1, 0)
+        self.assertFalse(
+            self.trial_executor.has_resources(cpu_only_trial.resources))
+
+        self.cluster.add_node(num_cpus=1, num_gpus=1)
+        self.cluster.wait_for_nodes()
+
+        self.assertTrue(
+            self.trial_executor.has_resources(cpu_only_trial.resources))
+        self.trial_executor.start_trial(cpu_only_trial)
+
+        cpu_only_trial2 = create_trial(1, 0)
+        self.assertTrue(
+            self.trial_executor.has_resources(cpu_only_trial2.resources))
+        self.trial_executor.start_trial(cpu_only_trial2)
+
+        cpu_only_trial3 = create_trial(1, 0)
+        self.assertFalse(
+            self.trial_executor.has_resources(cpu_only_trial3.resources))
+
+
 class LocalModeExecutorTest(RayTrialExecutorTest):
     def setUp(self):
         self.trial_executor = RayTrialExecutor(queue_trials=False)