[xray] Add error table and push error messages to driver through node manager. (#2256)

* Fix documentation indentation.

* Add error table to GCS and push error messages through node manager.

* Add type to error data.

* Linting

* Fix failure_test bug.

* Linting.

* Enable one more test.

* Attempt to fix doc building.

* Restructuring

* Fixes

* More fixes.

* Move current_time_ms function into util.h.

test/failure_test.py

@@ -269,9 +269,6 @@ class WorkerDeath(unittest.TestCase):
     def tearDown(self):
         ray.worker.cleanup()
 
-    @unittest.skipIf(
-        os.environ.get("RAY_USE_XRAY") == "1",
-        "This test does not work with xray yet.")
     def testWorkerRaisingException(self):
         ray.init(num_workers=1, driver_mode=ray.SILENT_MODE)
 
@@ -287,9 +284,6 @@ class WorkerDeath(unittest.TestCase):
         wait_for_errors(ray_constants.WORKER_DIED_PUSH_ERROR, 1)
         self.assertEqual(len(ray.error_info()), 2)
 
-    @unittest.skipIf(
-        os.environ.get("RAY_USE_XRAY") == "1",
-        "This test does not work with xray yet.")
     def testWorkerDying(self):
         ray.init(num_workers=0, driver_mode=ray.SILENT_MODE)
 
@@ -303,7 +297,7 @@ class WorkerDeath(unittest.TestCase):
         wait_for_errors(ray_constants.WORKER_DIED_PUSH_ERROR, 1)
 
         self.assertEqual(len(ray.error_info()), 1)
-        self.assertIn("died or was killed while executing the task",
+        self.assertIn("died or was killed while executing",
                       ray.error_info()[0]["message"])
 
     @unittest.skipIf(

test/runtest.py

@@ -2243,7 +2243,7 @@ class GlobalStateAPI(unittest.TestCase):
             worker_ids = set(ray.get([f.remote() for _ in range(10)]))
 
         worker_info = ray.global_state.workers()
-        self.assertEqual(len(worker_info), num_workers)
+        assert len(worker_info) >= num_workers
         for worker_id, info in worker_info.items():
             self.assertIn("node_ip_address", info)
             self.assertIn("local_scheduler_socket", info)