[WIP] Large changes to make the tests pass. (#376)

* Revert "Make tests more informative (#372)"

This reverts commit fd353250c8.

* fix bugs, in particular deactivate worker service on driver and remove condition variables

* changes to minimize the changes in this PR

* switch from faulty mutex synchronization to using atomics

* Increase the default size of the message queues, to accommodate exporting large numbers of remote functions. This is a temporary fix, but not a long term solution.

* Reorganize the scheduler export code to queue up exports. This does not solve the underlying problem yet, but sets up a solution.

* Start a separate thread on driver to print error messages by constantly querying the scheduler. This is a temporary solution because the solution based on starting a worker service for the driver which the scheduler can push error messages to is buggy.

* Fix segfault in taskcapsule destructor.

* Move tests for catching errors into a separate test file.

* Revert "roll back grpc (#368)"

This reverts commit c01ef95d04.

test/array_test.py

@@ -9,19 +9,13 @@ from numpy.testing import assert_equal, assert_almost_equal
 import ray.array.remote as ra
 import ray.array.distributed as da
 
-class TestRemoteArrays(unittest.TestCase):
+class RemoteArrayTest(unittest.TestCase):
 
-  @classmethod
-  def setUpClass(cls):
+  def testMethods(self):
     for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
       reload(module)
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(start_ray_local=True)
 
-  @classmethod
-  def tearDownClass(cls):
-    ray.worker.cleanup()
-
-  def test_methods(self):
     # test eye
     object_id = ra.eye.remote(3)
     val = ray.get(object_id)
@@ -47,32 +41,40 @@ class TestRemoteArrays(unittest.TestCase):
     r_val = ray.get(r_id)
     assert_almost_equal(np.dot(q_val, r_val), a_val)
 
-class TestDistributedArrays(unittest.TestCase):
-
-  @classmethod
-  def setUpClass(cls):
-    for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
-      reload(module)
-    ray.init(start_ray_local=True, num_workers=10, num_objstores=2)
-
-  @classmethod
-  def tearDownClass(cls):
     ray.worker.cleanup()
 
-  def test_serialization(self):
+class DistributedArrayTest(unittest.TestCase):
+
+  def testSerialization(self):
+    for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
+      reload(module)
+    ray.init(start_ray_local=True, num_workers=0)
+
     x = da.DistArray([2, 3, 4], np.array([[[ray.put(0)]]]))
     capsule, _ = ray.serialization.serialize(ray.worker.global_worker.handle, x)
     y = ray.serialization.deserialize(ray.worker.global_worker.handle, capsule)
     self.assertEqual(x.shape, y.shape)
     self.assertEqual(x.objectids[0, 0, 0].id, y.objectids[0, 0, 0].id)
 
-  def test_assemble(self):
+    ray.worker.cleanup()
+
+  def testAssemble(self):
+    for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
+      reload(module)
+    ray.init(start_ray_local=True, num_workers=1)
+
     a = ra.ones.remote([da.BLOCK_SIZE, da.BLOCK_SIZE])
     b = ra.zeros.remote([da.BLOCK_SIZE, da.BLOCK_SIZE])
     x = da.DistArray([2 * da.BLOCK_SIZE, da.BLOCK_SIZE], np.array([[a], [b]]))
     assert_equal(x.assemble(), np.vstack([np.ones([da.BLOCK_SIZE, da.BLOCK_SIZE]), np.zeros([da.BLOCK_SIZE, da.BLOCK_SIZE])]))
 
-  def test_methods(self):
+    ray.worker.cleanup()
+
+  def testMethods(self):
+    for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
+      reload(module)
+    ray.init(start_ray_local=True, num_objstores=2, num_workers=10)
+
     x = da.zeros.remote([9, 25, 51], "float")
     assert_equal(ray.get(da.assemble.remote(x)), np.zeros([9, 25, 51]))
 
@@ -205,5 +207,7 @@ class TestDistributedArrays(unittest.TestCase):
       d2 = np.random.randint(1, 35)
       test_dist_qr(d1, d2)
 
+    ray.worker.cleanup()
+
 if __name__ == "__main__":
-    unittest.main(verbosity=2)
+  unittest.main(verbosity=2)