Various cleanups: remove start_ray_local from ray.init, remove unused code, fix "pip install numbuf". (#193)

* Remove start_ray_local from ray.init and change default number of workers to 10.

* Remove alexnet example.

* Move array methods to experimental.

* Remove TRPO example.

* Remove old files.

* Compile plasma when we build numbuf.

* Address comments.

test/array_test.py

@@ -12,15 +12,15 @@ import sys
 if sys.version_info >= (3, 0):
   from importlib import reload
 
-import ray.array.remote as ra
-import ray.array.distributed as da
+import ray.experimental.array.remote as ra
+import ray.experimental.array.distributed as da
 
 class RemoteArrayTest(unittest.TestCase):
 
   def testMethods(self):
     for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
       reload(module)
-    ray.init(start_ray_local=True)
+    ray.init(num_workers=1)
 
     # test eye
     object_id = ra.eye.remote(3)
@@ -54,7 +54,7 @@ class DistributedArrayTest(unittest.TestCase):
   def testAssemble(self):
     for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
       reload(module)
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     a = ra.ones.remote([da.BLOCK_SIZE, da.BLOCK_SIZE])
     b = ra.zeros.remote([da.BLOCK_SIZE, da.BLOCK_SIZE])

test/failure_test.py

@@ -24,7 +24,7 @@ def wait_for_errors(error_type, num_errors, timeout=10):
 class FailureTest(unittest.TestCase):
   def testUnknownSerialization(self):
     reload(test_functions)
-    ray.init(start_ray_local=True, num_workers=1, driver_mode=ray.SILENT_MODE)
+    ray.init(num_workers=1, driver_mode=ray.SILENT_MODE)
 
     test_functions.test_unknown_type.remote()
     wait_for_errors(b"TaskError", 1)
@@ -35,7 +35,7 @@ class FailureTest(unittest.TestCase):
 
 class TaskSerializationTest(unittest.TestCase):
   def testReturnAndPassUnknownType(self):
-    ray.init(start_ray_local=True, num_workers=1, driver_mode=ray.SILENT_MODE)
+    ray.init(num_workers=1, driver_mode=ray.SILENT_MODE)
 
     class Foo(object):
       pass
@@ -57,7 +57,7 @@ class TaskSerializationTest(unittest.TestCase):
 class TaskStatusTest(unittest.TestCase):
   def testFailedTask(self):
     reload(test_functions)
-    ray.init(start_ray_local=True, num_workers=3, driver_mode=ray.SILENT_MODE)
+    ray.init(num_workers=3, driver_mode=ray.SILENT_MODE)
 
     test_functions.throw_exception_fct1.remote()
     test_functions.throw_exception_fct1.remote()
@@ -87,7 +87,7 @@ class TaskStatusTest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testFailImportingRemoteFunction(self):
-    ray.init(start_ray_local=True, num_workers=2, driver_mode=ray.SILENT_MODE)
+    ray.init(num_workers=2, driver_mode=ray.SILENT_MODE)
 
     # This example is somewhat contrived. It should be successfully pickled, and
     # then it should throw an exception when it is unpickled. This may depend a
@@ -115,7 +115,7 @@ class TaskStatusTest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testFailImportingReusableVariable(self):
-    ray.init(start_ray_local=True, num_workers=2, driver_mode=ray.SILENT_MODE)
+    ray.init(num_workers=2, driver_mode=ray.SILENT_MODE)
 
     # This will throw an exception when the reusable variable is imported on the
     # workers.
@@ -131,7 +131,7 @@ class TaskStatusTest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testFailReinitializingVariable(self):
-    ray.init(start_ray_local=True, num_workers=2, driver_mode=ray.SILENT_MODE)
+    ray.init(num_workers=2, driver_mode=ray.SILENT_MODE)
 
     def initializer():
       return 0
@@ -149,7 +149,7 @@ class TaskStatusTest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testFailedFunctionToRun(self):
-    ray.init(start_ray_local=True, num_workers=2, driver_mode=ray.SILENT_MODE)
+    ray.init(num_workers=2, driver_mode=ray.SILENT_MODE)
 
     def f(worker):
       if ray.worker.global_worker.mode == ray.WORKER_MODE:

test/microbenchmarks.py

@@ -18,7 +18,7 @@ class MicroBenchmarkTest(unittest.TestCase):
 
   def testTiming(self):
     reload(test_functions)
-    ray.init(start_ray_local=True, num_workers=3)
+    ray.init(num_workers=3)
 
     # measure the time required to submit a remote task to the scheduler
     elapsed_times = []
@@ -88,7 +88,7 @@ class MicroBenchmarkTest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testCache(self):
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     A = np.random.rand(1, 1000000)
     v = np.random.rand(1000000)

test/runtest.py

@@ -16,8 +16,8 @@ if sys.version_info >= (3, 0):
   from importlib import reload
 
 import ray.test.test_functions as test_functions
-import ray.array.remote as ra
-import ray.array.distributed as da
+import ray.experimental.array.remote as ra
+import ray.experimental.array.distributed as da
 
 def assert_equal(obj1, obj2):
   if type(obj1).__module__ == np.__name__ or type(obj2).__module__ == np.__name__:
@@ -129,7 +129,7 @@ except AttributeError:
 class SerializationTest(unittest.TestCase):
 
   def testRecursiveObjects(self):
-    ray.init(start_ray_local=True, num_workers=0)
+    ray.init(num_workers=0)
 
     class ClassA(object):
       pass
@@ -160,7 +160,7 @@ class SerializationTest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testPassingArgumentsByValue(self):
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     @ray.remote
     def f(x):
@@ -184,7 +184,7 @@ class SerializationTest(unittest.TestCase):
 class WorkerTest(unittest.TestCase):
 
   def testPutGet(self):
-    ray.init(start_ray_local=True, num_workers=0)
+    ray.init(num_workers=0)
 
     for i in range(100):
       value_before = i * 10 ** 6
@@ -215,7 +215,7 @@ class WorkerTest(unittest.TestCase):
 class APITest(unittest.TestCase):
 
   def testRegisterClass(self):
-    ray.init(start_ray_local=True, num_workers=0)
+    ray.init(num_workers=0)
 
     # Check that putting an object of a class that has not been registered
     # throws an exception.
@@ -233,7 +233,7 @@ class APITest(unittest.TestCase):
 
   def testKeywordArgs(self):
     reload(test_functions)
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     x = test_functions.keyword_fct1.remote(1)
     self.assertEqual(ray.get(x), "1 hello")
@@ -270,7 +270,7 @@ class APITest(unittest.TestCase):
 
   def testVariableNumberOfArgs(self):
     reload(test_functions)
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     x = test_functions.varargs_fct1.remote(0, 1, 2)
     self.assertEqual(ray.get(x), "0 1 2")
@@ -284,14 +284,14 @@ class APITest(unittest.TestCase):
 
   def testNoArgs(self):
     reload(test_functions)
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     ray.get(test_functions.no_op.remote())
 
     ray.worker.cleanup()
 
   def testDefiningRemoteFunctions(self):
-    ray.init(start_ray_local=True, num_workers=3)
+    ray.init(num_workers=3)
 
     # Test that we can define a remote function in the shell.
     @ray.remote
@@ -345,13 +345,13 @@ class APITest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testGetMultiple(self):
-    ray.init(start_ray_local=True, num_workers=0)
+    ray.init(num_workers=0)
     object_ids = [ray.put(i) for i in range(10)]
     self.assertEqual(ray.get(object_ids), list(range(10)))
     ray.worker.cleanup()
 
   def testWait(self):
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     @ray.remote
     def f(delay):
@@ -401,7 +401,7 @@ class APITest(unittest.TestCase):
       ray.reusables.bar.append(1)
       return ray.reusables.bar
 
-    ray.init(start_ray_local=True, num_workers=2)
+    ray.init(num_workers=2)
 
     self.assertEqual(ray.get(use_foo.remote()), 1)
     self.assertEqual(ray.get(use_foo.remote()), 1)
@@ -425,7 +425,7 @@ class APITest(unittest.TestCase):
       sys.path.append(4)
     ray.worker.global_worker.run_function_on_all_workers(f)
 
-    ray.init(start_ray_local=True, num_workers=2)
+    ray.init(num_workers=2)
 
     @ray.remote
     def get_state():
@@ -448,7 +448,7 @@ class APITest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testRunningFunctionOnAllWorkers(self):
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     def f(worker_info):
       sys.path.append("fake_directory")
@@ -471,7 +471,7 @@ class APITest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testPassingInfoToAllWorkers(self):
-    ray.init(start_ray_local=True, num_workers=10)
+    ray.init(num_workers=10)
 
     def f(worker_info):
       sys.path.append(worker_info)
@@ -498,7 +498,7 @@ class APITest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testLoggingAPI(self):
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     def events():
       # This is a hack for getting the event log. It is not part of the API.
@@ -551,7 +551,7 @@ class PythonModeTest(unittest.TestCase):
 
   def testPythonMode(self):
     reload(test_functions)
-    ray.init(start_ray_local=True, driver_mode=ray.PYTHON_MODE)
+    ray.init(driver_mode=ray.PYTHON_MODE)
 
     @ray.remote
     def f():
@@ -574,7 +574,7 @@ class PythonModeTest(unittest.TestCase):
 
   def testReusableVariablesInPythonMode(self):
     reload(test_functions)
-    ray.init(start_ray_local=True, driver_mode=ray.PYTHON_MODE)
+    ray.init(driver_mode=ray.PYTHON_MODE)
 
     def l_init():
       return []
@@ -612,7 +612,7 @@ class PythonModeTest(unittest.TestCase):
 class ReusablesTest(unittest.TestCase):
 
   def testReusables(self):
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     # Test that we can add a variable to the key-value store.
 
@@ -688,7 +688,7 @@ class ReusablesTest(unittest.TestCase):
     ray.worker.cleanup()
 
   def testUsingReusablesOnDriver(self):
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     # Test that we can add a variable to the key-value store.
 
@@ -731,7 +731,7 @@ class UtilsTest(unittest.TestCase):
     # The functionality being tested here is really multi-node functionality,
     # but this test just uses a single node.
 
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     source_text = "hello world"
 

test/stress_tests.py

@@ -65,7 +65,7 @@ class TaskTests(unittest.TestCase):
         ray.worker.cleanup()
 
   def testGettingAndPutting(self):
-    ray.init(start_ray_local=True, num_workers=1)
+    ray.init(num_workers=1)
 
     for n in range(8):
       x = np.zeros(10 ** n)

test/tensorflow_test.py

@@ -9,7 +9,7 @@ import ray
 class TensorFlowTest(unittest.TestCase):
 
   def testTensorFlowVariables(self):
-    ray.init(start_ray_local=True, num_workers=2)
+    ray.init(num_workers=2)
 
     x_data = tf.placeholder(tf.float32, shape=[100])
     y_data = tf.placeholder(tf.float32, shape=[100])