From 9dfe7c625e94dce2390e3a0bca9aa4d81af25910 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sun, 5 Jun 2016 21:57:37 -0700 Subject: [PATCH] change directory structure, rename array libraries (#87) --- doc/reference-counting.md | 6 +- lib/python/{ => halo}/arrays/__init__.py | 0 .../arrays/distributed}/__init__.py | 0 .../dist => halo/arrays/distributed}/core.py | 28 +-- .../arrays/distributed}/linalg.py | 18 +- .../arrays/distributed}/random.py | 4 +- .../single => halo/arrays/remote}/__init__.py | 0 .../single => halo/arrays/remote}/core.py | 0 .../single => halo/arrays/remote}/linalg.py | 0 .../single => halo/arrays/remote}/random.py | 0 test/arrays_test.py | 162 +++++++++--------- test/runtest.py | 12 +- test/shell.py | 4 +- test/testrecv.py | 16 +- 14 files changed, 125 insertions(+), 125 deletions(-) rename lib/python/{ => halo}/arrays/__init__.py (100%) rename lib/python/{arrays/dist => halo/arrays/distributed}/__init__.py (100%) rename lib/python/{arrays/dist => halo/arrays/distributed}/core.py (89%) rename lib/python/{arrays/dist => halo/arrays/distributed}/linalg.py (92%) rename lib/python/{arrays/dist => halo/arrays/distributed}/random.py (74%) rename lib/python/{arrays/single => halo/arrays/remote}/__init__.py (100%) rename lib/python/{arrays/single => halo/arrays/remote}/core.py (100%) rename lib/python/{arrays/single => halo/arrays/remote}/linalg.py (100%) rename lib/python/{arrays/single => halo/arrays/remote}/random.py (100%) diff --git a/doc/reference-counting.md b/doc/reference-counting.md index 1d2a43c23..7a452297d 100644 --- a/doc/reference-counting.md +++ b/doc/reference-counting.md @@ -77,14 +77,14 @@ because they must be passed into `AliasObjRefs` at some point). The following problem has not yet been resolved. In the following code, the result `x` will be garbage. ```python -x = halo.pull(single.zeros([10, 10], "float")) +x = halo.pull(ra.zeros([10, 10], "float")) ``` -When `single.zeros` is called, a worker will create an array of zeros and store +When `ra.zeros` is called, a worker will create an array of zeros and store it in an object store. An object reference to the output is returned. The call to `halo.pull` will not copy data from the object store process to the worker process, but will instead give the worker process a pointer to shared memory. After the `halo.pull` call completes, the object reference returned by -`single.zeros` will go out of scope, and the object it refers to will be +`ra.zeros` will go out of scope, and the object it refers to will be deallocated from the object store. This will cause the memory that `x` points to to be garbage. diff --git a/lib/python/arrays/__init__.py b/lib/python/halo/arrays/__init__.py similarity index 100% rename from lib/python/arrays/__init__.py rename to lib/python/halo/arrays/__init__.py diff --git a/lib/python/arrays/dist/__init__.py b/lib/python/halo/arrays/distributed/__init__.py similarity index 100% rename from lib/python/arrays/dist/__init__.py rename to lib/python/halo/arrays/distributed/__init__.py diff --git a/lib/python/arrays/dist/core.py b/lib/python/halo/arrays/distributed/core.py similarity index 89% rename from lib/python/arrays/dist/core.py rename to lib/python/halo/arrays/distributed/core.py index 7ecef0a44..8a0a3d7f9 100644 --- a/lib/python/arrays/dist/core.py +++ b/lib/python/halo/arrays/distributed/core.py @@ -1,6 +1,6 @@ from typing import List import numpy as np -import arrays.single as single +import halo.arrays.remote as ra import halo __all__ = ["BLOCK_SIZE", "DistArray", "assemble", "zeros", "ones", "copy", @@ -87,14 +87,14 @@ def numpy_to_dist(a): def zeros(shape, dtype_name="float"): result = DistArray(shape) for index in np.ndindex(*result.num_blocks): - result.objrefs[index] = single.zeros(DistArray.compute_block_shape(index, shape), dtype_name=dtype_name) + result.objrefs[index] = ra.zeros(DistArray.compute_block_shape(index, shape), dtype_name=dtype_name) return result @halo.remote([List[int], str], [DistArray]) def ones(shape, dtype_name="float"): result = DistArray(shape) for index in np.ndindex(*result.num_blocks): - result.objrefs[index] = single.ones(DistArray.compute_block_shape(index, shape), dtype_name=dtype_name) + result.objrefs[index] = ra.ones(DistArray.compute_block_shape(index, shape), dtype_name=dtype_name) return result @halo.remote([DistArray], [DistArray]) @@ -112,9 +112,9 @@ def eye(dim1, dim2=-1, dtype_name="float"): for (i, j) in np.ndindex(*result.num_blocks): block_shape = DistArray.compute_block_shape([i, j], shape) if i == j: - result.objrefs[i, j] = single.eye(block_shape[0], block_shape[1], dtype_name=dtype_name) + result.objrefs[i, j] = ra.eye(block_shape[0], block_shape[1], dtype_name=dtype_name) else: - result.objrefs[i, j] = single.zeros(block_shape, dtype_name=dtype_name) + result.objrefs[i, j] = ra.zeros(block_shape, dtype_name=dtype_name) return result @halo.remote([DistArray], [DistArray]) @@ -124,11 +124,11 @@ def triu(a): result = DistArray(a.shape) for (i, j) in np.ndindex(*result.num_blocks): if i < j: - result.objrefs[i, j] = single.copy(a.objrefs[i, j]) + result.objrefs[i, j] = ra.copy(a.objrefs[i, j]) elif i == j: - result.objrefs[i, j] = single.triu(a.objrefs[i, j]) + result.objrefs[i, j] = ra.triu(a.objrefs[i, j]) else: - result.objrefs[i, j] = single.zeros_like(a.objrefs[i, j]) + result.objrefs[i, j] = ra.zeros_like(a.objrefs[i, j]) return result @halo.remote([DistArray], [DistArray]) @@ -138,11 +138,11 @@ def tril(a): result = DistArray(a.shape) for (i, j) in np.ndindex(*result.num_blocks): if i > j: - result.objrefs[i, j] = single.copy(a.objrefs[i, j]) + result.objrefs[i, j] = ra.copy(a.objrefs[i, j]) elif i == j: - result.objrefs[i, j] = single.tril(a.objrefs[i, j]) + result.objrefs[i, j] = ra.tril(a.objrefs[i, j]) else: - result.objrefs[i, j] = single.zeros_like(a.objrefs[i, j]) + result.objrefs[i, j] = ra.zeros_like(a.objrefs[i, j]) return result @halo.remote([np.ndarray], [np.ndarray]) @@ -209,7 +209,7 @@ def transpose(a): result = DistArray([a.shape[1], a.shape[0]]) for i in range(result.num_blocks[0]): for j in range(result.num_blocks[1]): - result.objrefs[i, j] = single.transpose(a.objrefs[j, i]) + result.objrefs[i, j] = ra.transpose(a.objrefs[j, i]) return result # TODO(rkn): support broadcasting? @@ -219,7 +219,7 @@ def add(x1, x2): raise Exception("add expects arguments `x1` and `x2` to have the same shape, but x1.shape = {}, and x2.shape = {}.".format(x1.shape, x2.shape)) result = DistArray(x1.shape) for index in np.ndindex(*result.num_blocks): - result.objrefs[index] = single.add(x1.objrefs[index], x2.objrefs[index]) + result.objrefs[index] = ra.add(x1.objrefs[index], x2.objrefs[index]) return result # TODO(rkn): support broadcasting? @@ -229,5 +229,5 @@ def subtract(x1, x2): raise Exception("subtract expects arguments `x1` and `x2` to have the same shape, but x1.shape = {}, and x2.shape = {}.".format(x1.shape, x2.shape)) result = DistArray(x1.shape) for index in np.ndindex(*result.num_blocks): - result.objrefs[index] = single.subtract(x1.objrefs[index], x2.objrefs[index]) + result.objrefs[index] = ra.subtract(x1.objrefs[index], x2.objrefs[index]) return result diff --git a/lib/python/arrays/dist/linalg.py b/lib/python/halo/arrays/distributed/linalg.py similarity index 92% rename from lib/python/arrays/dist/linalg.py rename to lib/python/halo/arrays/distributed/linalg.py index b4d82bfe7..a04fb92f1 100644 --- a/lib/python/arrays/dist/linalg.py +++ b/lib/python/halo/arrays/distributed/linalg.py @@ -1,7 +1,7 @@ from typing import List import numpy as np -import arrays.single as single +import halo.arrays.remote as ra import halo from core import * @@ -35,14 +35,14 @@ def tsqr(a): current_rs = [] for i in range(num_blocks): block = a.objrefs[i, 0] - q, r = single.linalg.qr(block) + q, r = ra.linalg.qr(block) q_tree[i, 0] = q current_rs.append(r) for j in range(1, K): new_rs = [] for i in range(int(np.ceil(1.0 * len(current_rs) / 2))): - stacked_rs = single.vstack(*current_rs[(2 * i):(2 * i + 2)]) - q, r = single.linalg.qr(stacked_rs) + stacked_rs = ra.vstack(*current_rs[(2 * i):(2 * i + 2)]) + q, r = ra.linalg.qr(stacked_rs) q_tree[i, j] = q new_rs.append(r) current_rs = new_rs @@ -74,7 +74,7 @@ def tsqr(a): lower = [a.shape[1], 0] upper = [2 * a.shape[1], BLOCK_SIZE] ith_index /= 2 - q_block_current = single.dot(q_block_current, single.subarray(q_tree[ith_index, j], lower, upper)) + q_block_current = ra.dot(q_block_current, ra.subarray(q_tree[ith_index, j], lower, upper)) q_result.objrefs[i] = q_block_current r = current_rs[0] return q_result, r @@ -164,9 +164,9 @@ def qr(a): y_res.objrefs[j, i] = y_val.objrefs[j - i, 0] if a.shape[0] > a.shape[1]: # in this case, R needs to be square - R_shape = halo.pull(single.shape(R)) - eye_temp = single.eye(R_shape[1], R_shape[0], dtype_name=result_dtype) - r_res.objrefs[i, i] = single.dot(eye_temp, R) + R_shape = halo.pull(ra.shape(R)) + eye_temp = ra.eye(R_shape[1], R_shape[0], dtype_name=result_dtype) + r_res.objrefs[i, i] = ra.dot(eye_temp, R) else: r_res.objrefs[i, i] = R Ts.append(numpy_to_dist(t)) @@ -176,7 +176,7 @@ def qr(a): for r in range(i, a.num_blocks[0]): y_ri = y_val.objrefs[r - i, 0] W_rcs.append(qr_helper2(y_ri, a_work.objrefs[r, c])) - W_c = single.sum(0, *W_rcs) + W_c = ra.sum(0, *W_rcs) for r in range(i, a.num_blocks[0]): y_ri = y_val.objrefs[r - i, 0] A_rc = qr_helper1(a_work.objrefs[r, c], y_ri, t, W_c) diff --git a/lib/python/arrays/dist/random.py b/lib/python/halo/arrays/distributed/random.py similarity index 74% rename from lib/python/arrays/dist/random.py rename to lib/python/halo/arrays/distributed/random.py index b3d4a236c..5ca9b38d9 100644 --- a/lib/python/arrays/dist/random.py +++ b/lib/python/halo/arrays/distributed/random.py @@ -1,7 +1,7 @@ from typing import List import numpy as np -import arrays.single as single +import halo.arrays.remote as ra import halo from core import * @@ -11,7 +11,7 @@ def normal(shape): num_blocks = DistArray.compute_num_blocks(shape) objrefs = np.empty(num_blocks, dtype=object) for index in np.ndindex(*num_blocks): - objrefs[index] = single.random.normal(DistArray.compute_block_shape(index, shape)) + objrefs[index] = ra.random.normal(DistArray.compute_block_shape(index, shape)) result = DistArray() result.construct(shape, objrefs) return result diff --git a/lib/python/arrays/single/__init__.py b/lib/python/halo/arrays/remote/__init__.py similarity index 100% rename from lib/python/arrays/single/__init__.py rename to lib/python/halo/arrays/remote/__init__.py diff --git a/lib/python/arrays/single/core.py b/lib/python/halo/arrays/remote/core.py similarity index 100% rename from lib/python/arrays/single/core.py rename to lib/python/halo/arrays/remote/core.py diff --git a/lib/python/arrays/single/linalg.py b/lib/python/halo/arrays/remote/linalg.py similarity index 100% rename from lib/python/arrays/single/linalg.py rename to lib/python/halo/arrays/remote/linalg.py diff --git a/lib/python/arrays/single/random.py b/lib/python/halo/arrays/remote/random.py similarity index 100% rename from lib/python/arrays/single/random.py rename to lib/python/halo/arrays/remote/random.py diff --git a/test/arrays_test.py b/test/arrays_test.py index 9aa0edf3d..41d64e742 100644 --- a/test/arrays_test.py +++ b/test/arrays_test.py @@ -8,8 +8,8 @@ import time import subprocess32 as subprocess import os -import arrays.single as single -import arrays.dist as dist +import halo.arrays.remote as ra +import halo.arrays.distributed as da from google.protobuf.text_format import * @@ -25,25 +25,25 @@ class ArraysSingleTest(unittest.TestCase): services.start_singlenode_cluster(return_drivers=False, num_workers_per_objstore=1, worker_path=test_path) # test eye - ref = single.eye(3) + ref = ra.eye(3) val = halo.pull(ref) self.assertTrue(np.alltrue(val == np.eye(3))) # test zeros - ref = single.zeros([3, 4, 5]) + ref = ra.zeros([3, 4, 5]) val = halo.pull(ref) self.assertTrue(np.alltrue(val == np.zeros([3, 4, 5]))) # test qr - pass by value val_a = np.random.normal(size=[10, 13]) - ref_q, ref_r = single.linalg.qr(val_a) + ref_q, ref_r = ra.linalg.qr(val_a) val_q = halo.pull(ref_q) val_r = halo.pull(ref_r) self.assertTrue(np.allclose(np.dot(val_q, val_r), val_a)) # test qr - pass by objref - a = single.random.normal([10, 13]) - ref_q, ref_r = single.linalg.qr(a) + a = ra.random.normal([10, 13]) + ref_q, ref_r = ra.linalg.qr(a) val_a = halo.pull(a) val_q = halo.pull(ref_q) val_r = halo.pull(ref_r) @@ -56,7 +56,7 @@ class ArraysDistTest(unittest.TestCase): def testSerialization(self): [w] = services.start_singlenode_cluster(return_drivers=True) - x = dist.DistArray() + x = da.DistArray() x.construct([2, 3, 4], np.array([[[halo.push(0, w)]]])) capsule, _ = serialization.serialize(w.handle, x) # TODO(rkn): THIS REQUIRES A WORKER_HANDLE y = serialization.deserialize(w.handle, capsule) # TODO(rkn): THIS REQUIRES A WORKER_HANDLE @@ -70,11 +70,11 @@ class ArraysDistTest(unittest.TestCase): test_path = os.path.join(test_dir, "testrecv.py") services.start_singlenode_cluster(return_drivers=False, num_workers_per_objstore=1, worker_path=test_path) - a = single.ones([dist.BLOCK_SIZE, dist.BLOCK_SIZE]) - b = single.zeros([dist.BLOCK_SIZE, dist.BLOCK_SIZE]) - x = dist.DistArray() - x.construct([2 * dist.BLOCK_SIZE, dist.BLOCK_SIZE], np.array([[a], [b]])) - self.assertTrue(np.alltrue(x.assemble() == np.vstack([np.ones([dist.BLOCK_SIZE, dist.BLOCK_SIZE]), np.zeros([dist.BLOCK_SIZE, dist.BLOCK_SIZE])]))) + a = ra.ones([da.BLOCK_SIZE, da.BLOCK_SIZE]) + b = ra.zeros([da.BLOCK_SIZE, da.BLOCK_SIZE]) + x = da.DistArray() + x.construct([2 * da.BLOCK_SIZE, da.BLOCK_SIZE], np.array([[a], [b]])) + self.assertTrue(np.alltrue(x.assemble() == np.vstack([np.ones([da.BLOCK_SIZE, da.BLOCK_SIZE]), np.zeros([da.BLOCK_SIZE, da.BLOCK_SIZE])]))) services.cleanup() @@ -83,88 +83,88 @@ class ArraysDistTest(unittest.TestCase): test_path = os.path.join(test_dir, "testrecv.py") services.start_singlenode_cluster(return_drivers=False, num_objstores=2, num_workers_per_objstore=5, worker_path=test_path) - x = dist.zeros([9, 25, 51], "float") - y = dist.assemble(x) + x = da.zeros([9, 25, 51], "float") + y = da.assemble(x) self.assertTrue(np.alltrue(halo.pull(y) == np.zeros([9, 25, 51]))) - x = dist.ones([11, 25, 49], dtype_name="float") - y = dist.assemble(x) + x = da.ones([11, 25, 49], dtype_name="float") + y = da.assemble(x) self.assertTrue(np.alltrue(halo.pull(y) == np.ones([11, 25, 49]))) - x = dist.random.normal([11, 25, 49]) - y = dist.copy(x) - z = dist.assemble(x) - w = dist.assemble(y) + x = da.random.normal([11, 25, 49]) + y = da.copy(x) + z = da.assemble(x) + w = da.assemble(y) self.assertTrue(np.alltrue(halo.pull(z) == halo.pull(w))) - x = dist.eye(25, dtype_name="float") - y = dist.assemble(x) + x = da.eye(25, dtype_name="float") + y = da.assemble(x) self.assertTrue(np.alltrue(halo.pull(y) == np.eye(25))) - x = dist.random.normal([25, 49]) - y = dist.triu(x) - z = dist.assemble(y) - w = dist.assemble(x) + x = da.random.normal([25, 49]) + y = da.triu(x) + z = da.assemble(y) + w = da.assemble(x) self.assertTrue(np.alltrue(halo.pull(z) == np.triu(halo.pull(w)))) - x = dist.random.normal([25, 49]) - y = dist.tril(x) - z = dist.assemble(y) - w = dist.assemble(x) + x = da.random.normal([25, 49]) + y = da.tril(x) + z = da.assemble(y) + w = da.assemble(x) self.assertTrue(np.alltrue(halo.pull(z) == np.tril(halo.pull(w)))) - x = dist.random.normal([25, 49]) - y = dist.random.normal([49, 18]) - z = dist.dot(x, y) - w = dist.assemble(z) - u = dist.assemble(x) - v = dist.assemble(y) + x = da.random.normal([25, 49]) + y = da.random.normal([49, 18]) + z = da.dot(x, y) + w = da.assemble(z) + u = da.assemble(x) + v = da.assemble(y) np.allclose(halo.pull(w), np.dot(halo.pull(u), halo.pull(v))) self.assertTrue(np.allclose(halo.pull(w), np.dot(halo.pull(u), halo.pull(v)))) # test add - x = dist.random.normal([23, 42]) - y = dist.random.normal([23, 42]) - z = dist.add(x, y) - z_full = dist.assemble(z) - x_full = dist.assemble(x) - y_full = dist.assemble(y) + x = da.random.normal([23, 42]) + y = da.random.normal([23, 42]) + z = da.add(x, y) + z_full = da.assemble(z) + x_full = da.assemble(x) + y_full = da.assemble(y) self.assertTrue(np.allclose(halo.pull(z_full), halo.pull(x_full) + halo.pull(y_full))) # test subtract - x = dist.random.normal([33, 40]) - y = dist.random.normal([33, 40]) - z = dist.subtract(x, y) - z_full = dist.assemble(z) - x_full = dist.assemble(x) - y_full = dist.assemble(y) + x = da.random.normal([33, 40]) + y = da.random.normal([33, 40]) + z = da.subtract(x, y) + z_full = da.assemble(z) + x_full = da.assemble(x) + y_full = da.assemble(y) self.assertTrue(np.allclose(halo.pull(z_full), halo.pull(x_full) - halo.pull(y_full))) # test transpose - x = dist.random.normal([234, 432]) - y = dist.transpose(x) - x_full = dist.assemble(x) - y_full = dist.assemble(y) + x = da.random.normal([234, 432]) + y = da.transpose(x) + x_full = da.assemble(x) + y_full = da.assemble(y) self.assertTrue(np.alltrue(halo.pull(x_full).T == halo.pull(y_full))) # test numpy_to_dist - x = dist.random.normal([23, 45]) - y = dist.assemble(x) - z = dist.numpy_to_dist(y) - w = dist.assemble(z) - x_full = dist.assemble(x) - z_full = dist.assemble(z) + x = da.random.normal([23, 45]) + y = da.assemble(x) + z = da.numpy_to_dist(y) + w = da.assemble(z) + x_full = da.assemble(x) + z_full = da.assemble(z) self.assertTrue(np.alltrue(halo.pull(x_full) == halo.pull(z_full))) self.assertTrue(np.alltrue(halo.pull(y) == halo.pull(w))) - # test dist.tsqr - for shape in [[123, dist.BLOCK_SIZE], [7, dist.BLOCK_SIZE], [dist.BLOCK_SIZE, dist.BLOCK_SIZE], [dist.BLOCK_SIZE, 7], [10 * dist.BLOCK_SIZE, dist.BLOCK_SIZE]]: - x = dist.random.normal(shape) + # test da.tsqr + for shape in [[123, da.BLOCK_SIZE], [7, da.BLOCK_SIZE], [da.BLOCK_SIZE, da.BLOCK_SIZE], [da.BLOCK_SIZE, 7], [10 * da.BLOCK_SIZE, da.BLOCK_SIZE]]: + x = da.random.normal(shape) K = min(shape) - q, r = dist.linalg.tsqr(x) - x_full = dist.assemble(x) + q, r = da.linalg.tsqr(x) + x_full = da.assemble(x) x_val = halo.pull(x_full) - q_full = dist.assemble(q) + q_full = da.assemble(q) q_val = halo.pull(q_full) r_val = halo.pull(r) self.assertTrue(r_val.shape == (K, shape[1])) @@ -172,17 +172,17 @@ class ArraysDistTest(unittest.TestCase): self.assertTrue(np.allclose(x_val, np.dot(q_val, r_val))) self.assertTrue(np.allclose(np.dot(q_val.T, q_val), np.eye(K))) - # test dist.linalg.modified_lu + # test da.linalg.modified_lu def test_modified_lu(d1, d2): print "testing dist_modified_lu with d1 = " + str(d1) + ", d2 = " + str(d2) assert d1 >= d2 k = min(d1, d2) - m = single.random.normal([d1, d2]) - q, r = single.linalg.qr(m) - l, u, s = dist.linalg.modified_lu(dist.numpy_to_dist(q)) + m = ra.random.normal([d1, d2]) + q, r = ra.linalg.qr(m) + l, u, s = da.linalg.modified_lu(da.numpy_to_dist(q)) q_val = halo.pull(q) r_val = halo.pull(r) - l_full = dist.assemble(l) + l_full = da.assemble(l) l_val = halo.pull(l_full) u_val = halo.pull(u) s_val = halo.pull(s) @@ -199,11 +199,11 @@ class ArraysDistTest(unittest.TestCase): # test dist_tsqr_hr def test_dist_tsqr_hr(d1, d2): print "testing dist_tsqr_hr with d1 = " + str(d1) + ", d2 = " + str(d2) - a = dist.random.normal([d1, d2]) - y, t, y_top, r = dist.linalg.tsqr_hr(a) - a_full = dist.assemble(a) + a = da.random.normal([d1, d2]) + y, t, y_top, r = da.linalg.tsqr_hr(a) + a_full = da.assemble(a) a_val = halo.pull(a_full) - y_full = dist.assemble(y) + y_full = da.assemble(y) y_val = halo.pull(y_full) t_val = halo.pull(t) y_top_val = halo.pull(y_top) @@ -214,17 +214,17 @@ class ArraysDistTest(unittest.TestCase): self.assertTrue(np.allclose(np.dot(q.T, q), np.eye(min(d1, d2)))) # check that q.T * q = I self.assertTrue(np.allclose(np.dot(q, r_val), a_val)) # check that a = (I - y * t * y_thalo.T) * r - for d1, d2 in [(123, dist.BLOCK_SIZE), (7, dist.BLOCK_SIZE), (dist.BLOCK_SIZE, dist.BLOCK_SIZE), (dist.BLOCK_SIZE, 7), (10 * dist.BLOCK_SIZE, dist.BLOCK_SIZE)]: + for d1, d2 in [(123, da.BLOCK_SIZE), (7, da.BLOCK_SIZE), (da.BLOCK_SIZE, da.BLOCK_SIZE), (da.BLOCK_SIZE, 7), (10 * da.BLOCK_SIZE, da.BLOCK_SIZE)]: test_dist_tsqr_hr(d1, d2) def test_dist_qr(d1, d2): print "testing qr with d1 = {}, and d2 = {}.".format(d1, d2) - a = dist.random.normal([d1, d2]) + a = da.random.normal([d1, d2]) K = min(d1, d2) - q, r = dist.linalg.qr(a) - a_full = dist.assemble(a) - q_full = dist.assemble(q) - r_full = dist.assemble(r) + q, r = da.linalg.qr(a) + a_full = da.assemble(a) + q_full = da.assemble(q) + r_full = da.assemble(r) a_val = halo.pull(a_full) q_val = halo.pull(q_full) r_val = halo.pull(r_full) @@ -235,7 +235,7 @@ class ArraysDistTest(unittest.TestCase): self.assertTrue(np.alltrue(r_val == np.triu(r_val))) self.assertTrue(np.allclose(a_val, np.dot(q_val, r_val))) - for d1, d2 in [(123, dist.BLOCK_SIZE), (7, dist.BLOCK_SIZE), (dist.BLOCK_SIZE, dist.BLOCK_SIZE), (dist.BLOCK_SIZE, 7), (13, 21), (34, 35), (8, 7)]: + for d1, d2 in [(123, da.BLOCK_SIZE), (7, da.BLOCK_SIZE), (da.BLOCK_SIZE, da.BLOCK_SIZE), (da.BLOCK_SIZE, 7), (13, 21), (34, 35), (8, 7)]: test_dist_qr(d1, d2) test_dist_qr(d2, d1) for _ in range(20): diff --git a/test/runtest.py b/test/runtest.py index 58a597e50..f9e125c17 100644 --- a/test/runtest.py +++ b/test/runtest.py @@ -14,8 +14,8 @@ import halo_pb2 import types_pb2 import test_functions -import arrays.single as single -import arrays.dist as dist +import halo.arrays.remote as ra +import halo.arrays.distributed as da class SerializationTest(unittest.TestCase): @@ -266,7 +266,7 @@ class ReferenceCountingTest(unittest.TestCase): del y self.assertTrue(halo.scheduler_info()["reference_counts"][objref_val:(objref_val + 3)] == [-1, -1, -1]) - z = dist.zeros([dist.BLOCK_SIZE, 2 * dist.BLOCK_SIZE], "float") + z = da.zeros([da.BLOCK_SIZE, 2 * da.BLOCK_SIZE], "float") time.sleep(0.1) objref_val = z.val self.assertTrue(halo.scheduler_info()["reference_counts"][objref_val:(objref_val + 3)] == [1, 1, 1]) @@ -275,9 +275,9 @@ class ReferenceCountingTest(unittest.TestCase): time.sleep(0.1) self.assertTrue(halo.scheduler_info()["reference_counts"][objref_val:(objref_val + 3)] == [-1, -1, -1]) - x = single.zeros([10, 10], "float") - y = single.zeros([10, 10], "float") - z = single.dot(x, y) + x = ra.zeros([10, 10], "float") + y = ra.zeros([10, 10], "float") + z = ra.dot(x, y) objref_val = x.val time.sleep(0.1) self.assertTrue(halo.scheduler_info()["reference_counts"][objref_val:(objref_val + 3)] == [1, 1, 1]) diff --git a/test/shell.py b/test/shell.py index c930b7ec5..ce194d17a 100644 --- a/test/shell.py +++ b/test/shell.py @@ -6,8 +6,8 @@ import halo.services as services import halo.worker as worker import test_functions -import arrays.single as single -import arrays.dist as dist +import halo.arrays.remote as ra +import halo.arrays.distributed as da from grpc.beta import implementations import halo_pb2 diff --git a/test/testrecv.py b/test/testrecv.py index 23b54f011..cff3127cf 100644 --- a/test/testrecv.py +++ b/test/testrecv.py @@ -3,8 +3,8 @@ import argparse import numpy as np import test_functions -import arrays.single as single -import arrays.dist as dist +import halo.arrays.remote as ra +import halo.arrays.distributed as da import halo import halo.services as services @@ -20,12 +20,12 @@ if __name__ == '__main__': worker.connect(args.scheduler_address, args.objstore_address, args.worker_address) halo.register_module(test_functions) - halo.register_module(single) - halo.register_module(single.random) - halo.register_module(single.linalg) - halo.register_module(dist) - halo.register_module(dist.random) - halo.register_module(dist.linalg) + halo.register_module(ra) + halo.register_module(ra.random) + halo.register_module(ra.linalg) + halo.register_module(da) + halo.register_module(da.random) + halo.register_module(da.linalg) halo.register_module(sys.modules[__name__]) worker.main_loop()