Run flake8 in Travis and make code PEP8 compliant. (#387)

This commit is contained in:
Robert Nishihara
2017-03-21 12:57:54 -07:00
committed by Philipp Moritz
parent 083e7a28ad
commit ba02fc0eb0
54 changed files with 2391 additions and 1313 deletions
+147 -60
View File
@@ -12,24 +12,33 @@ import string
import sys
from collections import namedtuple
import ray.test.test_functions as test_functions
if sys.version_info >= (3, 0):
from importlib import reload
import ray.test.test_functions as test_functions
import ray.experimental.array.remote as ra
import ray.experimental.array.distributed as da
def assert_equal(obj1, obj2):
if type(obj1).__module__ == np.__name__ or type(obj2).__module__ == np.__name__:
if (hasattr(obj1, "shape") and obj1.shape == ()) or (hasattr(obj2, "shape") and obj2.shape == ()):
module_numpy = (type(obj1).__module__ == np.__name__ or
type(obj2).__module__ == np.__name__)
if module_numpy:
empty_shape = ((hasattr(obj1, "shape") and obj1.shape == ()) or
(hasattr(obj2, "shape") and obj2.shape == ()))
if empty_shape:
# This is a special case because currently np.testing.assert_equal fails
# because we do not properly handle different numerical types.
assert obj1 == obj2, "Objects {} and {} are different.".format(obj1, obj2)
assert obj1 == obj2, ("Objects {} and {} are "
"different.".format(obj1, obj2))
else:
np.testing.assert_equal(obj1, obj2)
elif hasattr(obj1, "__dict__") and hasattr(obj2, "__dict__"):
special_keys = ["_pytype_"]
assert set(list(obj1.__dict__.keys()) + special_keys) == set(list(obj2.__dict__.keys()) + special_keys), "Objects {} and {} are different.".format(obj1, obj2)
assert (set(list(obj1.__dict__.keys()) + special_keys) ==
set(list(obj2.__dict__.keys()) + special_keys)), ("Objects {} and "
"{} are "
"different."
.format(obj1,
obj2))
for key in obj1.__dict__.keys():
if key not in special_keys:
assert_equal(obj1.__dict__[key], obj2.__dict__[key])
@@ -38,24 +47,29 @@ def assert_equal(obj1, obj2):
for key in obj1.keys():
assert_equal(obj1[key], obj2[key])
elif type(obj1) is list or type(obj2) is list:
assert len(obj1) == len(obj2), "Objects {} and {} are lists with different lengths.".format(obj1, obj2)
assert len(obj1) == len(obj2), ("Objects {} and {} are lists with "
"different lengths.".format(obj1, obj2))
for i in range(len(obj1)):
assert_equal(obj1[i], obj2[i])
elif type(obj1) is tuple or type(obj2) is tuple:
assert len(obj1) == len(obj2), "Objects {} and {} are tuples with different lengths.".format(obj1, obj2)
assert len(obj1) == len(obj2), ("Objects {} and {} are tuples with "
"different lengths.".format(obj1, obj2))
for i in range(len(obj1)):
assert_equal(obj1[i], obj2[i])
elif ray.serialization.is_named_tuple(type(obj1)) or ray.serialization.is_named_tuple(type(obj2)):
assert len(obj1) == len(obj2), "Objects {} and {} are named tuples with different lengths.".format(obj1, obj2)
elif (ray.serialization.is_named_tuple(type(obj1)) or
ray.serialization.is_named_tuple(type(obj2))):
assert len(obj1) == len(obj2), ("Objects {} and {} are named tuples with "
"different lengths.".format(obj1, obj2))
for i in range(len(obj1)):
assert_equal(obj1[i], obj2[i])
else:
assert obj1 == obj2, "Objects {} and {} are different.".format(obj1, obj2)
if sys.version_info >= (3, 0):
long_extras = [0, np.array([["hi", u"hi"], [1.3, 1]])]
else:
long_extras = [long(0), np.array([["hi", u"hi"], [1.3, long(1)]])]
long_extras = [long(0), np.array([["hi", u"hi"], [1.3, long(1)]])] # noqa: E501,F821
PRIMITIVE_OBJECTS = [0, 0.0, 0.9, 1 << 62, "a", string.printable, "\u262F",
u"hello world", u"\xff\xfe\x9c\x001\x000\x00", None, True,
@@ -65,45 +79,55 @@ PRIMITIVE_OBJECTS = [0, 0.0, 0.9, 1 << 62, "a", string.printable, "\u262F",
np.random.normal(size=[100, 100]), np.array(["hi", 3]),
np.array(["hi", 3], dtype=object)] + long_extras
COMPLEX_OBJECTS = [[[[[[[[[[[[[]]]]]]]]]]]],
{"obj{}".format(i): np.random.normal(size=[100, 100]) for i in range(10)},
#{(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {}}}}}}}}}}}}},
((((((((((),),),),),),),),),),
{"a": {"b": {"c": {"d": {}}}}}
]
COMPLEX_OBJECTS = [
[[[[[[[[[[[[]]]]]]]]]]]],
{"obj{}".format(i): np.random.normal(size=[100, 100]) for i in range(10)},
# {(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {
# (): {(): {}}}}}}}}}}}}},
((((((((((),),),),),),),),),),
{"a": {"b": {"c": {"d": {}}}}}]
class Foo(object):
def __init__(self):
pass
class Bar(object):
def __init__(self):
for i, val in enumerate(PRIMITIVE_OBJECTS + COMPLEX_OBJECTS):
setattr(self, "field{}".format(i), val)
class Baz(object):
def __init__(self):
self.foo = Foo()
self.bar = Bar()
def method(self, arg):
pass
class Qux(object):
def __init__(self):
self.objs = [Foo(), Bar(), Baz()]
class SubQux(Qux):
def __init__(self):
Qux.__init__(self)
class CustomError(Exception):
pass
Point = namedtuple("Point", ["x", "y"])
NamedTupleExample = namedtuple("Example", "field1, field2, field3, field4, field5")
NamedTupleExample = namedtuple("Example",
"field1, field2, field3, field4, field5")
CUSTOM_OBJECTS = [Exception("Test object."), CustomError(), Point(11, y=22),
Foo(), Bar(), Baz(), # Qux(), SubQux(),
Foo(), Bar(), Baz(), # Qux(), SubQux(),
NamedTupleExample(1, 1.0, "hi", np.zeros([3, 5]), [1, 2, 3])]
BASE_OBJECTS = PRIMITIVE_OBJECTS + COMPLEX_OBJECTS + CUSTOM_OBJECTS
@@ -112,8 +136,9 @@ LIST_OBJECTS = [[obj] for obj in BASE_OBJECTS]
TUPLE_OBJECTS = [(obj,) for obj in BASE_OBJECTS]
# The check that type(obj).__module__ != "numpy" should be unnecessary, but
# otherwise this seems to fail on Mac OS X on Travis.
DICT_OBJECTS = ([{obj: obj} for obj in PRIMITIVE_OBJECTS if obj.__hash__ is not None and type(obj).__module__ != "numpy"] +
# DICT_OBJECTS = ([{obj: obj} for obj in BASE_OBJECTS if obj.__hash__ is not None] +
DICT_OBJECTS = ([{obj: obj} for obj in PRIMITIVE_OBJECTS
if (obj.__hash__ is not None and
type(obj).__module__ != "numpy")] +
[{0: obj} for obj in BASE_OBJECTS])
RAY_TEST_OBJECTS = BASE_OBJECTS + LIST_OBJECTS + TUPLE_OBJECTS + DICT_OBJECTS
@@ -124,7 +149,10 @@ try:
cloudpickle.dumps(Point)
except AttributeError:
cloudpickle_command = "pip install --upgrade cloudpickle"
raise Exception("You have an older version of cloudpickle that is not able to serialize namedtuples. Try running \n\n{}\n\n".format(cloudpickle_command))
raise Exception("You have an older version of cloudpickle that is not able "
"to serialize namedtuples. Try running "
"\n\n{}\n\n".format(cloudpickle_command))
class SerializationTest(unittest.TestCase):
@@ -155,7 +183,7 @@ class SerializationTest(unittest.TestCase):
# Check that exceptions are thrown when we serialize the recursive objects.
for obj in recursive_objects:
self.assertRaises(Exception, lambda : ray.put(obj))
self.assertRaises(Exception, lambda: ray.put(obj))
ray.worker.cleanup()
@@ -181,6 +209,7 @@ class SerializationTest(unittest.TestCase):
ray.worker.cleanup()
class WorkerTest(unittest.TestCase):
def testPythonWorkers(self):
@@ -228,6 +257,7 @@ class WorkerTest(unittest.TestCase):
ray.worker.cleanup()
class APITest(unittest.TestCase):
def testRegisterClass(self):
@@ -237,10 +267,10 @@ class APITest(unittest.TestCase):
# throws an exception.
class TempClass(object):
pass
self.assertRaises(Exception, lambda : ray.put(Foo))
self.assertRaises(Exception, lambda: ray.put(Foo))
# Check that registering a class that Ray cannot serialize efficiently
# raises an exception.
self.assertRaises(Exception, lambda : ray.register_class(type(True)))
self.assertRaises(Exception, lambda: ray.register_class(type(True)))
# Check that registering the same class with pickle works.
ray.register_class(type(float), pickle=True)
self.assertEqual(ray.get(ray.put(float)), float)
@@ -328,7 +358,9 @@ class APITest(unittest.TestCase):
print("Still using old definition of f, trying again.")
# Test that we can close over plain old data.
data = [np.zeros([3, 5]), (1, 2, "a"), [0.0, 1.0, 1 << 62], 1 << 60, {"a": np.zeros(3)}]
data = [np.zeros([3, 5]), (1, 2, "a"), [0.0, 1.0, 1 << 62], 1 << 60,
{"a": np.zeros(3)}]
@ray.remote
def g():
return data
@@ -339,18 +371,22 @@ class APITest(unittest.TestCase):
def h():
return np.zeros([3, 5])
assert_equal(ray.get(h.remote()), np.zeros([3, 5]))
@ray.remote
def j():
return time.time()
ray.get(j.remote())
# Test that we can define remote functions that call other remote functions.
# Test that we can define remote functions that call other remote
# functions.
@ray.remote
def k(x):
return x + 1
@ray.remote
def l(x):
return ray.get(k.remote(x))
@ray.remote
def m(x):
return ray.get(l.remote(x))
@@ -398,7 +434,7 @@ class APITest(unittest.TestCase):
# Verify that calling wait with duplicate object IDs throws an exception.
x = ray.put(1)
self.assertRaises(Exception, lambda : ray.wait([x, x]))
self.assertRaises(Exception, lambda: ray.wait([x, x]))
ray.worker.cleanup()
@@ -435,11 +471,14 @@ class APITest(unittest.TestCase):
ray.worker.cleanup()
def testCachingEnvironmentVariables(self):
# Test that we can define environment variables before the driver is connected.
# Test that we can define environment variables before the driver is
# connected.
def foo_initializer():
return 1
def bar_initializer():
return []
def bar_reinitializer(bar):
return []
ray.env.foo = ray.EnvironmentVariable(foo_initializer)
@@ -448,6 +487,7 @@ class APITest(unittest.TestCase):
@ray.remote
def use_foo():
return ray.env.foo
@ray.remote
def use_bar():
ray.env.bar.append(1)
@@ -463,16 +503,20 @@ class APITest(unittest.TestCase):
ray.worker.cleanup()
def testCachingFunctionsToRun(self):
# Test that we export functions to run on all workers before the driver is connected.
# Test that we export functions to run on all workers before the driver is
# connected.
def f(worker_info):
sys.path.append(1)
ray.worker.global_worker.run_function_on_all_workers(f)
def f(worker_info):
sys.path.append(2)
ray.worker.global_worker.run_function_on_all_workers(f)
def g(worker_info):
sys.path.append(3)
ray.worker.global_worker.run_function_on_all_workers(g)
def f(worker_info):
sys.path.append(4)
ray.worker.global_worker.run_function_on_all_workers(f)
@@ -505,13 +549,16 @@ class APITest(unittest.TestCase):
def f(worker_info):
sys.path.append("fake_directory")
ray.worker.global_worker.run_function_on_all_workers(f)
@ray.remote
def get_path1():
return sys.path
self.assertEqual("fake_directory", ray.get(get_path1.remote())[-1])
def f(worker_info):
sys.path.pop(-1)
ray.worker.global_worker.run_function_on_all_workers(f)
# Create a second remote function to guarantee that when we call
# get_path2.remote(), the second function to run will have been run on the
# worker.
@@ -528,6 +575,7 @@ class APITest(unittest.TestCase):
def f(worker_info):
sys.path.append(worker_info)
ray.worker.global_worker.run_function_on_all_workers(f)
@ray.remote
def get_path():
time.sleep(1)
@@ -542,6 +590,7 @@ class APITest(unittest.TestCase):
counters = [worker_info["counter"] for worker_info in worker_infos]
# We use range(11) because the driver also runs the function.
self.assertEqual(set(counters), set(range(11)))
# Clean up the worker paths.
def f(worker_info):
sys.path.pop(-1)
@@ -555,7 +604,8 @@ class APITest(unittest.TestCase):
def events():
# This is a hack for getting the event log. It is not part of the API.
keys = ray.worker.global_worker.redis_client.keys("event_log:*")
return [ray.worker.global_worker.redis_client.lrange(key, 0, -1) for key in keys]
return [ray.worker.global_worker.redis_client.lrange(key, 0, -1)
for key in keys]
def wait_for_num_events(num_events, timeout=10):
start_time = time.time()
@@ -604,25 +654,28 @@ class APITest(unittest.TestCase):
# accidentally call an older version.
ray.init(num_workers=2)
num_remote_functions = 100
num_calls = 200
@ray.remote
def f():
return 1
results1 = [f.remote() for _ in range(num_calls)]
@ray.remote
def f():
return 2
results2 = [f.remote() for _ in range(num_calls)]
@ray.remote
def f():
return 3
results3 = [f.remote() for _ in range(num_calls)]
@ray.remote
def f():
return 4
results4 = [f.remote() for _ in range(num_calls)]
@ray.remote
def f():
return 5
@@ -637,16 +690,20 @@ class APITest(unittest.TestCase):
@ray.remote
def g():
return 1
@ray.remote
@ray.remote # noqa: F811
def g():
return 2
@ray.remote
@ray.remote # noqa: F811
def g():
return 3
@ray.remote
@ray.remote # noqa: F811
def g():
return 4
@ray.remote
@ray.remote # noqa: F811
def g():
return 5
@@ -668,6 +725,7 @@ class APITest(unittest.TestCase):
ray.worker.cleanup()
class PythonModeTest(unittest.TestCase):
def testPythonMode(self):
@@ -678,17 +736,21 @@ class PythonModeTest(unittest.TestCase):
def f():
return np.ones([3, 4, 5])
xref = f.remote()
assert_equal(xref, np.ones([3, 4, 5])) # remote functions should return by value
assert_equal(xref, ray.get(xref)) # ray.get should be the identity
# Remote functions should return by value.
assert_equal(xref, np.ones([3, 4, 5]))
# Check that ray.get is the identity.
assert_equal(xref, ray.get(xref))
y = np.random.normal(size=[11, 12])
assert_equal(y, ray.put(y)) # ray.put should be the identity
# Check that ray.put is the identity.
assert_equal(y, ray.put(y))
# make sure objects are immutable, this example is why we need to copy
# Make sure objects are immutable, this example is why we need to copy
# arguments before passing them into remote functions in python mode
aref = test_functions.python_mode_f.remote()
assert_equal(aref, np.array([0, 0]))
bref = test_functions.python_mode_g.remote(aref)
assert_equal(aref, np.array([0, 0])) # python_mode_g should not mutate aref
# Make sure python_mode_g does not mutate aref.
assert_equal(aref, np.array([0, 0]))
assert_equal(bref, np.array([1, 0]))
ray.worker.cleanup()
@@ -699,6 +761,7 @@ class PythonModeTest(unittest.TestCase):
def l_init():
return []
def l_reinit(l):
return []
ray.env.l = ray.EnvironmentVariable(l_init, l_reinit)
@@ -717,7 +780,8 @@ class PythonModeTest(unittest.TestCase):
assert_equal(ray.get(use_l.remote()), [1])
assert_equal(ray.get(use_l.remote()), [1])
# Make sure the local copy of the environment variable has not been mutated.
# Make sure the local copy of the environment variable has not been
# mutated.
assert_equal(l, [])
l = ray.env.l
assert_equal(l, [])
@@ -730,6 +794,7 @@ class PythonModeTest(unittest.TestCase):
ray.worker.cleanup()
class EnvironmentVariablesTest(unittest.TestCase):
def testEnvironmentVariables(self):
@@ -739,6 +804,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
def foo_initializer():
return 1
def foo_reinitializer(foo):
return foo
@@ -752,7 +818,8 @@ class EnvironmentVariablesTest(unittest.TestCase):
self.assertEqual(ray.get(use_foo.remote()), 1)
self.assertEqual(ray.get(use_foo.remote()), 1)
# Test that we can add a variable to the key-value store, mutate it, and reset it.
# Test that we can add a variable to the key-value store, mutate it, and
# reset it.
def bar_initializer():
return [1, 2, 3]
@@ -771,6 +838,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
def baz_initializer():
return np.zeros([4])
def baz_reinitializer(baz):
for i in range(len(baz)):
baz[i] = 0
@@ -794,6 +862,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
def qux_initializer():
return 0
def qux_reinitializer(x):
return x + 1
@@ -815,6 +884,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
def foo_initializer():
return []
def foo_reinitializer(foo):
return []
@@ -846,6 +916,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
ray.worker.cleanup()
class UtilsTest(unittest.TestCase):
def testCopyingDirectory(self):
@@ -894,6 +965,7 @@ class UtilsTest(unittest.TestCase):
ray.worker.cleanup()
class ResourcesTest(unittest.TestCase):
def testResourceConstraints(self):
@@ -901,13 +973,16 @@ class ResourcesTest(unittest.TestCase):
ray.init(num_workers=num_workers, num_cpus=10, num_gpus=2)
# Attempt to wait for all of the workers to start up.
ray.worker.global_worker.run_function_on_all_workers(lambda worker_info: sys.path.append(worker_info["counter"]))
ray.worker.global_worker.run_function_on_all_workers(
lambda worker_info: sys.path.append(worker_info["counter"]))
@ray.remote(num_cpus=0)
def get_worker_id():
time.sleep(1)
return sys.path[-1]
while True:
if len(set(ray.get([get_worker_id.remote() for _ in range(num_workers)]))) == num_workers:
if len(set(ray.get([get_worker_id.remote()
for _ in range(num_workers)]))) == num_workers:
break
time_buffer = 0.3
@@ -974,13 +1049,16 @@ class ResourcesTest(unittest.TestCase):
ray.init(num_workers=num_workers, num_cpus=10, num_gpus=10)
# Attempt to wait for all of the workers to start up.
ray.worker.global_worker.run_function_on_all_workers(lambda worker_info: sys.path.append(worker_info["counter"]))
ray.worker.global_worker.run_function_on_all_workers(
lambda worker_info: sys.path.append(worker_info["counter"]))
@ray.remote(num_cpus=0)
def get_worker_id():
time.sleep(1)
return sys.path[-1]
while True:
if len(set(ray.get([get_worker_id.remote() for _ in range(num_workers)]))) == num_workers:
if len(set(ray.get([get_worker_id.remote()
for _ in range(num_workers)]))) == num_workers:
break
@ray.remote(num_cpus=1, num_gpus=9)
@@ -1021,8 +1099,8 @@ class ResourcesTest(unittest.TestCase):
def testMultipleLocalSchedulers(self):
# This test will define a bunch of tasks that can only be assigned to
# specific local schedulers, and we will check that they are assigned to the
# correct local schedulers.
# specific local schedulers, and we will check that they are assigned to
# the correct local schedulers.
address_info = ray.worker._init(start_ray_local=True,
num_local_schedulers=3,
num_cpus=[100, 5, 10],
@@ -1088,7 +1166,8 @@ class ResourcesTest(unittest.TestCase):
results.append(run_on_0_2.remote())
return names, results
store_names = [object_store_address.name for object_store_address in address_info["object_store_addresses"]]
store_names = [object_store_address.name for object_store_address
in address_info["object_store_addresses"]]
def validate_names_and_results(names, results):
for name, result in zip(names, ray.get(results)):
@@ -1099,7 +1178,8 @@ class ResourcesTest(unittest.TestCase):
elif name == "run_on_2":
self.assertIn(result, [store_names[2]])
elif name == "run_on_0_1_2":
self.assertIn(result, [store_names[0], store_names[1], store_names[2]])
self.assertIn(result, [store_names[0], store_names[1],
store_names[2]])
elif name == "run_on_1_2":
self.assertIn(result, [store_names[1], store_names[2]])
elif name == "run_on_0_2":
@@ -1128,6 +1208,7 @@ class ResourcesTest(unittest.TestCase):
ray.worker.cleanup()
class WorkerPoolTests(unittest.TestCase):
def tearDown(self):
@@ -1177,6 +1258,7 @@ class WorkerPoolTests(unittest.TestCase):
ray.worker.cleanup()
class SchedulingAlgorithm(unittest.TestCase):
def attempt_to_load_balance(self, remote_function, args, total_tasks,
@@ -1184,21 +1266,24 @@ class SchedulingAlgorithm(unittest.TestCase):
num_attempts=20):
attempts = 0
while attempts < num_attempts:
locations = ray.get([remote_function.remote(*args) for _ in range(total_tasks)])
locations = ray.get([remote_function.remote(*args)
for _ in range(total_tasks)])
names = set(locations)
counts = [locations.count(name) for name in names]
print("Counts are {}.".format(counts))
if len(names) == num_local_schedulers and all([count >= minimum_count for count in counts]):
if len(names) == num_local_schedulers and all([count >= minimum_count
for count in counts]):
break
attempts += 1
self.assertLess(attempts, num_attempts)
def testLoadBalancing(self):
# This test ensures that tasks are being assigned to all local schedulers in
# a roughly equal manner.
# This test ensures that tasks are being assigned to all local schedulers
# in a roughly equal manner.
num_workers = 21
num_local_schedulers = 3
ray.worker._init(start_ray_local=True, num_workers=num_workers, num_local_schedulers=num_local_schedulers)
ray.worker._init(start_ray_local=True, num_workers=num_workers,
num_local_schedulers=num_local_schedulers)
@ray.remote
def f():
@@ -1211,11 +1296,12 @@ class SchedulingAlgorithm(unittest.TestCase):
ray.worker.cleanup()
def testLoadBalancingWithDependencies(self):
# This test ensures that tasks are being assigned to all local schedulers in
# a roughly equal manner even when the tasks have dependencies.
# This test ensures that tasks are being assigned to all local schedulers
# in a roughly equal manner even when the tasks have dependencies.
num_workers = 3
num_local_schedulers = 3
ray.worker._init(start_ray_local=True, num_workers=num_workers, num_local_schedulers=num_local_schedulers)
ray.worker._init(start_ray_local=True, num_workers=num_workers,
num_local_schedulers=num_local_schedulers)
@ray.remote
def f(x):
@@ -1229,5 +1315,6 @@ class SchedulingAlgorithm(unittest.TestCase):
ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)