wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-01 18:22:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Run flake8 in Travis and make code PEP8 compliant. (#387)

Browse Source
This commit is contained in:
Robert Nishihara
2017-03-21 12:57:54 -07:00
committed by Philipp Moritz
parent 083e7a28ad
commit ba02fc0eb0
54 changed files with 2391 additions and 1313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/actor.py
+60 -21
View File
@@ -18,6 +18,7 @@ import ray.experimental.state as state
# the worker is currently allowed to use.
gpu_ids = []
def get_gpu_ids():
"""Get the IDs of the GPU that are available to the worker.
@@ -26,12 +27,15 @@ def get_gpu_ids():
"""
return gpu_ids
def random_string():
return np.random.bytes(20)
def random_actor_id():
return ray.local_scheduler.ObjectID(random_string())
def get_actor_method_function_id(attr):
"""Get the function ID corresponding to an actor method.
@@ -47,10 +51,14 @@ def get_actor_method_function_id(attr):
assert len(function_id) == 20
return ray.local_scheduler.ObjectID(function_id)
def fetch_and_register_actor(key, worker):
"""Import an actor."""
driver_id, actor_id_str, actor_name, module, pickled_class, assigned_gpu_ids, actor_method_names = \
worker.redis_client.hmget(key, ["driver_id", "actor_id", "name", "module", "class", "gpu_ids", "actor_method_names"])
(driver_id, actor_id_str, actor_name,
module, pickled_class, assigned_gpu_ids,
actor_method_names) = worker.redis_client.hmget(
key, ["driver_id", "actor_id", "name", "module", "class", "gpu_ids",
"actor_method_names"])
actor_id = ray.local_scheduler.ObjectID(actor_id_str)
actor_name = actor_name.decode("ascii")
module = module.decode("ascii")
@@ -64,12 +72,14 @@ def fetch_and_register_actor(key, worker):
class TemporaryActor(object):
pass
worker.actors[actor_id_str] = TemporaryActor()
def temporary_actor_method(*xs):
raise Exception("The actor with name {} failed to be imported, and so "
"cannot execute this method".format(actor_name))
for actor_method_name in actor_method_names:
function_id = get_actor_method_function_id(actor_method_name).id()
worker.functions[driver_id][function_id] = (actor_method_name, temporary_actor_method)
worker.functions[driver_id][function_id] = (actor_method_name,
temporary_actor_method)
try:
unpickled_class = pickling.loads(pickled_class)
@@ -84,11 +94,15 @@ def fetch_and_register_actor(key, worker):
# TODO(pcm): Why is the below line necessary?
unpickled_class.__module__ = module
worker.actors[actor_id_str] = unpickled_class.__new__(unpickled_class)
for (k, v) in inspect.getmembers(unpickled_class, predicate=(lambda x: inspect.isfunction(x) or inspect.ismethod(x))):
for (k, v) in inspect.getmembers(
unpickled_class, predicate=(lambda x: (inspect.isfunction(x) or
inspect.ismethod(x)))):
function_id = get_actor_method_function_id(k).id()
worker.functions[driver_id][function_id] = (k, v)
# We do not set worker.function_properties[driver_id][function_id] because
# we currently do need the actor worker to submit new tasks for the actor.
# We do not set worker.function_properties[driver_id][function_id]
# because we currently do need the actor worker to submit new tasks for
# the actor.
def select_local_scheduler(local_schedulers, num_gpus, worker):
"""Select a local scheduler to assign this actor to.
@@ -119,15 +133,19 @@ def select_local_scheduler(local_schedulers, num_gpus, worker):
# Loop through all of the local schedulers.
for local_scheduler in local_schedulers:
# See if there are enough available GPUs on this local scheduler.
local_scheduler_total_gpus = int(float(local_scheduler[b"num_gpus"].decode("ascii")))
gpus_in_use = worker.redis_client.hget(local_scheduler[b"ray_client_id"], b"gpus_in_use")
local_scheduler_total_gpus = int(float(
local_scheduler[b"num_gpus"].decode("ascii")))
gpus_in_use = worker.redis_client.hget(local_scheduler[b"ray_client_id"],
b"gpus_in_use")
gpus_in_use = 0 if gpus_in_use is None else int(gpus_in_use)
if gpus_in_use + num_gpus <= local_scheduler_total_gpus:
# Attempt to reserve some GPUs for this actor.
new_gpus_in_use = worker.redis_client.hincrby(local_scheduler[b"ray_client_id"], b"gpus_in_use", num_gpus)
new_gpus_in_use = worker.redis_client.hincrby(
local_scheduler[b"ray_client_id"], b"gpus_in_use", num_gpus)
if new_gpus_in_use > local_scheduler_total_gpus:
# If we failed to reserve the GPUs, undo the increment.
worker.redis_client.hincrby(local_scheduler[b"ray_client_id"], b"gpus_in_use", num_gpus)
worker.redis_client.hincrby(local_scheduler[b"ray_client_id"],
b"gpus_in_use", num_gpus)
else:
# We succeeded at reserving the GPUs, so we are done.
local_scheduler_id = local_scheduler[b"ray_client_id"]
@@ -135,10 +153,13 @@ def select_local_scheduler(local_schedulers, num_gpus, worker):
break
if local_scheduler_id is None:
raise Exception("Could not find a node with enough GPUs to create this "
"actor. The local scheduler information is {}.".format(local_schedulers))
"actor. The local scheduler information is {}."
.format(local_schedulers))
return local_scheduler_id, gpu_ids
def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus, worker):
def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus,
worker):
"""Export an actor to redis.
Args:
@@ -158,13 +179,16 @@ def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus, worker
driver_id = worker.task_driver_id.id()
for actor_method_name in actor_method_names:
function_id = get_actor_method_function_id(actor_method_name).id()
worker.function_properties[driver_id][function_id] = (1, num_cpus, num_gpus)
worker.function_properties[driver_id][function_id] = (1, num_cpus,
num_gpus)
# Select a local scheduler for the actor.
local_schedulers = state.get_local_schedulers(worker)
local_scheduler_id, gpu_ids = select_local_scheduler(local_schedulers, num_gpus, worker)
local_scheduler_id, gpu_ids = select_local_scheduler(local_schedulers,
num_gpus, worker)
worker.redis_client.publish("actor_notifications", actor_id.id() + local_scheduler_id)
worker.redis_client.publish("actor_notifications",
actor_id.id() + local_scheduler_id)
d = {"driver_id": driver_id,
"actor_id": actor_id.id(),
@@ -176,6 +200,7 @@ def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus, worker
worker.redis_client.hmset(key, d)
worker.redis_client.rpush("Exports", key)
def actor(*args, **kwargs):
def make_actor_decorator(num_cpus=1, num_gpus=0):
def make_actor(Class):
@@ -189,7 +214,8 @@ def actor(*args, **kwargs):
raise Exception("Actors currently do not support **kwargs.")
function_id = get_actor_method_function_id(attr)
# TODO(pcm): Extend args with keyword args.
object_ids = ray.worker.global_worker.submit_task(function_id, "", args,
object_ids = ray.worker.global_worker.submit_task(function_id, "",
args,
actor_id=actor_id)
if len(object_ids) == 1:
return object_ids[0]
@@ -199,24 +225,34 @@ def actor(*args, **kwargs):
class NewClass(object):
def __init__(self, *args, **kwargs):
self._ray_actor_id = random_actor_id()
self._ray_actor_methods = {k: v for (k, v) in inspect.getmembers(Class, predicate=(lambda x: inspect.isfunction(x) or inspect.ismethod(x)))}
export_actor(self._ray_actor_id, Class, self._ray_actor_methods.keys(), num_cpus, num_gpus, ray.worker.global_worker)
self._ray_actor_methods = {
k: v for (k, v) in inspect.getmembers(
Class, predicate=(lambda x: (inspect.isfunction(x) or
inspect.ismethod(x))))}
export_actor(self._ray_actor_id, Class,
self._ray_actor_methods.keys(), num_cpus, num_gpus,
ray.worker.global_worker)
# Call __init__ as a remote function.
if "__init__" in self._ray_actor_methods.keys():
actor_method_call(self._ray_actor_id, "__init__", *args, **kwargs)
else:
print("WARNING: this object has no __init__ method.")
# Make tab completion work.
def __dir__(self):
return self._ray_actor_methods
def __getattribute__(self, attr):
# The following is needed so we can still access self.actor_methods.
if attr in ["_ray_actor_id", "_ray_actor_methods"]:
return super(NewClass, self).__getattribute__(attr)
if attr in self._ray_actor_methods.keys():
return lambda *args, **kwargs: actor_method_call(self._ray_actor_id, attr, *args, **kwargs)
return lambda *args, **kwargs: actor_method_call(
self._ray_actor_id, attr, *args, **kwargs)
# There is no method with this name, so raise an exception.
raise AttributeError("'{}' Actor object has no attribute '{}'".format(Class, attr))
raise AttributeError("'{}' Actor object has no attribute '{}'"
.format(Class, attr))
def __repr__(self):
return "Actor(" + self._ray_actor_id.hex() + ")"
@@ -230,7 +266,9 @@ def actor(*args, **kwargs):
return make_actor_decorator(num_cpus=1, num_gpus=0)(Class)
# In this case, the actor decorator is something like @ray.actor(num_gpus=1).
if len(args) == 0 and len(kwargs) > 0 and all([key in ["num_cpus", "num_gpus"] for key in kwargs.keys()]):
if len(args) == 0 and len(kwargs) > 0 and all([key
in ["num_cpus", "num_gpus"]
for key in kwargs.keys()]):
num_cpus = kwargs["num_cpus"] if "num_cpus" in kwargs.keys() else 1
num_gpus = kwargs["num_gpus"] if "num_gpus" in kwargs.keys() else 0
return make_actor_decorator(num_cpus=num_cpus, num_gpus=num_gpus)
@@ -240,4 +278,5 @@ def actor(*args, **kwargs):
"some of the arguments 'num_cpus' or 'num_gpus' as in "
"'ray.actor(num_gpus=1)'.")
ray.worker.global_worker.fetch_and_register["Actor"] = fetch_and_register_actor