mirror of
https://github.com/wassname/ray.git
synced 2026-07-09 13:25:22 +08:00
Treat actor creation like a regular task. (#1668)
* Treat actor creation like a regular task. * Small cleanups. * Change semantics of actor resource handling. * Bug fix. * Minor linting * Bug fix * Fix jenkins test. * Fix actor tests * Some cleanups * Bug fix * Fix bug. * Remove cached actor tasks when a driver is removed. * Add more info to taskspec in global state API. * Fix cyclic import bug in tune. * Fix * Fix linting. * Fix linting. * Don't schedule any tasks (especially actor creaiton tasks) on local schedulers with 0 CPUs. * Bug fix. * Add test for 0 CPU case * Fix linting * Address comments. * Fix typos and add comment. * Add assertion and fix test.
This commit is contained in:
Robert Nishihara
committed by
Stephanie Wang
Stephanie Wang
parent
3c080f4baa
commit
96913be939
+130
-47
@@ -49,6 +49,7 @@ NIL_ID = 20 * b"\xff"
|
||||
NIL_LOCAL_SCHEDULER_ID = NIL_ID
|
||||
NIL_FUNCTION_ID = NIL_ID
|
||||
NIL_ACTOR_ID = NIL_ID
|
||||
NIL_ACTOR_HANDLE_ID = NIL_ID
|
||||
|
||||
# This must be kept in sync with the `error_types` array in
|
||||
# common/state/error_table.h.
|
||||
@@ -58,6 +59,19 @@ PUT_RECONSTRUCTION_ERROR_TYPE = b"put_reconstruction"
|
||||
# This must be kept in sync with the `scheduling_state` enum in common/task.h.
|
||||
TASK_STATUS_RUNNING = 8
|
||||
|
||||
# Default resource requirements for remote functions.
|
||||
DEFAULT_REMOTE_FUNCTION_CPUS = 1
|
||||
DEFAULT_REMOTE_FUNCTION_GPUS = 0
|
||||
# Default resource requirements for actors when no resource requirements are
|
||||
# specified.
|
||||
DEFAULT_ACTOR_METHOD_CPUS_SIMPLE_CASE = 1
|
||||
DEFAULT_ACTOR_CREATION_CPUS_SIMPLE_CASE = 0
|
||||
# Default resource requirements for actors when some resource requirements are
|
||||
# specified.
|
||||
DEFAULT_ACTOR_METHOD_CPUS_SPECIFIED_CASE = 0
|
||||
DEFAULT_ACTOR_CREATION_CPUS_SPECIFIED_CASE = 1
|
||||
DEFAULT_ACTOR_CREATION_GPUS_SPECIFIED_CASE = 0
|
||||
|
||||
|
||||
class FunctionID(object):
|
||||
def __init__(self, function_id):
|
||||
@@ -222,6 +236,10 @@ class Worker(object):
|
||||
self.make_actor = None
|
||||
self.actors = {}
|
||||
self.actor_task_counter = 0
|
||||
# A set of all of the actor class keys that have been imported by the
|
||||
# import thread. It is safe to convert this worker into an actor of
|
||||
# these types.
|
||||
self.imported_actor_classes = set()
|
||||
# The number of threads Plasma should use when putting an object in the
|
||||
# object store.
|
||||
self.memcopy_threads = 12
|
||||
@@ -358,7 +376,8 @@ class Worker(object):
|
||||
# and make sure that the objects are in fact the same. We also
|
||||
# should return an error code to the caller instead of printing a
|
||||
# message.
|
||||
print("This object already exists in the object store.")
|
||||
print("The object with ID {} already exists in the object store."
|
||||
.format(object_id))
|
||||
|
||||
def retrieve_and_deserialize(self, object_ids, timeout, error_timeout=10):
|
||||
start_time = time.time()
|
||||
@@ -485,7 +504,8 @@ class Worker(object):
|
||||
|
||||
def submit_task(self, function_id, args, actor_id=None,
|
||||
actor_handle_id=None, actor_counter=0,
|
||||
is_actor_checkpoint_method=False,
|
||||
is_actor_checkpoint_method=False, actor_creation_id=None,
|
||||
actor_creation_dummy_object_id=None,
|
||||
execution_dependencies=None):
|
||||
"""Submit a remote task to the scheduler.
|
||||
|
||||
@@ -502,15 +522,33 @@ class Worker(object):
|
||||
actor_counter: The counter of the actor task.
|
||||
is_actor_checkpoint_method: True if this is an actor checkpoint
|
||||
task and false otherwise.
|
||||
actor_creation_id: The ID of the actor to create, if this is an
|
||||
actor creation task.
|
||||
actor_creation_dummy_object_id: If this task is an actor method,
|
||||
then this argument is the dummy object ID associated with the
|
||||
actor creation task for the corresponding actor.
|
||||
execution_dependencies: The execution dependencies for this task.
|
||||
|
||||
Returns:
|
||||
The return object IDs for this task.
|
||||
"""
|
||||
with log_span("ray:submit_task", worker=self):
|
||||
check_main_thread()
|
||||
if actor_id is None:
|
||||
assert actor_handle_id is None
|
||||
actor_id = ray.local_scheduler.ObjectID(NIL_ACTOR_ID)
|
||||
actor_handle_id = ray.local_scheduler.ObjectID(NIL_ACTOR_ID)
|
||||
actor_handle_id = ray.local_scheduler.ObjectID(
|
||||
NIL_ACTOR_HANDLE_ID)
|
||||
else:
|
||||
assert actor_handle_id is not None
|
||||
|
||||
if actor_creation_id is None:
|
||||
actor_creation_id = ray.local_scheduler.ObjectID(NIL_ACTOR_ID)
|
||||
|
||||
if actor_creation_dummy_object_id is None:
|
||||
actor_creation_dummy_object_id = (
|
||||
ray.local_scheduler.ObjectID(NIL_ID))
|
||||
|
||||
# Put large or complex arguments that are passed by value in the
|
||||
# object store first.
|
||||
args_for_local_scheduler = []
|
||||
@@ -541,6 +579,8 @@ class Worker(object):
|
||||
function_properties.num_return_vals,
|
||||
self.current_task_id,
|
||||
self.task_index,
|
||||
actor_creation_id,
|
||||
actor_creation_dummy_object_id,
|
||||
actor_id,
|
||||
actor_handle_id,
|
||||
actor_counter,
|
||||
@@ -801,6 +841,29 @@ class Worker(object):
|
||||
data={"function_id": function_id.id(),
|
||||
"function_name": function_name})
|
||||
|
||||
def _become_actor(self, task):
|
||||
"""Turn this worker into an actor.
|
||||
|
||||
Args:
|
||||
task: The actor creation task.
|
||||
"""
|
||||
assert self.actor_id == NIL_ACTOR_ID
|
||||
arguments = task.arguments()
|
||||
assert len(arguments) == 1
|
||||
self.actor_id = task.actor_creation_id().id()
|
||||
class_id = arguments[0]
|
||||
|
||||
key = b"ActorClass:" + class_id
|
||||
|
||||
# Wait for the actor class key to have been imported by the import
|
||||
# thread. TODO(rkn): It shouldn't be possible to end up in an infinite
|
||||
# loop here, but we should push an error to the driver if too much time
|
||||
# is spent here.
|
||||
while key not in self.imported_actor_classes:
|
||||
time.sleep(0.001)
|
||||
|
||||
self.fetch_and_register_actor(key, task.required_resources(), self)
|
||||
|
||||
def _wait_for_and_process_task(self, task):
|
||||
"""Wait for a task to be ready and process the task.
|
||||
|
||||
@@ -808,6 +871,14 @@ class Worker(object):
|
||||
task: The task to execute.
|
||||
"""
|
||||
function_id = task.function_id()
|
||||
|
||||
# TODO(rkn): It would be preferable for actor creation tasks to share
|
||||
# more of the code path with regular task execution.
|
||||
if (task.actor_creation_id() !=
|
||||
ray.local_scheduler.ObjectID(NIL_ACTOR_ID)):
|
||||
self._become_actor(task)
|
||||
return
|
||||
|
||||
# Wait until the function to be executed has actually been registered
|
||||
# on this worker. We will push warnings to the user if we spend too
|
||||
# long in this loop.
|
||||
@@ -1379,7 +1450,7 @@ def _init(address_info=None,
|
||||
address_info["local_scheduler_socket_names"][0]),
|
||||
"webui_url": address_info["webui_url"]}
|
||||
connect(driver_address_info, object_id_seed=object_id_seed,
|
||||
mode=driver_mode, worker=global_worker, actor_id=NIL_ACTOR_ID)
|
||||
mode=driver_mode, worker=global_worker)
|
||||
return address_info
|
||||
|
||||
|
||||
@@ -1678,13 +1749,10 @@ def import_thread(worker, mode):
|
||||
elif key.startswith(b"FunctionsToRun"):
|
||||
fetch_and_execute_function_to_run(key, worker=worker)
|
||||
elif key.startswith(b"ActorClass"):
|
||||
# If this worker is an actor that is supposed to construct this
|
||||
# class, fetch the actor and class information and construct
|
||||
# the class.
|
||||
class_id = key.split(b":", 1)[1]
|
||||
if (worker.actor_id != NIL_ACTOR_ID and
|
||||
worker.class_id == class_id):
|
||||
worker.fetch_and_register_actor(key, worker)
|
||||
# Keep track of the fact that this actor class has been
|
||||
# exported so that we know it is safe to turn this worker into
|
||||
# an actor of that class.
|
||||
worker.imported_actor_classes.add(key)
|
||||
else:
|
||||
raise Exception("This code should be unreachable.")
|
||||
|
||||
@@ -1721,12 +1789,14 @@ def import_thread(worker, mode):
|
||||
worker=worker):
|
||||
fetch_and_execute_function_to_run(key,
|
||||
worker=worker)
|
||||
elif key.startswith(b"Actor"):
|
||||
# Only get the actor if the actor ID matches the actor
|
||||
# ID of this worker.
|
||||
actor_id, = worker.redis_client.hmget(key, "actor_id")
|
||||
if worker.actor_id == actor_id:
|
||||
worker.fetch_and_register["Actor"](key, worker)
|
||||
elif key.startswith(b"ActorClass"):
|
||||
# Keep track of the fact that this actor class has been
|
||||
# exported so that we know it is safe to turn this
|
||||
# worker into an actor of that class.
|
||||
worker.imported_actor_classes.add(key)
|
||||
|
||||
# TODO(rkn): We may need to bring back the case of fetching
|
||||
# actor classes here.
|
||||
else:
|
||||
raise Exception("This code should be unreachable.")
|
||||
except redis.ConnectionError:
|
||||
@@ -1735,8 +1805,7 @@ def import_thread(worker, mode):
|
||||
pass
|
||||
|
||||
|
||||
def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
|
||||
actor_id=NIL_ACTOR_ID):
|
||||
def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker):
|
||||
"""Connect this worker to the local scheduler, to Plasma, and to Redis.
|
||||
|
||||
Args:
|
||||
@@ -1746,8 +1815,6 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
|
||||
deterministic.
|
||||
mode: The mode of the worker. One of SCRIPT_MODE, WORKER_MODE,
|
||||
PYTHON_MODE, and SILENT_MODE.
|
||||
actor_id: The ID of the actor running on this worker. If this worker is
|
||||
not an actor, then this is NIL_ACTOR_ID.
|
||||
"""
|
||||
check_main_thread()
|
||||
# Do some basic checking to make sure we didn't call ray.init twice.
|
||||
@@ -1757,7 +1824,9 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
|
||||
assert worker.cached_remote_functions_and_actors is not None, error_message
|
||||
# Initialize some fields.
|
||||
worker.worker_id = random_string()
|
||||
worker.actor_id = actor_id
|
||||
# All workers start out as non-actors. A worker can be turned into an actor
|
||||
# after it is created.
|
||||
worker.actor_id = NIL_ACTOR_ID
|
||||
worker.connected = True
|
||||
worker.set_mode(mode)
|
||||
# The worker.events field is used to aggregate logging information and
|
||||
@@ -1854,15 +1923,8 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
|
||||
worker.plasma_client = plasma.connect(info["store_socket_name"],
|
||||
info["manager_socket_name"],
|
||||
64)
|
||||
# Create the local scheduler client.
|
||||
if worker.actor_id != NIL_ACTOR_ID:
|
||||
num_gpus = int(worker.redis_client.hget(b"Actor:" + actor_id,
|
||||
"num_gpus"))
|
||||
else:
|
||||
num_gpus = 0
|
||||
worker.local_scheduler_client = ray.local_scheduler.LocalSchedulerClient(
|
||||
info["local_scheduler_socket_name"], worker.worker_id, worker.actor_id,
|
||||
is_worker, num_gpus)
|
||||
info["local_scheduler_socket_name"], worker.worker_id, is_worker)
|
||||
|
||||
# If this is a driver, set the current task ID, the task driver ID, and set
|
||||
# the task index to 0.
|
||||
@@ -1906,6 +1968,8 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
|
||||
worker.task_index,
|
||||
ray.local_scheduler.ObjectID(NIL_ACTOR_ID),
|
||||
ray.local_scheduler.ObjectID(NIL_ACTOR_ID),
|
||||
ray.local_scheduler.ObjectID(NIL_ACTOR_ID),
|
||||
ray.local_scheduler.ObjectID(NIL_ACTOR_ID),
|
||||
nil_actor_counter,
|
||||
False,
|
||||
[],
|
||||
@@ -1923,12 +1987,6 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
|
||||
# driver task.
|
||||
worker.current_task_id = driver_task.task_id()
|
||||
|
||||
# If this is an actor, get the ID of the corresponding class for the actor.
|
||||
if worker.actor_id != NIL_ACTOR_ID:
|
||||
actor_key = b"Actor:" + worker.actor_id
|
||||
class_id = worker.redis_client.hget(actor_key, "class_id")
|
||||
worker.class_id = class_id
|
||||
|
||||
# Initialize the serialization library. This registers some classes, and so
|
||||
# it must be run before we export all of the cached remote functions.
|
||||
_initialize_serialization()
|
||||
@@ -2457,10 +2515,16 @@ def remote(*args, **kwargs):
|
||||
"""
|
||||
worker = global_worker
|
||||
|
||||
def make_remote_decorator(num_return_vals, resources, max_calls,
|
||||
checkpoint_interval, func_id=None):
|
||||
def make_remote_decorator(num_return_vals, num_cpus, num_gpus, resources,
|
||||
max_calls, checkpoint_interval, func_id=None):
|
||||
def remote_decorator(func_or_class):
|
||||
if inspect.isfunction(func_or_class) or is_cython(func_or_class):
|
||||
# Set the remote function default resources.
|
||||
resources["CPU"] = (DEFAULT_REMOTE_FUNCTION_CPUS
|
||||
if num_cpus is None else num_cpus)
|
||||
resources["GPU"] = (DEFAULT_REMOTE_FUNCTION_GPUS
|
||||
if num_gpus is None else num_gpus)
|
||||
|
||||
function_properties = FunctionProperties(
|
||||
num_return_vals=num_return_vals,
|
||||
resources=resources,
|
||||
@@ -2468,8 +2532,28 @@ def remote(*args, **kwargs):
|
||||
return remote_function_decorator(func_or_class,
|
||||
function_properties)
|
||||
if inspect.isclass(func_or_class):
|
||||
# Set the actor default resources.
|
||||
if num_cpus is None and num_gpus is None and resources == {}:
|
||||
# In the default case, actors acquire no resources for
|
||||
# their lifetime, and actor methods will require 1 CPU.
|
||||
resources["CPU"] = DEFAULT_ACTOR_CREATION_CPUS_SIMPLE_CASE
|
||||
actor_method_cpus = DEFAULT_ACTOR_METHOD_CPUS_SIMPLE_CASE
|
||||
else:
|
||||
# If any resources are specified, then all resources are
|
||||
# acquired for the actor's lifetime and no resources are
|
||||
# associated with methods.
|
||||
resources["CPU"] = (
|
||||
DEFAULT_ACTOR_CREATION_CPUS_SPECIFIED_CASE
|
||||
if num_cpus is None else num_cpus)
|
||||
resources["GPU"] = (
|
||||
DEFAULT_ACTOR_CREATION_GPUS_SPECIFIED_CASE
|
||||
if num_gpus is None else num_gpus)
|
||||
actor_method_cpus = (
|
||||
DEFAULT_ACTOR_METHOD_CPUS_SPECIFIED_CASE)
|
||||
|
||||
return worker.make_actor(func_or_class, resources,
|
||||
checkpoint_interval)
|
||||
checkpoint_interval,
|
||||
actor_method_cpus)
|
||||
raise Exception("The @ray.remote decorator must be applied to "
|
||||
"either a function or to a class.")
|
||||
|
||||
@@ -2535,8 +2619,8 @@ def remote(*args, **kwargs):
|
||||
return remote_decorator
|
||||
|
||||
# Handle resource arguments
|
||||
num_cpus = kwargs["num_cpus"] if "num_cpus" in kwargs else 1
|
||||
num_gpus = kwargs["num_gpus"] if "num_gpus" in kwargs else 0
|
||||
num_cpus = kwargs["num_cpus"] if "num_cpus" in kwargs else None
|
||||
num_gpus = kwargs["num_gpus"] if "num_gpus" in kwargs else None
|
||||
resources = kwargs.get("resources", {})
|
||||
if not isinstance(resources, dict):
|
||||
raise Exception("The 'resources' keyword argument must be a "
|
||||
@@ -2544,8 +2628,6 @@ def remote(*args, **kwargs):
|
||||
.format(type(resources)))
|
||||
assert "CPU" not in resources, "Use the 'num_cpus' argument."
|
||||
assert "GPU" not in resources, "Use the 'num_gpus' argument."
|
||||
resources["CPU"] = num_cpus
|
||||
resources["GPU"] = num_gpus
|
||||
# Handle other arguments.
|
||||
num_return_vals = (kwargs["num_return_vals"] if "num_return_vals"
|
||||
in kwargs else 1)
|
||||
@@ -2556,13 +2638,14 @@ def remote(*args, **kwargs):
|
||||
if _mode() == WORKER_MODE:
|
||||
if "function_id" in kwargs:
|
||||
function_id = kwargs["function_id"]
|
||||
return make_remote_decorator(num_return_vals, resources, max_calls,
|
||||
return make_remote_decorator(num_return_vals, num_cpus, num_gpus,
|
||||
resources, max_calls,
|
||||
checkpoint_interval, function_id)
|
||||
|
||||
if len(args) == 1 and len(kwargs) == 0 and callable(args[0]):
|
||||
# This is the case where the decorator is just @ray.remote.
|
||||
return make_remote_decorator(
|
||||
num_return_vals, resources,
|
||||
num_return_vals, num_cpus, num_gpus, resources,
|
||||
max_calls, checkpoint_interval)(args[0])
|
||||
else:
|
||||
# This is the case where the decorator is something like
|
||||
@@ -2580,5 +2663,5 @@ def remote(*args, **kwargs):
|
||||
"resources", "max_calls",
|
||||
"checkpoint_interval"], error_string
|
||||
assert "function_id" not in kwargs
|
||||
return make_remote_decorator(num_return_vals, resources, max_calls,
|
||||
checkpoint_interval)
|
||||
return make_remote_decorator(num_return_vals, num_cpus, num_gpus,
|
||||
resources, max_calls, checkpoint_interval)
|
||||
|
||||
Reference in New Issue
Block a user