wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-18 12:40:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[xray] Implement Actor Reconstruction (#3332)

Browse Source 
* Implement Actor Reconstruction

* fix

* fix actor handle __del__

* fix lint

* add comment

* Remove actorCreationDummyObjectId

* address comments

* fix

* address comments

* avoid copy

* change log to debug

* fix error name
This commit is contained in:
Hao Chen
2018-12-13 21:28:58 -08:00
committed by Stephanie Wang
parent 2455de78ce
commit e7b51cbd1b
25 changed files with 779 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/actor.py
+16 -5
View File
@@ -271,12 +271,14 @@ class ActorClass(object):
each actor method.
"""
def __init__(self, modified_class, class_id, checkpoint_interval, num_cpus,
num_gpus, resources, actor_method_cpus):
def __init__(self, modified_class, class_id, checkpoint_interval,
max_reconstructions, num_cpus, num_gpus, resources,
actor_method_cpus):
self._modified_class = modified_class
self._class_id = class_id
self._class_name = modified_class.__name__
self._checkpoint_interval = checkpoint_interval
self._max_reconstructions = max_reconstructions
self._num_cpus = num_cpus
self._num_gpus = num_gpus
self._resources = resources
@@ -413,6 +415,7 @@ class ActorClass(object):
function_id,
creation_args,
actor_creation_id=actor_id,
max_actor_reconstructions=self._max_reconstructions,
num_return_vals=1,
resources=resources,
placement_resources=actor_placement_resources)
@@ -775,12 +778,19 @@ class ActorHandle(object):
def make_actor(cls, num_cpus, num_gpus, resources, actor_method_cpus,
checkpoint_interval):
checkpoint_interval, max_reconstructions):
if checkpoint_interval is None:
checkpoint_interval = -1
if max_reconstructions is None:
max_reconstructions = 0
if checkpoint_interval == 0:
raise Exception("checkpoint_interval must be greater than 0.")
if not (ray_constants.NO_RECONSTRUCTION <= max_reconstructions <=
ray_constants.INFINITE_RECONSTRUCTION):
raise Exception("max_reconstructions must be in range [%d, %d]." %
(ray_constants.NO_RECONSTRUCTION,
ray_constants.INFINITE_RECONSTRUCTION))
# Modify the class to have an additional method that will be used for
# terminating the worker.
@@ -872,8 +882,9 @@ def make_actor(cls, num_cpus, num_gpus, resources, actor_method_cpus,
class_id = _random_string()
return ActorClass(Class, class_id, checkpoint_interval, num_cpus, num_gpus,
resources, actor_method_cpus)
return ActorClass(Class, class_id, checkpoint_interval,
max_reconstructions, num_cpus, num_gpus, resources,
actor_method_cpus)
ray.worker.global_worker.make_actor = make_actor
python/ray/ray_constants.py
+5
View File
@@ -76,3 +76,8 @@ LOGGER_LEVEL = "info"
LOGGER_LEVEL_CHOICES = ['debug', 'info', 'warning', 'error', 'critical']
LOGGER_LEVEL_HELP = ("The logging level threshold, choices=['debug', 'info',"
" 'warning', 'error', 'critical'], default='info'")
# A constant indicating that an actor doesn't need reconstructions.
NO_RECONSTRUCTION = 0
# A constant indicating that an actor should be reconstructed infinite times.
INFINITE_RECONSTRUCTION = 2**30
python/ray/test/cluster_utils.py
+23 -8
View File
@@ -34,6 +34,7 @@ class Cluster(object):
self.head_node = None
self.worker_nodes = {}
self.redis_address = None
self.connected = False
if not initialize_head and connect:
raise RuntimeError("Cannot connect to uninitialized cluster.")
@@ -41,14 +42,19 @@ class Cluster(object):
head_node_args = head_node_args or {}
self.add_node(**head_node_args)
if connect:
redis_password = head_node_args.get("redis_password")
output_info = ray.init(
redis_address=self.redis_address,
redis_password=redis_password)
logger.info(output_info)
self.connect(head_node_args)
if shutdown_at_exit:
atexit.register(self.shutdown)
def connect(self, head_node_args):
assert self.redis_address is not None
assert not self.connected
redis_password = head_node_args.get("redis_password")
output_info = ray.init(
redis_address=self.redis_address, redis_password=redis_password)
logger.info(output_info)
self.connected = True
def add_node(self, **override_kwargs):
"""Adds a node to the local Ray Cluster.
@@ -83,7 +89,7 @@ class Cluster(object):
process_dict_copy = services.all_processes.copy()
for key in services.all_processes:
services.all_processes[key] = []
node = Node(process_dict_copy)
node = Node(address_info, process_dict_copy)
self.head_node = node
else:
address_info = services.start_ray_node(
@@ -93,7 +99,7 @@ class Cluster(object):
process_dict_copy = services.all_processes.copy()
for key in services.all_processes:
services.all_processes[key] = []
node = Node(process_dict_copy)
node = Node(address_info, process_dict_copy)
self.worker_nodes[node] = address_info
logger.info("Starting Node with raylet socket {}".format(
address_info["raylet_socket_names"]))
@@ -182,8 +188,9 @@ class Cluster(object):
class Node(object):
"""Abstraction for a Ray node."""
def __init__(self, process_dict):
def __init__(self, address_info, process_dict):
# TODO(rliaw): Is there a unique identifier for a node?
self.address_info = address_info
self.process_dict = process_dict
def kill_plasma_store(self):
@@ -224,3 +231,11 @@ class Node(object):
def all_processes_alive(self):
return not any(self.dead_processes())
def get_plasma_store_name(self):
"""Return the plasma store name.
Assuming one plasma store per raylet, this may be used as a unique
identifier for a node.
"""
return self.address_info['object_store_addresses'][0]
python/ray/worker.py
+23 -10
View File
@@ -525,6 +525,7 @@ class Worker(object):
is_actor_checkpoint_method=False,
actor_creation_id=None,
actor_creation_dummy_object_id=None,
max_actor_reconstructions=0,
execution_dependencies=None,
num_return_vals=None,
resources=None,
@@ -622,12 +623,12 @@ class Worker(object):
assert not self.current_task_id.is_nil()
# Submit the task to local scheduler.
task = ray.raylet.Task(
driver_id, ray.ObjectID(
function_id.id()), args_for_local_scheduler,
num_return_vals, self.current_task_id, task_index,
actor_creation_id, actor_creation_dummy_object_id, actor_id,
actor_handle_id, actor_counter, execution_dependencies,
resources, placement_resources)
driver_id, ray.ObjectID(function_id.id()),
args_for_local_scheduler, num_return_vals,
self.current_task_id, task_index, actor_creation_id,
actor_creation_dummy_object_id, max_actor_reconstructions,
actor_id, actor_handle_id, actor_counter,
execution_dependencies, resources, placement_resources)
self.raylet_client.submit_task(task)
return task.returns()
@@ -2098,7 +2099,7 @@ def connect(info,
worker.current_task_id,
worker.task_index,
ray.ObjectID(NIL_ACTOR_ID),
ray.ObjectID(NIL_ACTOR_ID),
ray.ObjectID(NIL_ACTOR_ID), 0,
ray.ObjectID(NIL_ACTOR_ID),
ray.ObjectID(NIL_ACTOR_ID),
nil_actor_counter, [], {"CPU": 0}, {})
@@ -2512,6 +2513,7 @@ def make_decorator(num_return_vals=None,
resources=None,
max_calls=None,
checkpoint_interval=None,
max_reconstructions=None,
worker=None):
def decorator(function_or_class):
if (inspect.isfunction(function_or_class)
@@ -2520,6 +2522,9 @@ def make_decorator(num_return_vals=None,
if checkpoint_interval is not None:
raise Exception("The keyword 'checkpoint_interval' is not "
"allowed for remote functions.")
if max_reconstructions is not None:
raise Exception("The keyword 'max_reconstructions' is not "
"allowed for remote functions.")
return ray.remote_function.RemoteFunction(
function_or_class, num_cpus, num_gpus, resources,
@@ -2549,7 +2554,7 @@ def make_decorator(num_return_vals=None,
return worker.make_actor(function_or_class, cpus_to_use, num_gpus,
resources, actor_method_cpus,
checkpoint_interval)
checkpoint_interval, max_reconstructions)
raise Exception("The @ray.remote decorator must be applied to "
"either a function or to a class.")
@@ -2591,6 +2596,11 @@ def remote(*args, **kwargs):
third-party libraries or to reclaim resources that cannot easily be
released, e.g., GPU memory that was acquired by TensorFlow). By
default this is infinite.
* **max_reconstructions**: Only for *actors*. This specifies the maximum
number of times that the actor should be reconstructed when it dies
unexpectedly. The minimum valid value is 0 (default), which indicates
that the actor doesn't need to be reconstructed. And the maximum valid
value is ray.ray_constants.INFINITE_RECONSTRUCTIONS.
This can be done as follows:
@@ -2616,14 +2626,15 @@ def remote(*args, **kwargs):
"with no arguments and no parentheses, for example "
"'@ray.remote', or it must be applied using some of "
"the arguments 'num_return_vals', 'num_cpus', 'num_gpus', "
"'resources', 'max_calls', or 'checkpoint_interval', like "
"'resources', 'max_calls', 'checkpoint_interval',"
"or 'max_reconstructions', like "
"'@ray.remote(num_return_vals=2, "
"resources={\"CustomResource\": 1})'.")
assert len(args) == 0 and len(kwargs) > 0, error_string
for key in kwargs:
assert key in [
"num_return_vals", "num_cpus", "num_gpus", "resources",
"max_calls", "checkpoint_interval"
"max_calls", "checkpoint_interval", "max_reconstructions"
], error_string
num_cpus = kwargs["num_cpus"] if "num_cpus" in kwargs else None
@@ -2641,6 +2652,7 @@ def remote(*args, **kwargs):
num_return_vals = kwargs.get("num_return_vals")
max_calls = kwargs.get("max_calls")
checkpoint_interval = kwargs.get("checkpoint_interval")
max_reconstructions = kwargs.get("max_reconstructions")
return make_decorator(
num_return_vals=num_return_vals,
@@ -2649,4 +2661,5 @@ def remote(*args, **kwargs):
resources=resources,
max_calls=max_calls,
checkpoint_interval=checkpoint_interval,
max_reconstructions=max_reconstructions,
worker=worker)