Add option of load_code_from_local which is required in cross-language ray call. (#3675)

This commit is contained in:
Yuhong Guo
2019-02-21 12:37:17 +08:00
committed by Hao Chen
parent e3066d1fa5
commit 1f864a02bc
9 changed files with 270 additions and 105 deletions
+13 -4
View File
@@ -152,6 +152,7 @@ class Worker(object):
# A dictionary that maps from driver id to SerializationContext
# TODO: clean up the SerializationContext once the job finished.
self.serialization_context_map = {}
self.load_code_from_local = False
self.function_actor_manager = FunctionActorManager(self)
# Identity of the driver that this worker is processing.
# It is a DriverID.
@@ -915,8 +916,9 @@ class Worker(object):
assert self.actor_id.is_nil()
self.actor_id = task.actor_creation_id()
self.actor_creation_task_id = task.task_id()
self.function_actor_manager.load_actor(driver_id,
function_descriptor)
actor_class = self.function_actor_manager.load_actor_class(
driver_id, function_descriptor)
self.actors[self.actor_id] = actor_class.__new__(actor_class)
self.actor_checkpoint_info[self.actor_id] = ActorCheckpointInfo(
num_tasks_since_last_checkpoint=0,
last_checkpoint_timestamp=int(1000 * time.time()),
@@ -1271,6 +1273,7 @@ def init(redis_address=None,
plasma_store_socket_name=None,
raylet_socket_name=None,
temp_dir=None,
load_code_from_local=False,
_internal_config=None):
"""Connect to an existing Ray cluster or start one and connect to it.
@@ -1346,6 +1349,8 @@ def init(redis_address=None,
used by the raylet process.
temp_dir (str): If provided, it will specify the root temporary
directory for the Ray process.
load_code_from_local: Whether code should be loaded from a local module
or from the GCS.
_internal_config (str): JSON configuration for overriding
RayConfig defaults. For testing purposes ONLY.
@@ -1427,6 +1432,7 @@ def init(redis_address=None,
plasma_store_socket_name=plasma_store_socket_name,
raylet_socket_name=raylet_socket_name,
temp_dir=temp_dir,
load_code_from_local=load_code_from_local,
_internal_config=_internal_config,
)
# Start the Ray processes. We set shutdown_at_exit=False because we
@@ -1516,7 +1522,8 @@ def init(redis_address=None,
mode=driver_mode,
log_to_driver=log_to_driver,
worker=global_worker,
driver_id=driver_id)
driver_id=driver_id,
load_code_from_local=load_code_from_local)
for hook in _post_init_hooks:
hook()
@@ -1745,7 +1752,8 @@ def connect(info,
mode=WORKER_MODE,
log_to_driver=False,
worker=global_worker,
driver_id=None):
driver_id=None,
load_code_from_local=False):
"""Connect this worker to the local scheduler, to Plasma, and to Redis.
Args:
@@ -1802,6 +1810,7 @@ def connect(info,
worker.actor_id = ActorID.nil()
worker.connected = True
worker.set_mode(mode)
worker.load_code_from_local = load_code_from_local
# If running Ray in LOCAL_MODE, there is no need to create call
# create_worker or to start the worker service.