wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-20 12:40:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

[gRPC] Migrate gcs data structures to protobuf (#5024)

Browse Source
This commit is contained in:
Hao Chen
2019-06-25 14:31:19 -07:00
committed by Philipp Moritz
parent bd8aceb896
commit 0131353d42
52 changed files with 1465 additions and 1642 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/worker.py
+20 -20
View File
@@ -47,7 +47,7 @@ from ray import (
from ray import import_thread
from ray import profiling
from ray.core.generated.ErrorType import ErrorType
from ray.gcs_utils import ErrorType
from ray.exceptions import (
RayActorError,
RayError,
@@ -461,11 +461,11 @@ class Worker(object):
# Otherwise, return an exception object based on
# the error type.
error_type = int(metadata)
if error_type == ErrorType.WORKER_DIED:
if error_type == ErrorType.Value("WORKER_DIED"):
return RayWorkerError()
elif error_type == ErrorType.ACTOR_DIED:
elif error_type == ErrorType.Value("ACTOR_DIED"):
return RayActorError()
elif error_type == ErrorType.OBJECT_UNRECONSTRUCTABLE:
elif error_type == ErrorType.Value("OBJECT_UNRECONSTRUCTABLE"):
return UnreconstructableError(ray.ObjectID(object_id.binary()))
else:
assert False, "Unrecognized error type " + str(error_type)
@@ -1637,7 +1637,7 @@ def listen_error_messages_raylet(worker, task_error_queue, threads_stopped):
# Really we should just subscribe to the errors for this specific job.
# However, currently all errors seem to be published on the same channel.
error_pubsub_channel = str(
ray.gcs_utils.TablePubsub.ERROR_INFO).encode("ascii")
ray.gcs_utils.TablePubsub.Value("ERROR_INFO_PUBSUB")).encode("ascii")
worker.error_message_pubsub_client.subscribe(error_pubsub_channel)
# worker.error_message_pubsub_client.psubscribe("*")
@@ -1656,21 +1656,19 @@ def listen_error_messages_raylet(worker, task_error_queue, threads_stopped):
if msg is None:
threads_stopped.wait(timeout=0.01)
continue
gcs_entry = ray.gcs_utils.GcsEntry.GetRootAsGcsEntry(
msg["data"], 0)
assert gcs_entry.EntriesLength() == 1
error_data = ray.gcs_utils.ErrorTableData.GetRootAsErrorTableData(
gcs_entry.Entries(0), 0)
driver_id = error_data.DriverId()
gcs_entry = ray.gcs_utils.GcsEntry.FromString(msg["data"])
assert len(gcs_entry.entries) == 1
error_data = ray.gcs_utils.ErrorTableData.FromString(
gcs_entry.entries[0])
driver_id = error_data.driver_id
if driver_id not in [
worker.task_driver_id.binary(),
DriverID.nil().binary()
]:
continue
error_message = ray.utils.decode(error_data.ErrorMessage())
if (ray.utils.decode(
error_data.Type()) == ray_constants.TASK_PUSH_ERROR):
error_message = error_data.error_message
if (error_data.type == ray_constants.TASK_PUSH_ERROR):
# Delay it a bit to see if we can suppress it
task_error_queue.put((error_message, time.time()))
else:
@@ -1878,14 +1876,16 @@ def connect(node,
{}, # resource_map.
{}, # placement_resource_map.
)
task_table_data = ray.gcs_utils.TaskTableData()
task_table_data.task = driver_task._serialized_raylet_task()
# Add the driver task to the task table.
ray.state.state._execute_command(driver_task.task_id(),
"RAY.TABLE_ADD",
ray.gcs_utils.TablePrefix.RAYLET_TASK,
ray.gcs_utils.TablePubsub.RAYLET_TASK,
driver_task.task_id().binary(),
driver_task._serialized_raylet_task())
ray.state.state._execute_command(
driver_task.task_id(), "RAY.TABLE_ADD",
ray.gcs_utils.TablePrefix.Value("RAYLET_TASK"),
ray.gcs_utils.TablePubsub.Value("RAYLET_TASK_PUBSUB"),
driver_task.task_id().binary(),
task_table_data.SerializeToString())
# Set the driver's current task ID to the task ID assigned to the
# driver task.