wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 23:54:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Allow more fields for object metadata (#12484)

Browse Source
This commit is contained in:
Philipp Moritz
2020-11-29 21:50:18 -08:00
committed by GitHub
parent f1cc33a6a6
commit cf73ccddae
5 changed files with 36 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/_raylet.pyx
+3 -2
View File
@@ -292,13 +292,14 @@ cdef prepare_args(
else:
serialized_arg = worker.get_serialization_context().serialize(arg)
metadata = serialized_arg.metadata
metadata_fields = metadata.split(b",")
if language != Language.PYTHON:
if metadata not in [
if metadata_fields[0] not in [
ray_constants.OBJECT_METADATA_TYPE_CROSS_LANGUAGE,
ray_constants.OBJECT_METADATA_TYPE_RAW,
ray_constants.OBJECT_METADATA_TYPE_ACTOR_HANDLE]:
raise Exception("Can't transfer {} data to {}".format(
metadata, language))
metadata_fields[0], language))
size = serialized_arg.total_bytes
# TODO(edoakes): any objects containing ObjectRefs are spilled to
python/ray/ray_constants.py
+5
View File
@@ -178,6 +178,11 @@ WORKER_PROCESS_TYPE_RESTORE_WORKER = (
LOG_MONITOR_MAX_OPEN_FILES = 200
# The object metadata field uses the following format: It is a comma
# separated list of fields. The first field is mandatory and is the
# type of the object (see types below) or an integer, which is interpreted
# as an error value.
# A constant used as object metadata to indicate the object is cross language.
OBJECT_METADATA_TYPE_CROSS_LANGUAGE = b"XLANG"
# A constant used as object metadata to indicate the object is python specific.
python/ray/serialization.py
+11 -9
View File
@@ -219,10 +219,10 @@ class SerializationContext:
raise DeserializationError()
return obj
def _deserialize_msgpack_data(self, data, metadata):
def _deserialize_msgpack_data(self, data, metadata_fields):
msgpack_data, pickle5_data = split_buffer(data)
if metadata == ray_constants.OBJECT_METADATA_TYPE_PYTHON:
if metadata_fields[0] == ray_constants.OBJECT_METADATA_TYPE_PYTHON:
python_objects = self._deserialize_pickle5_data(pickle5_data)
else:
python_objects = []
@@ -240,23 +240,25 @@ class SerializationContext:
def _deserialize_object(self, data, metadata, object_ref):
if metadata:
if metadata in [
metadata_fields = metadata.split(b",")
if metadata_fields[0] in [
ray_constants.OBJECT_METADATA_TYPE_CROSS_LANGUAGE,
ray_constants.OBJECT_METADATA_TYPE_PYTHON
]:
return self._deserialize_msgpack_data(data, metadata)
return self._deserialize_msgpack_data(data, metadata_fields)
# Check if the object should be returned as raw bytes.
if metadata == ray_constants.OBJECT_METADATA_TYPE_RAW:
if metadata_fields[0] == ray_constants.OBJECT_METADATA_TYPE_RAW:
if data is None:
return b""
return data.to_pybytes()
elif metadata == ray_constants.OBJECT_METADATA_TYPE_ACTOR_HANDLE:
obj = self._deserialize_msgpack_data(data, metadata)
elif metadata_fields[
0] == ray_constants.OBJECT_METADATA_TYPE_ACTOR_HANDLE:
obj = self._deserialize_msgpack_data(data, metadata_fields)
return actor_handle_deserializer(obj)
# Otherwise, return an exception object based on
# the error type.
try:
error_type = int(metadata)
error_type = int(metadata_fields[0])
except Exception:
raise Exception(f"Can't deserialize object: {object_ref}, "
f"metadata: {metadata}")
@@ -265,7 +267,7 @@ class SerializationContext:
# TODO (kfstorm): exception serialization should be language
# independent.
if error_type == ErrorType.Value("TASK_EXECUTION_EXCEPTION"):
obj = self._deserialize_msgpack_data(data, metadata)
obj = self._deserialize_msgpack_data(data, metadata_fields)
return RayError.from_bytes(obj)
elif error_type == ErrorType.Value("WORKER_DIED"):
return WorkerCrashedError()