wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-09 22:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Change Python's ObjectID to ObjectRef (#9353)

Browse Source
This commit is contained in:
Hao Chen
2020-07-10 17:49:04 +08:00
committed by GitHub
parent 6311e5a947
commit d49dadf891
91 changed files with 959 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/includes/function_descriptor.pxi
+2 -2
View File
@@ -253,7 +253,7 @@ cdef class PythonFunctionDescriptor(FunctionDescriptor):
"""Get the function id calculated from this descriptor.
Returns:
The value of ray.ObjectID that represents the function id.
The value of ray.ObjectRef that represents the function id.
"""
if not self._function_id:
self._function_id = self._get_function_id()
@@ -266,7 +266,7 @@ cdef class PythonFunctionDescriptor(FunctionDescriptor):
descriptor.
Returns:
ray.ObjectID to represent the function descriptor.
ray.ObjectRef to represent the function descriptor.
"""
function_id_hash = hashlib.sha1()
# Include the function module and name in the hash.
python/ray/includes/global_state_accessor.pxd
+2 -2
View File
@@ -12,8 +12,8 @@ from ray.includes.unique_ids cimport (
cdef extern from "ray/gcs/gcs_client/global_state_accessor.h" nogil:
cdef cppclass CGlobalStateAccessor "ray::gcs::GlobalStateAccessor":
CGlobalStateAccessor(const c_string &redis_address,
const c_string &redis_password,
c_bool is_test)
const c_string &redis_password,
c_bool is_test)
c_bool Connect()
void Disconnect()
c_vector[c_string] GetAllJobInfo()
python/ray/includes/global_state_accessor.pxi
+17 -8
View File
@@ -16,12 +16,17 @@ cdef class GlobalStateAccessor:
cdef:
unique_ptr[CGlobalStateAccessor] inner
def __init__(self, redis_address, redis_password, c_bool is_test_client=False):
def __init__(self, redis_address, redis_password,
c_bool is_test_client=False):
if not redis_password:
redis_password = ""
self.inner.reset(
new CGlobalStateAccessor(redis_address.encode("ascii"),
redis_password.encode("ascii"), is_test_client))
new CGlobalStateAccessor(
redis_address.encode("ascii"),
redis_password.encode("ascii"),
is_test_client,
),
)
def connect(self):
return self.inner.get().Connect()
@@ -41,8 +46,9 @@ cdef class GlobalStateAccessor:
def get_object_table(self):
return self.inner.get().GetAllObjectInfo()
def get_object_info(self, object_id):
object_info = self.inner.get().GetObjectInfo(CObjectID.FromBinary(object_id.binary()))
def get_object_info(self, object_ref):
object_info = self.inner.get().GetObjectInfo(
CObjectID.FromBinary(object_ref.binary()))
if object_info:
return c_string(object_info.get().data(), object_info.get().size())
return None
@@ -51,19 +57,22 @@ cdef class GlobalStateAccessor:
return self.inner.get().GetAllActorInfo()
def get_actor_info(self, actor_id):
actor_info = self.inner.get().GetActorInfo(CActorID.FromBinary(actor_id.binary()))
actor_info = self.inner.get().GetActorInfo(
CActorID.FromBinary(actor_id.binary()))
if actor_info:
return c_string(actor_info.get().data(), actor_info.get().size())
return None
def get_node_resource_info(self, node_id):
return self.inner.get().GetNodeResourceInfo(CClientID.FromBinary(node_id.binary()))
return self.inner.get().GetNodeResourceInfo(
CClientID.FromBinary(node_id.binary()))
def get_worker_table(self):
return self.inner.get().GetAllWorkerInfo()
def get_worker_info(self, worker_id):
worker_info = self.inner.get().GetWorkerInfo(CWorkerID.FromBinary(worker_id.binary()))
worker_info = self.inner.get().GetWorkerInfo(
CWorkerID.FromBinary(worker_id.binary()))
if worker_info:
return c_string(worker_info.get().data(), worker_info.get().size())
return None
python/ray/includes/object_ref.pxi
+72
View File
@@ -0,0 +1,72 @@
from ray.includes.unique_ids cimport CObjectID
import ray
cdef class ObjectRef(BaseID):
def __init__(self, id):
check_id(id)
self.data = CObjectID.FromBinary(<c_string>id)
self.in_core_worker = False
worker = ray.worker.global_worker
# TODO(edoakes): We should be able to remove the in_core_worker flag.
# But there are still some dummy object refs being created outside the
# context of a core worker.
if hasattr(worker, "core_worker"):
worker.core_worker.add_object_ref_reference(self)
self.in_core_worker = True
def __dealloc__(self):
if self.in_core_worker:
try:
worker = ray.worker.global_worker
worker.core_worker.remove_object_ref_reference(self)
except Exception as e:
# There is a strange error in rllib that causes the above to
# fail. Somehow the global 'ray' variable corresponding to the
# imported package is None when this gets called. Unfortunately
# this is hard to debug because __dealloc__ is called during
# garbage collection so we can't get a good stack trace. In any
# case, there's not much we can do besides ignore it
# (re-importing ray won't help).
pass
cdef CObjectID native(self):
return <CObjectID>self.data
def size(self):
return CObjectID.Size()
def binary(self):
return self.data.Binary()
def hex(self):
return decode(self.data.Hex())
def is_nil(self):
return self.data.IsNil()
def task_id(self):
return TaskID(self.data.TaskId().Binary())
cdef size_t hash(self):
return self.data.Hash()
@classmethod
def nil(cls):
return cls(CObjectID.Nil().Binary())
@classmethod
def from_random(cls):
return cls(CObjectID.FromRandom().Binary())
def __await__(self):
# Delayed import because this can only be imported in py3.
from ray.async_compat import get_async
return get_async(self).__await__()
def as_future(self):
# Delayed import because this can only be imported in py3.
from ray.async_compat import get_async
return get_async(self)
python/ray/includes/serialization.pxi
+17 -12
View File
@@ -177,7 +177,8 @@ cdef class MessagePackSerializer(object):
raise Exception('Unrecognized ext type id: {}'.format(code))
try:
gc.disable() # Performance optimization for msgpack.
return msgpack.loads(s, ext_hook=_ext_hook, raw=False, strict_map_key=False)
return msgpack.loads(s, ext_hook=_ext_hook, raw=False,
strict_map_key=False)
finally:
gc.enable()
@@ -371,11 +372,11 @@ cdef class Pickle5Writer:
cdef class SerializedObject(object):
cdef:
object _metadata
object _contained_object_ids
object _contained_object_refs
def __init__(self, metadata, contained_object_ids=None):
def __init__(self, metadata, contained_object_refs=None):
self._metadata = metadata
self._contained_object_ids = contained_object_ids or []
self._contained_object_refs = contained_object_refs or []
@property
def total_bytes(self):
@@ -387,8 +388,8 @@ cdef class SerializedObject(object):
return self._metadata
@property
def contained_object_ids(self):
return self._contained_object_ids
def contained_object_refs(self):
return self._contained_object_refs
@cython.boundscheck(False)
@cython.wraparound(False)
@@ -404,9 +405,9 @@ cdef class Pickle5SerializedObject(SerializedObject):
object _total_bytes
def __init__(self, metadata, inband, Pickle5Writer writer,
contained_object_ids):
contained_object_refs):
super(Pickle5SerializedObject, self).__init__(metadata,
contained_object_ids)
contained_object_refs)
self.inband = inband
self.writer = writer
# cached total bytes
@@ -438,13 +439,17 @@ cdef class MessagePackSerializedObject(SerializedObject):
def __init__(self, metadata, msgpack_data,
SerializedObject nest_serialized_object=None):
if nest_serialized_object:
contained_object_ids = nest_serialized_object.contained_object_ids
contained_object_refs = (
nest_serialized_object.contained_object_refs
)
total_bytes = nest_serialized_object.total_bytes
else:
contained_object_ids = []
contained_object_refs = []
total_bytes = 0
super(MessagePackSerializedObject, self).__init__(metadata,
contained_object_ids)
super(MessagePackSerializedObject, self).__init__(
metadata,
contained_object_refs,
)
self.nest_serialized_object = nest_serialized_object
self.msgpack_header = msgpack_header = msgpack.dumps(len(msgpack_data))
self.msgpack_data = msgpack_data
python/ray/includes/unique_ids.pxi
+3 -69
View File
@@ -126,75 +126,6 @@ cdef class UniqueID(BaseID):
return self.data.Hash()
cdef class ObjectID(BaseID):
def __init__(self, id):
check_id(id)
self.data = CObjectID.FromBinary(<c_string>id)
self.in_core_worker = False
worker = ray.worker.global_worker
# TODO(edoakes): We should be able to remove the in_core_worker flag.
# But there are still some dummy object IDs being created outside the
# context of a core worker.
if hasattr(worker, "core_worker"):
worker.core_worker.add_object_id_reference(self)
self.in_core_worker = True
def __dealloc__(self):
if self.in_core_worker:
try:
worker = ray.worker.global_worker
worker.core_worker.remove_object_id_reference(self)
except Exception as e:
# There is a strange error in rllib that causes the above to
# fail. Somehow the global 'ray' variable corresponding to the
# imported package is None when this gets called. Unfortunately
# this is hard to debug because __dealloc__ is called during
# garbage collection so we can't get a good stack trace. In any
# case, there's not much we can do besides ignore it
# (re-importing ray won't help).
pass
cdef CObjectID native(self):
return <CObjectID>self.data
def size(self):
return CObjectID.Size()
def binary(self):
return self.data.Binary()
def hex(self):
return decode(self.data.Hex())
def is_nil(self):
return self.data.IsNil()
def task_id(self):
return TaskID(self.data.TaskId().Binary())
cdef size_t hash(self):
return self.data.Hash()
@classmethod
def nil(cls):
return cls(CObjectID.Nil().Binary())
@classmethod
def from_random(cls):
return cls(CObjectID.FromRandom().Binary())
def __await__(self):
# Delayed import because this can only be imported in py3.
from ray.async_compat import get_async
return get_async(self).__await__()
def as_future(self):
# Delayed import because this can only be imported in py3.
from ray.async_compat import get_async
return get_async(self)
cdef class TaskID(BaseID):
cdef CTaskID data
@@ -397,6 +328,9 @@ cdef class ActorClassID(UniqueID):
cdef CActorClassID native(self):
return <CActorClassID>self.data
# This type alias is for backward compatibility.
ObjectID = ObjectRef
_ID_TYPES = [
ActorCheckpointID,
ActorClassID,