wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 21:38:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Core] Ownership-based Object Directory - Added support for object spilling in the ownership-based object directory. (#13948)

Browse Source 
* Add support for object spilling in the ownership-based object directory.

* Move owner address hashmap into pinned_objects_ and objects_pending_spill_.

* Update local object manager tests.

* Feedback and misc. fixes.

* Move spilled unpin callback lambda to std::binded private method.

* Skip test_delete_objects_multi_node test on MacOS for now.
This commit is contained in:
Clark Zinzow
2021-02-11 11:36:22 -07:00
committed by GitHub
parent 4db86404ad
commit cd7e567a57
18 changed files with 615 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/_raylet.pxd
+2 -1
View File
@@ -101,7 +101,8 @@ cdef class CoreWorker:
cdef _create_put_buffer(self, shared_ptr[CBuffer] &metadata,
size_t data_size, ObjectRef object_ref,
c_vector[CObjectID] contained_ids,
CObjectID *c_object_id, shared_ptr[CBuffer] *data)
CObjectID *c_object_id, shared_ptr[CBuffer] *data,
owner_address=*)
cdef store_task_outputs(
self, worker, outputs, const c_vector[CObjectID] return_ids,
c_vector[shared_ptr[CRayObject]] *returns)
python/ray/_raylet.pyx
+20 -7
View File
@@ -628,7 +628,8 @@ cdef void gc_collect() nogil:
cdef c_vector[c_string] spill_objects_handler(
const c_vector[CObjectID]& object_ids_to_spill) nogil:
const c_vector[CObjectID]& object_ids_to_spill,
const c_vector[c_string]& owner_addresses) nogil:
cdef c_vector[c_string] return_urls
with gil:
object_refs = VectorToObjectRefs(object_ids_to_spill)
@@ -636,7 +637,8 @@ cdef c_vector[c_string] spill_objects_handler(
with ray.worker._changeproctitle(
ray_constants.WORKER_PROCESS_TYPE_SPILL_WORKER,
ray_constants.WORKER_PROCESS_TYPE_SPILL_WORKER_IDLE):
urls = external_storage.spill_objects(object_refs)
urls = external_storage.spill_objects(
object_refs, owner_addresses)
for url in urls:
return_urls.push_back(url)
except Exception:
@@ -930,7 +932,11 @@ cdef class CoreWorker:
cdef _create_put_buffer(self, shared_ptr[CBuffer] &metadata,
size_t data_size, ObjectRef object_ref,
c_vector[CObjectID] contained_ids,
CObjectID *c_object_id, shared_ptr[CBuffer] *data):
CObjectID *c_object_id, shared_ptr[CBuffer] *data,
owner_address=None):
cdef:
CAddress c_owner_address
if object_ref is None:
with nogil:
check_status(CCoreWorkerProcess.GetCoreWorker().CreateOwned(
@@ -938,11 +944,16 @@ cdef class CoreWorker:
c_object_id, data))
else:
c_object_id[0] = object_ref.native()
if owner_address is None:
c_owner_address = CCoreWorkerProcess.GetCoreWorker(
).GetRpcAddress()
else:
c_owner_address = CAddress()
c_owner_address.ParseFromString(owner_address)
with nogil:
check_status(CCoreWorkerProcess.GetCoreWorker().CreateExisting(
metadata, data_size, c_object_id[0],
CCoreWorkerProcess.GetCoreWorker().GetRpcAddress(),
data))
c_owner_address, data))
# If data is nullptr, that means the ObjectRef already existed,
# which we ignore.
@@ -951,7 +962,8 @@ cdef class CoreWorker:
return data.get() == NULL
def put_file_like_object(
self, metadata, data_size, file_like, ObjectRef object_ref):
self, metadata, data_size, file_like, ObjectRef object_ref,
owner_address):
"""Directly create a new Plasma Store object from a file like
object. This avoids extra memory copy.
@@ -961,6 +973,7 @@ cdef class CoreWorker:
file_like: A python file object that provides the `readinto`
interface.
object_ref: The new ObjectRef.
owner_address: Owner address for this object ref.
"""
cdef:
CObjectID c_object_id
@@ -975,7 +988,7 @@ cdef class CoreWorker:
object_already_exists = self._create_put_buffer(
metadata_buf, data_size, object_ref,
ObjectRefsToVector([]),
&c_object_id, &data_buf)
&c_object_id, &data_buf, owner_address)
if object_already_exists:
logger.debug("Object already exists in 'put_file_like_object'.")
return
python/ray/external_storage.py
+42 -19
View File
@@ -80,6 +80,8 @@ class ExternalStorage(metaclass=abc.ABCMeta):
the external storage is invalid.
"""
HEADER_LENGTH = 24
def _get_objects_from_store(self, object_refs):
worker = ray.worker.global_worker
# Since the object should always exist in the plasma store before
@@ -89,18 +91,21 @@ class ExternalStorage(metaclass=abc.ABCMeta):
ray_object_pairs = worker.core_worker.get_if_local(object_refs)
return ray_object_pairs
def _put_object_to_store(self, metadata, data_size, file_like, object_ref):
def _put_object_to_store(self, metadata, data_size, file_like, object_ref,
owner_address):
worker = ray.worker.global_worker
worker.core_worker.put_file_like_object(metadata, data_size, file_like,
object_ref)
object_ref, owner_address)
def _write_multiple_objects(self, f: IO, object_refs: List[ObjectRef],
owner_addresses: List[str],
url: str) -> List[str]:
"""Fuse all given objects into a given file handle.
Args:
f(IO): File handle to fusion all given object refs.
object_refs(list): Object references to fusion to a single file.
owner_addresses(list): Owner addresses for the provided objects.
url(str): url where the object ref is stored
in the external storage.
@@ -112,13 +117,18 @@ class ExternalStorage(metaclass=abc.ABCMeta):
keys = []
offset = 0
ray_object_pairs = self._get_objects_from_store(object_refs)
for ref, (buf, metadata) in zip(object_refs, ray_object_pairs):
for ref, (buf, metadata), owner_address in zip(
object_refs, ray_object_pairs, owner_addresses):
address_len = len(owner_address)
metadata_len = len(metadata)
buf_len = len(buf)
# 16 bytes to store metadata and buffer length.
data_size_in_bytes = metadata_len + buf_len + 16
# 24 bytes to store owner address, metadata, and buffer lengths.
data_size_in_bytes = (
address_len + metadata_len + buf_len + self.HEADER_LENGTH)
f.write(address_len.to_bytes(8, byteorder="little"))
f.write(metadata_len.to_bytes(8, byteorder="little"))
f.write(buf_len.to_bytes(8, byteorder="little"))
f.write(owner_address)
f.write(metadata)
f.write(memoryview(buf))
url_with_offset = create_url_with_offset(
@@ -127,7 +137,8 @@ class ExternalStorage(metaclass=abc.ABCMeta):
offset += data_size_in_bytes
return keys
def _size_check(self, metadata_len, buffer_len, obtained_data_size):
def _size_check(self, address_len, metadata_len, buffer_len,
obtained_data_size):
"""Check whether or not the obtained_data_size is as expected.
Args:
@@ -138,9 +149,11 @@ class ExternalStorage(metaclass=abc.ABCMeta):
Raises:
ValueError if obtained_data_size is different from
metadata_len + buffer_len + 16(first 8 bytes to store length).
address_len + metadata_len + buffer_len +
24 (first 8 bytes to store length).
"""
data_size_in_bytes = metadata_len + buffer_len + 16
data_size_in_bytes = (
address_len + metadata_len + buffer_len + self.HEADER_LENGTH)
if data_size_in_bytes != obtained_data_size:
raise ValueError(
f"Obtained data has a size of {data_size_in_bytes}, "
@@ -148,7 +161,7 @@ class ExternalStorage(metaclass=abc.ABCMeta):
f"size of {obtained_data_size}.")
@abc.abstractmethod
def spill_objects(self, object_refs) -> List[str]:
def spill_objects(self, object_refs, owner_addresses) -> List[str]:
"""Spill objects to the external storage. Objects are specified
by their object refs.
@@ -191,7 +204,7 @@ class ExternalStorage(metaclass=abc.ABCMeta):
class NullStorage(ExternalStorage):
"""The class that represents an uninitialized external storage."""
def spill_objects(self, object_refs) -> List[str]:
def spill_objects(self, object_refs, owner_addresses) -> List[str]:
raise NotImplementedError("External storage is not initialized")
def restore_spilled_objects(self, object_refs, url_with_offset_list):
@@ -220,7 +233,7 @@ class FileSystemStorage(ExternalStorage):
raise ValueError("The given directory path to store objects, "
f"{self.directory_path}, could not be created.")
def spill_objects(self, object_refs) -> List[str]:
def spill_objects(self, object_refs, owner_addresses) -> List[str]:
if len(object_refs) == 0:
return []
# Always use the first object ref as a key when fusioning objects.
@@ -228,7 +241,8 @@ class FileSystemStorage(ExternalStorage):
filename = f"{first_ref.hex()}-multi-{len(object_refs)}"
url = f"{os.path.join(self.directory_path, filename)}"
with open(url, "wb") as f:
return self._write_multiple_objects(f, object_refs, url)
return self._write_multiple_objects(f, object_refs,
owner_addresses, url)
def restore_spilled_objects(self, object_refs: List[ObjectRef],
url_with_offset_list: List[str]):
@@ -243,13 +257,17 @@ class FileSystemStorage(ExternalStorage):
# Read a part of the file and recover the object.
with open(base_url, "rb") as f:
f.seek(offset)
address_len = int.from_bytes(f.read(8), byteorder="little")
metadata_len = int.from_bytes(f.read(8), byteorder="little")
buf_len = int.from_bytes(f.read(8), byteorder="little")
self._size_check(metadata_len, buf_len, parsed_result.size)
self._size_check(address_len, metadata_len, buf_len,
parsed_result.size)
total += buf_len
owner_address = f.read(address_len)
metadata = f.read(metadata_len)
# read remaining data to our buffer
self._put_object_to_store(metadata, buf_len, f, object_ref)
self._put_object_to_store(metadata, buf_len, f, object_ref,
owner_address)
return total
def delete_spilled_objects(self, urls: List[str]):
@@ -320,7 +338,7 @@ class ExternalStorageSmartOpenImpl(ExternalStorage):
self.transport_params = {"defer_seek": True}
self.transport_params.update(self.override_transport_params)
def spill_objects(self, object_refs) -> List[str]:
def spill_objects(self, object_refs, owner_addresses) -> List[str]:
if len(object_refs) == 0:
return []
from smart_open import open
@@ -331,7 +349,8 @@ class ExternalStorageSmartOpenImpl(ExternalStorage):
with open(
url, "wb",
transport_params=self.transport_params) as file_like:
return self._write_multiple_objects(file_like, object_refs, url)
return self._write_multiple_objects(file_like, object_refs,
owner_addresses, url)
def restore_spilled_objects(self, object_refs: List[ObjectRef],
url_with_offset_list: List[str]):
@@ -352,13 +371,16 @@ class ExternalStorageSmartOpenImpl(ExternalStorage):
# smart open seek reads the file from offset-end_of_the_file
# when the seek is called.
f.seek(offset)
address_len = int.from_bytes(f.read(8), byteorder="little")
metadata_len = int.from_bytes(f.read(8), byteorder="little")
buf_len = int.from_bytes(f.read(8), byteorder="little")
self._size_check(metadata_len, buf_len, parsed_result.size)
owner_address = f.read(address_len)
total += buf_len
metadata = f.read(metadata_len)
# read remaining data to our buffer
self._put_object_to_store(metadata, buf_len, f, object_ref)
self._put_object_to_store(metadata, buf_len, f, object_ref,
owner_address)
return total
def delete_spilled_objects(self, urls: List[str]):
@@ -397,16 +419,17 @@ def reset_external_storage():
_external_storage = NullStorage()
def spill_objects(object_refs):
def spill_objects(object_refs, owner_addresses):
"""Spill objects to the external storage. Objects are specified
by their object refs.
Args:
object_refs: The list of the refs of the objects to be spilled.
owner_addresses: The owner addresses of the provided object refs.
Returns:
A list of keys corresponding to the input object refs.
"""
return _external_storage.spill_objects(object_refs)
return _external_storage.spill_objects(object_refs, owner_addresses)
def restore_spilled_objects(object_refs: List[ObjectRef],
python/ray/includes/libcoreworker.pxd
+3 -1
View File
@@ -241,7 +241,9 @@ cdef extern from "ray/core_worker/core_worker.h" nogil:
(void(const CWorkerID &) nogil) on_worker_shutdown
(CRayStatus() nogil) check_signals
(void() nogil) gc_collect
(c_vector[c_string](const c_vector[CObjectID] &) nogil) spill_objects
(c_vector[c_string](
const c_vector[CObjectID] &,
const c_vector[c_string] &) nogil) spill_objects
(int64_t(
const c_vector[CObjectID] &,
const c_vector[c_string] &) nogil) restore_spilled_objects
python/ray/tests/test_object_spilling.py
+2 -1
View File
@@ -564,7 +564,8 @@ def test_delete_objects_on_worker_failure(object_spilling_config,
@pytest.mark.skipif(
platform.system() == "Windows", reason="Failing on Windows.")
platform.system() in ["Windows", "Darwin"],
reason="Failing on Windows and MacOS.")
def test_delete_objects_multi_node(multi_node_object_spilling_config,
ray_start_cluster):
# Limit our object store to 75 MiB of memory.