wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-29 13:32:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Plasma fetch (#36)

Browse Source 
* Fetch objects from remote plasma instances. Includes:
  - Configurable number of retries, with a timeout per retry
  - Support for multiple-object fetch
  - Test cases (integration only)

Update ray common

Remove attempts to retry object table lookup during fetch

lint

Fix a couple valgrind errors

Fix valgrind errors

Address Robert and Philipp's comments.

* Add fix from ray common and some TODOs.

* Update ray common

* Update ray common again

* Remove unused file
This commit is contained in:
Stephanie Wang
2016-10-18 18:20:59 -07:00
committed by Philipp Moritz
parent f189ca746b
commit ddfbd70dad
13 changed files with 1012 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/python/plasma.py
+32 -14
View File
@@ -42,7 +42,7 @@ class PlasmaClient(object):
plasma_client_library = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../build/plasma_client.so")
self.client = ctypes.cdll.LoadLibrary(plasma_client_library)
self.client.plasma_store_connect.restype = ctypes.c_void_p
self.client.plasma_connect.restype = ctypes.c_void_p
self.client.plasma_create.restype = None
self.client.plasma_get.restype = None
self.client.plasma_contains.restype = None
@@ -58,12 +58,12 @@ class PlasmaClient(object):
self.buffer_from_read_write_memory.argtypes = [ctypes.c_void_p, ctypes.c_int64]
self.buffer_from_read_write_memory.restype = ctypes.py_object
self.store_conn = ctypes.c_void_p(self.client.plasma_store_connect(socket_name))
if addr is not None and port is not None:
self.manager_conn = self.client.plasma_manager_connect(addr, port)
self.has_manager_conn = True
self.plasma_conn = ctypes.c_void_p(self.client.plasma_connect(socket_name, addr, port))
else:
self.manager_conn = -1 # not connected
self.has_manager_conn = False
self.plasma_conn = ctypes.c_void_p(self.client.plasma_connect(socket_name, None, 0))
def create(self, object_id, size, metadata=None):
"""Create a new buffer in the PlasmaStore for a particular object ID.
@@ -81,7 +81,7 @@ class PlasmaClient(object):
# Turn the metadata into the right type.
metadata = buffer("") if metadata is None else metadata
metadata = (ctypes.c_ubyte * len(metadata)).from_buffer_copy(metadata)
self.client.plasma_create(self.store_conn, make_plasma_id(object_id), size, ctypes.cast(metadata, ctypes.POINTER(ctypes.c_ubyte * len(metadata))), len(metadata), ctypes.byref(data))
self.client.plasma_create(self.plasma_conn, make_plasma_id(object_id), size, ctypes.cast(metadata, ctypes.POINTER(ctypes.c_ubyte * len(metadata))), len(metadata), ctypes.byref(data))
return self.buffer_from_read_write_memory(data, size)
def get(self, object_id):
@@ -97,7 +97,7 @@ class PlasmaClient(object):
data = ctypes.c_void_p()
metadata_size = ctypes.c_int64()
metadata = ctypes.c_void_p()
buf = self.client.plasma_get(self.store_conn, make_plasma_id(object_id), ctypes.byref(size), ctypes.byref(data), ctypes.byref(metadata_size), ctypes.byref(metadata))
buf = self.client.plasma_get(self.plasma_conn, make_plasma_id(object_id), ctypes.byref(size), ctypes.byref(data), ctypes.byref(metadata_size), ctypes.byref(metadata))
return self.buffer_from_memory(data, size)
def get_metadata(self, object_id):
@@ -113,7 +113,7 @@ class PlasmaClient(object):
data = ctypes.c_void_p()
metadata_size = ctypes.c_int64()
metadata = ctypes.c_void_p()
buf = self.client.plasma_get(self.store_conn, make_plasma_id(object_id), ctypes.byref(size), ctypes.byref(data), ctypes.byref(metadata_size), ctypes.byref(metadata))
buf = self.client.plasma_get(self.plasma_conn, make_plasma_id(object_id), ctypes.byref(size), ctypes.byref(data), ctypes.byref(metadata_size), ctypes.byref(metadata))
return self.buffer_from_memory(metadata, metadata_size)
def contains(self, object_id):
@@ -123,7 +123,7 @@ class PlasmaClient(object):
object_id (str): A string used to identify an object.
"""
has_object = ctypes.c_int()
self.client.plasma_contains(self.store_conn, make_plasma_id(object_id), ctypes.byref(has_object))
self.client.plasma_contains(self.plasma_conn, make_plasma_id(object_id), ctypes.byref(has_object))
has_object = has_object.value
if has_object == 1:
return True
@@ -141,7 +141,7 @@ class PlasmaClient(object):
Args:
object_id (str): A string used to identify an object.
"""
self.client.plasma_seal(self.store_conn, make_plasma_id(object_id))
self.client.plasma_seal(self.plasma_conn, make_plasma_id(object_id))
def delete(self, object_id):
"""Delete the buffer in the PlasmaStore for a particular object ID.
@@ -151,7 +151,7 @@ class PlasmaClient(object):
Args:
object_id (str): A string used to identify an object.
"""
self.client.plasma_delete(self.store_conn, make_plasma_id(object_id))
self.client.plasma_delete(self.plasma_conn, make_plasma_id(object_id))
def transfer(self, addr, port, object_id):
"""Transfer local object with id object_id to another plasma instance
@@ -161,13 +161,31 @@ class PlasmaClient(object):
port (int): Port number of the plasma instance the object is sent to.
object_id (str): A string used to identify an object.
"""
if self.manager_conn == -1:
if not self.has_manager_conn:
raise Exception("Not connected to the plasma manager socket")
self.client.plasma_transfer(self.manager_conn, addr, port, make_plasma_id(object_id))
self.client.plasma_transfer(self.plasma_conn, addr, port, make_plasma_id(object_id))
def fetch(self, object_ids):
"""Fetch the object with id object_id from another plasma manager instance.
Args:
object_id (str): A string used to identify an object.
"""
object_id_array = (len(object_ids) * PlasmaID)()
for i, object_id in enumerate(object_ids):
object_id_array[i] = make_plasma_id(object_id)
success_array = (len(object_ids) * ctypes.c_int)()
if not self.has_manager_conn:
raise Exception("Not connected to the plasma manager socket")
self.client.plasma_fetch(self.plasma_conn,
object_id_array._length_,
object_id_array,
success_array);
return [bool(success) for success in success_array]
def subscribe(self):
"""Subscribe to notifications about sealed objects."""
fd = self.client.plasma_subscribe(self.store_conn)
fd = self.client.plasma_subscribe(self.plasma_conn)
self.notification_sock = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM)
# Make the socket non-blocking.
self.notification_sock.setblocking(0)