wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-29 07:41:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Streaming data transfer supports cross language. (#7961)

Browse Source 
* add init parameters for java

* fix bug

* cython

* fix compile

* fix test_direct_tranfer

* comment

* ChannelCreationParameter

* fix comment

* builder

* lint and fix tests

* fix single process test

* fix checkstyle and lint

* checkstyle

* lint python

Co-authored-by: wanxing <wanxing@B-458DMD6M-1753.local>
This commit is contained in:
wanxing
2020-04-16 15:16:48 +08:00
committed by GitHub
parent 5a7882bb44
commit 9345d03ffb
36 changed files with 618 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/python/includes/libstreaming.pxd
+11 -6
View File
@@ -97,16 +97,21 @@ cdef extern from "message/message_bundle.h" namespace "ray::streaming" nogil:
void GetMessageListFromRawData(const uint8_t *data, uint32_t size, uint32_t msg_nums,
c_list[shared_ptr[CStreamingMessage]] &msg_list);
cdef extern from "channel.h" namespace "ray::streaming" nogil:
cdef struct CChannelCreationParameter "ray::streaming::ChannelCreationParameter":
CChannelCreationParameter()
CActorID actor_id;
shared_ptr[CRayFunction] async_function;
shared_ptr[CRayFunction] sync_function;
cdef extern from "queue/queue_client.h" namespace "ray::streaming" nogil:
cdef cppclass CReaderClient "ray::streaming::ReaderClient":
CReaderClient(CRayFunction &async_func,
CRayFunction &sync_func)
CReaderClient()
void OnReaderMessage(shared_ptr[CLocalMemoryBuffer] buffer);
shared_ptr[CLocalMemoryBuffer] OnReaderMessageSync(shared_ptr[CLocalMemoryBuffer] buffer);
cdef cppclass CWriterClient "ray::streaming::WriterClient":
CWriterClient(CRayFunction &async_func,
CRayFunction &sync_func)
CWriterClient()
void OnWriterMessage(shared_ptr[CLocalMemoryBuffer] buffer);
shared_ptr[CLocalMemoryBuffer] OnWriterMessageSync(shared_ptr[CLocalMemoryBuffer] buffer);
@@ -122,7 +127,7 @@ cdef extern from "data_reader.h" namespace "ray::streaming" nogil:
cdef cppclass CDataReader "ray::streaming::DataReader"(CStreamingCommon):
CDataReader(shared_ptr[CRuntimeContext] &runtime_context)
void Init(const c_vector[CObjectID] &input_ids,
const c_vector[CActorID] &actor_ids,
const c_vector[CChannelCreationParameter] &params,
const c_vector[uint64_t] &seq_ids,
const c_vector[uint64_t] &msg_ids,
int64_t timer_interval);
@@ -135,7 +140,7 @@ cdef extern from "data_writer.h" namespace "ray::streaming" nogil:
cdef cppclass CDataWriter "ray::streaming::DataWriter"(CStreamingCommon):
CDataWriter(shared_ptr[CRuntimeContext] &runtime_context)
CStreamingStatus Init(const c_vector[CObjectID] &channel_ids,
const c_vector[CActorID] &actor_ids,
const c_vector[CChannelCreationParameter] &params,
const c_vector[uint64_t] &message_ids,
const c_vector[uint64_t] &queue_size_vec);
long WriteMessageToBufferRing(
streaming/python/includes/transfer.pxi
+42 -28
View File
@@ -10,6 +10,7 @@ from libcpp.list cimport list as c_list
from ray.includes.common cimport (
CRayFunction,
LANGUAGE_PYTHON,
LANGUAGE_JAVA,
CBuffer
)
@@ -36,27 +37,43 @@ from ray.streaming.includes.libstreaming cimport (
CReaderClient,
CWriterClient,
CLocalMemoryBuffer,
CChannelCreationParameter,
)
from ray._raylet import JavaFunctionDescriptor
import logging
channel_logger = logging.getLogger(__name__)
cdef class ChannelCreationParameter:
cdef:
CChannelCreationParameter parameter
def __cinit__(self, ActorID actor_id, FunctionDescriptor async_func, FunctionDescriptor sync_func):
cdef:
shared_ptr[CRayFunction] async_func_ptr
shared_ptr[CRayFunction] sync_func_ptr
self.parameter = CChannelCreationParameter()
self.parameter.actor_id = (<ActorID>actor_id).data
if isinstance(async_func, JavaFunctionDescriptor):
self.parameter.async_function = make_shared[CRayFunction](LANGUAGE_JAVA, async_func.descriptor)
else:
self.parameter.async_function = make_shared[CRayFunction](LANGUAGE_PYTHON, async_func.descriptor)
if isinstance(sync_func, JavaFunctionDescriptor):
self.parameter.sync_function = make_shared[CRayFunction](LANGUAGE_JAVA, sync_func.descriptor)
else:
self.parameter.sync_function = make_shared[CRayFunction](LANGUAGE_PYTHON, sync_func.descriptor)
cdef CChannelCreationParameter get_parameter(self):
return self.parameter
cdef class ReaderClient:
cdef:
CReaderClient *client
def __cinit__(self,
FunctionDescriptor async_func,
FunctionDescriptor sync_func):
cdef:
CRayFunction async_native_func
CRayFunction sync_native_func
async_native_func = CRayFunction(LANGUAGE_PYTHON, async_func.descriptor)
sync_native_func = CRayFunction(LANGUAGE_PYTHON, sync_func.descriptor)
self.client = new CReaderClient(async_native_func, sync_native_func)
def __cinit__(self):
self.client = new CReaderClient()
def __dealloc__(self):
del self.client
@@ -85,15 +102,8 @@ cdef class WriterClient:
cdef:
CWriterClient * client
def __cinit__(self,
FunctionDescriptor async_func,
FunctionDescriptor sync_func):
cdef:
CRayFunction async_native_func
CRayFunction sync_native_func
async_native_func = CRayFunction(LANGUAGE_PYTHON, async_func.descriptor)
sync_native_func = CRayFunction(LANGUAGE_PYTHON, sync_func.descriptor)
self.client = new CWriterClient(async_native_func, sync_native_func)
def __cinit__(self):
self.client = new CWriterClient()
def __dealloc__(self):
del self.client
@@ -127,19 +137,21 @@ cdef class DataWriter:
@staticmethod
def create(list py_output_channels,
list output_actor_ids: list[ActorID],
list output_creation_parameters: list[ChannelCreationParameter],
uint64_t queue_size,
list py_msg_ids,
bytes config_bytes,
c_bool is_mock):
cdef:
c_vector[CObjectID] channel_ids = bytes_list_to_qid_vec(py_output_channels)
c_vector[CActorID] actor_ids
c_vector[CChannelCreationParameter] initial_parameters
c_vector[uint64_t] msg_ids
CDataWriter *c_writer
ChannelCreationParameter parameter
cdef const unsigned char[:] config_data
for actor_id in output_actor_ids:
actor_ids.push_back((<ActorID>actor_id).data)
for param in output_creation_parameters:
parameter = param
initial_parameters.push_back(parameter.get_parameter())
for py_msg_id in py_msg_ids:
msg_ids.push_back(<uint64_t>py_msg_id)
@@ -156,7 +168,7 @@ cdef class DataWriter:
c_vector[uint64_t] queue_size_vec
for i in range(channel_ids.size()):
queue_size_vec.push_back(queue_size)
cdef CStreamingStatus status = c_writer.Init(channel_ids, actor_ids, msg_ids, queue_size_vec)
cdef CStreamingStatus status = c_writer.Init(channel_ids, initial_parameters, msg_ids, queue_size_vec)
if remain_id_vec.size() != 0:
channel_logger.warning("failed queue amounts => %s", remain_id_vec.size())
if <uint32_t>status != <uint32_t> libstreaming.StatusOK:
@@ -205,7 +217,7 @@ cdef class DataReader:
@staticmethod
def create(list py_input_queues,
list input_actor_ids: list[ActorID],
list input_creation_parameters: list[ChannelCreationParameter],
list py_seq_ids,
list py_msg_ids,
int64_t timer_interval,
@@ -214,13 +226,15 @@ cdef class DataReader:
c_bool is_mock):
cdef:
c_vector[CObjectID] queue_id_vec = bytes_list_to_qid_vec(py_input_queues)
c_vector[CActorID] actor_ids
c_vector[CChannelCreationParameter] initial_parameters
c_vector[uint64_t] seq_ids
c_vector[uint64_t] msg_ids
CDataReader *c_reader
ChannelCreationParameter parameter
cdef const unsigned char[:] config_data
for actor_id in input_actor_ids:
actor_ids.push_back((<ActorID>actor_id).data)
for param in input_creation_parameters:
parameter = param
initial_parameters.push_back(parameter.get_parameter())
for py_seq_id in py_seq_ids:
seq_ids.push_back(<uint64_t>py_seq_id)
for py_msg_id in py_msg_ids:
@@ -233,7 +247,7 @@ cdef class DataReader:
if is_mock:
ctx.get().MarkMockTest()
c_reader = new CDataReader(ctx)
c_reader.Init(queue_id_vec, actor_ids, seq_ids, msg_ids, timer_interval)
c_reader.Init(queue_id_vec, initial_parameters, seq_ids, msg_ids, timer_interval)
channel_logger.info("create native reader succeed")
cdef DataReader reader = DataReader.__new__(DataReader)
reader.reader = c_reader