wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-29 08:49:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Streaming data transfer supports cross language. (#7961)

Browse Source 
* add init parameters for java

* fix bug

* cython

* fix compile

* fix test_direct_tranfer

* comment

* ChannelCreationParameter

* fix comment

* builder

* lint and fix tests

* fix single process test

* fix checkstyle and lint

* checkstyle

* lint python

Co-authored-by: wanxing <wanxing@B-458DMD6M-1753.local>
This commit is contained in:
wanxing
2020-04-16 15:16:48 +08:00
committed by GitHub
parent 5a7882bb44
commit 9345d03ffb
36 changed files with 618 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/python/tests/test_direct_transfer.py
+18 -14
View File
@@ -12,20 +12,8 @@ from ray.streaming.config import Config
@ray.remote
class Worker:
def __init__(self):
writer_async_func = PythonFunctionDescriptor(
__name__, self.on_writer_message.__name__, self.__class__.__name__)
writer_sync_func = PythonFunctionDescriptor(
__name__, self.on_writer_message_sync.__name__,
self.__class__.__name__)
self.writer_client = _streaming.WriterClient(writer_async_func,
writer_sync_func)
reader_async_func = PythonFunctionDescriptor(
__name__, self.on_reader_message.__name__, self.__class__.__name__)
reader_sync_func = PythonFunctionDescriptor(
__name__, self.on_reader_message_sync.__name__,
self.__class__.__name__)
self.reader_client = _streaming.ReaderClient(reader_async_func,
reader_sync_func)
self.writer_client = _streaming.WriterClient()
self.reader_client = _streaming.ReaderClient()
self.writer = None
self.output_channel_id = None
self.reader = None
@@ -35,6 +23,14 @@ class Worker:
Config.TASK_JOB_ID: ray.worker.global_worker.current_job_id,
Config.CHANNEL_TYPE: Config.NATIVE_CHANNEL
}
reader_async_func = PythonFunctionDescriptor(
__name__, self.on_reader_message.__name__, self.__class__.__name__)
reader_sync_func = PythonFunctionDescriptor(
__name__, self.on_reader_message_sync.__name__,
self.__class__.__name__)
transfer.ChannelCreationParametersBuilder.\
set_python_reader_function_descriptor(
reader_async_func, reader_sync_func)
self.writer = transfer.DataWriter([output_channel],
[pickle.loads(reader_actor)], conf)
self.output_channel_id = transfer.ChannelID(output_channel)
@@ -44,6 +40,14 @@ class Worker:
Config.TASK_JOB_ID: ray.worker.global_worker.current_job_id,
Config.CHANNEL_TYPE: Config.NATIVE_CHANNEL
}
writer_async_func = PythonFunctionDescriptor(
__name__, self.on_writer_message.__name__, self.__class__.__name__)
writer_sync_func = PythonFunctionDescriptor(
__name__, self.on_writer_message_sync.__name__,
self.__class__.__name__)
transfer.ChannelCreationParametersBuilder.\
set_python_writer_function_descriptor(
writer_async_func, writer_sync_func)
self.reader = transfer.DataReader([input_channel],
[pickle.loads(writer_actor)], conf)