wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 23:08:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix streaming ci (#8159)

Browse Source
This commit is contained in:
chaokunyang
2020-04-26 20:56:58 +08:00
committed by GitHub
parent 5bff707d20
commit 5cf49d5edd
18 changed files with 68 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/python/config.py
-1
View File
@@ -1,7 +1,6 @@
class Config:
STREAMING_JOB_NAME = "streaming.job.name"
STREAMING_OP_NAME = "streaming.op_name"
TASK_JOB_ID = "streaming.task_job_id"
STREAMING_WORKER_NAME = "streaming.worker_name"
# channel
CHANNEL_TYPE = "channel_type"
streaming/python/runtime/task.py
-3
View File
@@ -3,7 +3,6 @@ import pickle
import threading
from abc import ABC, abstractmethod
import ray
from ray.streaming.collector import OutputCollector
from ray.streaming.config import Config
from ray.streaming.context import RuntimeContextImpl
@@ -31,8 +30,6 @@ class StreamTask(ABC):
self.worker.config.get(Config.CHANNEL_SIZE,
Config.CHANNEL_SIZE_DEFAULT))
channel_conf[Config.CHANNEL_SIZE] = channel_size
channel_conf[Config.TASK_JOB_ID] = ray.runtime_context.\
_get_runtime_context().current_driver_id
channel_conf[Config.CHANNEL_TYPE] = self.worker.config \
.get(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL)
streaming/python/runtime/transfer.py
+17 -29
View File
@@ -147,29 +147,21 @@ class ChannelCreationParametersBuilder:
wrap initial parameters needed by a streaming queue
"""
_java_reader_async_function_descriptor = JavaFunctionDescriptor(
"org.ray.streaming.runtime.worker",
"onReaderMessage", "([B)V")
"io.ray.streaming.runtime.worker", "onReaderMessage", "([B)V")
_java_reader_sync_function_descriptor = JavaFunctionDescriptor(
"org.ray.streaming.runtime.worker",
"onReaderMessageSync", "([B)[B")
"io.ray.streaming.runtime.worker", "onReaderMessageSync", "([B)[B")
_java_writer_async_function_descriptor = JavaFunctionDescriptor(
"org.ray.streaming.runtime.worker",
"onWriterMessage", "([B)V")
"io.ray.streaming.runtime.worker", "onWriterMessage", "([B)V")
_java_writer_sync_function_descriptor = JavaFunctionDescriptor(
"org.ray.streaming.runtime.worker",
"onWriterMessageSync", "([B)[B")
"io.ray.streaming.runtime.worker", "onWriterMessageSync", "([B)[B")
_python_reader_async_function_descriptor = PythonFunctionDescriptor(
"ray.streaming.runtime.core.worker",
"on_reader_message", "JobWorker")
"ray.streaming.runtime.worker", "on_reader_message", "JobWorker")
_python_reader_sync_function_descriptor = PythonFunctionDescriptor(
"ray.streaming.runtime.core.worker",
"on_reader_message_sync", "JobWorker")
"ray.streaming.runtime.worker", "on_reader_message_sync", "JobWorker")
_python_writer_async_function_descriptor = PythonFunctionDescriptor(
"ray.streaming.runtime.core.worker",
"on_writer_message", "JobWorker")
"ray.streaming.runtime.worker", "on_writer_message", "JobWorker")
_python_writer_sync_function_descriptor = PythonFunctionDescriptor(
"ray.streaming.runtime.core.worker",
"on_writer_message_sync", "JobWorker")
"ray.streaming.runtime.worker", "on_writer_message_sync", "JobWorker")
def get_parameters(self):
return self._parameters
@@ -193,8 +185,8 @@ class ChannelCreationParametersBuilder:
self._python_reader_sync_function_descriptor)
return self
def build_parameters(self, actors, java_async_func,
java_sync_func, py_async_func, py_sync_func):
def build_parameters(self, actors, java_async_func, java_sync_func,
py_async_func, py_sync_func):
for handle in actors:
parameter = None
if handle._ray_actor_language == Language.PYTHON:
@@ -208,16 +200,16 @@ class ChannelCreationParametersBuilder:
@staticmethod
def set_python_writer_function_descriptor(async_function, sync_function):
ChannelCreationParametersBuilder.\
ChannelCreationParametersBuilder. \
_python_writer_async_function_descriptor = async_function
ChannelCreationParametersBuilder.\
ChannelCreationParametersBuilder. \
_python_writer_sync_function_descriptor = sync_function
@staticmethod
def set_python_reader_function_descriptor(async_function, sync_function):
ChannelCreationParametersBuilder.\
ChannelCreationParametersBuilder. \
_python_reader_async_function_descriptor = async_function
ChannelCreationParametersBuilder.\
ChannelCreationParametersBuilder. \
_python_reader_sync_function_descriptor = sync_function
@@ -251,8 +243,7 @@ class DataWriter:
is_mock = conf[Config.CHANNEL_TYPE] == Config.MEMORY_CHANNEL
self.writer = _streaming.DataWriter.create(
py_output_channels, creation_parameters.get_parameters(),
channel_size, py_msg_ids,
config_bytes, is_mock)
channel_size, py_msg_ids, config_bytes, is_mock)
logger.info("create DataWriter succeed")
@@ -307,8 +298,8 @@ class DataReader:
is_mock = conf[Config.CHANNEL_TYPE] == Config.MEMORY_CHANNEL
self.reader = _streaming.DataReader.create(
py_input_channels, creation_parameters.get_parameters(),
py_seq_ids, py_msg_ids,
timer_interval, is_recreate, config_bytes, is_mock)
py_seq_ids, py_msg_ids, timer_interval, is_recreate, config_bytes,
is_mock)
logger.info("create DataReader succeed")
def read(self, timeout_millis):
@@ -344,9 +335,6 @@ def _to_native_conf(conf):
config = streaming_pb.StreamingConfig()
if Config.STREAMING_JOB_NAME in conf:
config.job_name = conf[Config.STREAMING_JOB_NAME]
if Config.TASK_JOB_ID in conf:
job_id = conf[Config.TASK_JOB_ID]
config.task_job_id = job_id.hex()
if Config.STREAMING_WORKER_NAME in conf:
config.worker_name = conf[Config.STREAMING_WORKER_NAME]
if Config.STREAMING_OP_NAME in conf:
streaming/python/tests/test_direct_transfer.py
+2 -8
View File
@@ -19,10 +19,7 @@ class Worker:
self.reader = None
def init_writer(self, output_channel, reader_actor):
conf = {
Config.TASK_JOB_ID: ray.worker.global_worker.current_job_id,
Config.CHANNEL_TYPE: Config.NATIVE_CHANNEL
}
conf = {Config.CHANNEL_TYPE: Config.NATIVE_CHANNEL}
reader_async_func = PythonFunctionDescriptor(
__name__, self.on_reader_message.__name__, self.__class__.__name__)
reader_sync_func = PythonFunctionDescriptor(
@@ -36,10 +33,7 @@ class Worker:
self.output_channel_id = transfer.ChannelID(output_channel)
def init_reader(self, input_channel, writer_actor):
conf = {
Config.TASK_JOB_ID: ray.worker.global_worker.current_job_id,
Config.CHANNEL_TYPE: Config.NATIVE_CHANNEL
}
conf = {Config.CHANNEL_TYPE: Config.NATIVE_CHANNEL}
writer_async_func = PythonFunctionDescriptor(
__name__, self.on_writer_message.__name__, self.__class__.__name__)
writer_sync_func = PythonFunctionDescriptor(
streaming/python/tests/test_word_count.py
+34 -1
View File
@@ -1,3 +1,4 @@
import os
import ray
from ray.streaming import StreamingContext
@@ -21,5 +22,37 @@ def test_word_count():
ray.shutdown()
def test_simple_word_count():
ray.init(load_code_from_local=True, include_java=True)
ctx = StreamingContext.Builder() \
.build()
sink_file = "/tmp/ray_streaming_test_simple_word_count.txt"
if os.path.exists(sink_file):
os.remove(sink_file)
def sink_func(x):
with open(sink_file, "a") as f:
f.write("{}:{},".format(x[0], x[1]))
ctx.from_values("a", "b", "c") \
.set_parallelism(1) \
.flat_map(lambda x: [x, x]) \
.map(lambda x: (x, 1)) \
.key_by(lambda x: x[0]) \
.reduce(lambda old_value, new_value:
(old_value[0], old_value[1] + new_value[1])) \
.sink(sink_func)
ctx.submit("word_count")
import time
time.sleep(3)
ray.shutdown()
with open(sink_file, "r") as f:
result = f.read()
assert "a:2" in result
assert "b:2" in result
assert "c:2" in result
if __name__ == "__main__":
test_word_count()
# test_word_count()
test_simple_word_count()