wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-09 11:43:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Implement streaming job-worker. (#8780)

Browse Source
This commit is contained in:
Tianyi Chen
2020-06-10 14:13:55 +08:00
committed by GitHub
parent 04cffb7e65
commit ec5ecb661f
56 changed files with 1078 additions and 1213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/python/runtime/worker.py
+26 -19
View File
@@ -5,7 +5,7 @@ import ray.streaming._streaming as _streaming
import ray.streaming.generated.remote_call_pb2 as remote_call_pb
import ray.streaming.runtime.processor as processor
from ray.streaming.config import Config
from ray.streaming.runtime.graph import ExecutionGraph
from ray.streaming.runtime.graph import ExecutionVertexContext
from ray.streaming.runtime.task import SourceStreamTask, OneInputStreamTask
logger = logging.getLogger(__name__)
@@ -21,43 +21,50 @@ class JobWorker(object):
def __init__(self):
self.worker_context = None
self.task_id = None
self.execution_vertex_context = None
self.config = None
self.execution_graph = None
self.execution_task = None
self.execution_node = None
self.stream_processor = None
self.task_id = None
self.task = None
self.stream_processor = None
self.reader_client = None
self.writer_client = None
logger.info("Creating job worker succeeded.")
def init(self, worker_context_bytes):
worker_context = remote_call_pb.WorkerContext()
worker_context = remote_call_pb.PythonJobWorkerContext()
worker_context.ParseFromString(worker_context_bytes)
self.worker_context = worker_context
self.task_id = worker_context.task_id
self.config = worker_context.conf
execution_graph = ExecutionGraph(worker_context.graph)
self.execution_graph = execution_graph
self.execution_task = self.execution_graph. \
get_execution_task_by_task_id(self.task_id)
self.execution_node = self.execution_graph. \
get_execution_node_by_task_id(self.task_id)
operator = self.execution_node.stream_operator
# build vertex context from pb
self.execution_vertex_context = ExecutionVertexContext(
worker_context.execution_vertex_context)
# use vertex id as task id
self.task_id = self.execution_vertex_context.get_task_id()
# build and get processor from operator
operator = self.execution_vertex_context.stream_operator
self.stream_processor = processor.build_processor(operator)
logger.info(
"Initializing JobWorker, task_id: {}, operator: {}.".format(
"Initializing job worker, task_id: {}, operator: {}.".format(
self.task_id, self.stream_processor))
# get config from vertex
self.config = self.execution_vertex_context.config
if self.config.get(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL):
self.reader_client = _streaming.ReaderClient()
self.writer_client = _streaming.WriterClient()
self.task = self.create_stream_task()
self.task.start()
logger.info("JobWorker init succeed")
logger.info("Job worker init succeeded.")
return True
def start(self):
self.task.start()
logger.info("Job worker start succeeded.")
def create_stream_task(self):
if isinstance(self.stream_processor, processor.SourceProcessor):
return SourceStreamTask(self.task_id, self.stream_processor, self)