Support multiple core workers in one process (#7623)

streaming/python/includes/libstreaming.pxd

@@ -31,7 +31,6 @@ from ray.includes.unique_ids cimport (
     CTaskID,
     CObjectID,
 )
-from ray.includes.libcoreworker cimport CCoreWorker
 
 cdef extern from "status.h" namespace "ray::streaming" nogil:
     cdef cppclass CStreamingStatus "ray::streaming::StreamingStatus":
@@ -100,15 +99,13 @@ cdef extern from "message/message_bundle.h" namespace "ray::streaming" nogil:
 
 cdef extern from "queue/queue_client.h" namespace "ray::streaming" nogil:
     cdef cppclass CReaderClient "ray::streaming::ReaderClient":
-        CReaderClient(CCoreWorker *core_worker,
-                      CRayFunction &async_func,
+        CReaderClient(CRayFunction &async_func,
                       CRayFunction &sync_func)
         void OnReaderMessage(shared_ptr[CLocalMemoryBuffer] buffer);
         shared_ptr[CLocalMemoryBuffer] OnReaderMessageSync(shared_ptr[CLocalMemoryBuffer] buffer);
 
     cdef cppclass CWriterClient "ray::streaming::WriterClient":
-        CWriterClient(CCoreWorker *core_worker,
-                      CRayFunction &async_func,
+        CWriterClient(CRayFunction &async_func,
                       CRayFunction &sync_func)
         void OnWriterMessage(shared_ptr[CLocalMemoryBuffer] buffer);
         shared_ptr[CLocalMemoryBuffer] OnWriterMessageSync(shared_ptr[CLocalMemoryBuffer] buffer);

streaming/python/includes/transfer.pxi

@@ -19,14 +19,11 @@ from ray.includes.unique_ids cimport (
 )
 from ray._raylet cimport (
     Buffer,
-    CoreWorker,
     ActorID,
     ObjectID,
     FunctionDescriptor,
 )
 
-from ray.includes.libcoreworker cimport CCoreWorker
-
 cimport ray.streaming.includes.libstreaming as libstreaming
 from ray.streaming.includes.libstreaming cimport (
     CStreamingStatus,
@@ -52,16 +49,14 @@ cdef class ReaderClient:
         CReaderClient *client
 
     def __cinit__(self,
-                  CoreWorker worker,
                   FunctionDescriptor async_func,
                   FunctionDescriptor sync_func):
         cdef:
-            CCoreWorker *core_worker = worker.core_worker.get()
             CRayFunction async_native_func
             CRayFunction sync_native_func
         async_native_func = CRayFunction(LANGUAGE_PYTHON, async_func.descriptor)
         sync_native_func = CRayFunction(LANGUAGE_PYTHON, sync_func.descriptor)
-        self.client = new CReaderClient(core_worker, async_native_func, sync_native_func)
+        self.client = new CReaderClient(async_native_func, sync_native_func)
 
     def __dealloc__(self):
         del self.client
@@ -91,16 +86,14 @@ cdef class WriterClient:
         CWriterClient * client
 
     def __cinit__(self,
-                  CoreWorker worker,
                   FunctionDescriptor async_func,
                   FunctionDescriptor sync_func):
         cdef:
-            CCoreWorker *core_worker = worker.core_worker.get()
             CRayFunction async_native_func
             CRayFunction sync_native_func
         async_native_func = CRayFunction(LANGUAGE_PYTHON, async_func.descriptor)
         sync_native_func = CRayFunction(LANGUAGE_PYTHON, sync_func.descriptor)
-        self.client = new CWriterClient(core_worker, async_native_func, sync_native_func)
+        self.client = new CWriterClient(async_native_func, sync_native_func)
 
     def __dealloc__(self):
         del self.client

streaming/python/runtime/worker.py

@@ -48,7 +48,6 @@ class JobWorker(object):
                 self.task_id, self.stream_processor))
 
         if self.config.get(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL):
-            core_worker = ray.worker.global_worker.core_worker
             reader_async_func = PythonFunctionDescriptor(
                 __name__, self.on_reader_message.__name__,
                 self.__class__.__name__)
@@ -56,7 +55,7 @@ class JobWorker(object):
                 __name__, self.on_reader_message_sync.__name__,
                 self.__class__.__name__)
             self.reader_client = _streaming.ReaderClient(
-                core_worker, reader_async_func, reader_sync_func)
+                reader_async_func, reader_sync_func)
             writer_async_func = PythonFunctionDescriptor(
                 __name__, self.on_writer_message.__name__,
                 self.__class__.__name__)
@@ -64,7 +63,7 @@ class JobWorker(object):
                 __name__, self.on_writer_message_sync.__name__,
                 self.__class__.__name__)
             self.writer_client = _streaming.WriterClient(
-                core_worker, writer_async_func, writer_sync_func)
+                writer_async_func, writer_sync_func)
 
         self.task = self.create_stream_task()
         self.task.start()

streaming/python/tests/test_direct_transfer.py

@@ -12,21 +12,20 @@ from ray.streaming.config import Config
 @ray.remote
 class Worker:
     def __init__(self):
-        core_worker = ray.worker.global_worker.core_worker
         writer_async_func = PythonFunctionDescriptor(
             __name__, self.on_writer_message.__name__, self.__class__.__name__)
         writer_sync_func = PythonFunctionDescriptor(
             __name__, self.on_writer_message_sync.__name__,
             self.__class__.__name__)
-        self.writer_client = _streaming.WriterClient(
-            core_worker, writer_async_func, writer_sync_func)
+        self.writer_client = _streaming.WriterClient(writer_async_func,
+                                                     writer_sync_func)
         reader_async_func = PythonFunctionDescriptor(
             __name__, self.on_reader_message.__name__, self.__class__.__name__)
         reader_sync_func = PythonFunctionDescriptor(
             __name__, self.on_reader_message_sync.__name__,
             self.__class__.__name__)
-        self.reader_client = _streaming.ReaderClient(
-            core_worker, reader_async_func, reader_sync_func)
+        self.reader_client = _streaming.ReaderClient(reader_async_func,
+                                                     reader_sync_func)
         self.writer = None
         self.output_channel_id = None
         self.reader = None